<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\executables\Fetcher;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\Model;
/* Both FetcherTest and QueueServerTest load an executable that only
starts its process when this is not defined, and the suite loads both. */
if (!C\nsdefined("UNIT_TEST_MODE")) {
C\nsdefine("UNIT_TEST_MODE", true);
}
/**
* FetcherTest checks how a Fetcher decides when to send what it has
* found to a queue server.
*
* @author Chris Pollett
*/
class FetcherTest extends UnitTest
{
/**
* CLOSED_PORT is a port of this machine that nothing listens on, the
* discard port, so a download from it fails at once with no answer
*/
const CLOSED_PORT = 9;
/**
* LONGEST_HOLD is the seconds fetcher 5 held schedule 415 in the crawl
* logs of 18 October, the longest any fetcher held one there
*/
const LONGEST_HOLD = 683;
/**
* TEST_FETCHER is the fetcher number the restart case gives its
* fetchers, one no crawl uses, so its files are its own
*/
const TEST_FETCHER = 97;
/**
* TEST_CRAWL_TIME is the crawl timestamp the restart case gives its
* fetchers, used in the names of their files
*/
const TEST_CRAWL_TIME = 1700000099;
/**
* setUp makes SCHEDULES, the folder each case's fetcher keeps its
* schedules files in.
*/
public function setUp()
{
L\makePath(self::SCHEDULES);
}
/**
* tearDown deletes SCHEDULES, the folder the cases' fetchers keep
* their schedules files in.
*/
public function tearDown()
{
(new Model())->db->unlinkRecursive(self::SCHEDULES);
}
/**
* SCHEDULES is the folder, under tests/test_files, that the cases'
* fetchers keep their schedules files in.
*/
const SCHEDULES = C\TEST_DIR . "/test_files/fetcher_schedules";
/**
* passTakesManyUrlsOfFewHostsTestCase checks that a download pass of
* urls whose host addresses are not yet known stops at the allowance
* of hosts, not of urls. Three hundred urls of three hosts make one
* pass of NUM_MULTI_CURL_PAGES urls, and three hundred urls of three
* hundred hosts still stop at the allowance. The fetcher is made
* without its constructor, which would start a crawl process.
*/
public function passTakesManyUrlsOfFewHostsTestCase()
{
$fetcher = new class extends Fetcher {
/**
* __construct leaves the fetcher's settings at their defaults
* rather than starting a crawl process.
*/
public function __construct()
{
}
};
$fetcher->num_multi_curl = C\NUM_MULTI_CURL_PAGES;
$fetcher->hosts_with_errors = [];
$allowance = (int) ceil(C\NUM_MULTI_CURL_PAGES /
Fetcher::URLS_PER_HOST_LOOKUP);
foreach ([3 => C\NUM_MULTI_CURL_PAGES, 300 => $allowance] as
$num_hosts => $expected) {
$fetcher->to_crawl = [];
$fetcher->to_crawl_again = [];
for ($i = 0; $i < 300; $i++) {
$fetcher->to_crawl[] = ["https://host" . ($i % $num_hosts) .
".example.com/page$i.html", 1, 0, 0];
}
$this->assertEqual(count($fetcher->getFetchSites()), $expected,
"a pass of urls from $num_hosts hosts takes $expected urls");
}
}
/**
* firstSendComesEarlyAndTheGapDoublesTestCase checks that a web crawl's
* fetcher sends what it found once it has downloaded
* NUM_MULTI_CURL_PAGES urls, so the first pages of a crawl show up
* after the first download, and that each later send waits for twice
* as many downloads as the one before. Most urls in a download find
* nothing new, so a send is asked for by downloads, not by pages seen.
* The fetcher is made without its constructor, which would start a
* crawl process, and its send is replaced with a count of sends.
*/
public function firstSendComesEarlyAndTheGapDoublesTestCase()
{
$fetcher = new class extends Fetcher {
/**
* sends counts the calls a real fetcher would make to send.
* @var int
*/
public $sends = 0;
/**
* __construct leaves the fetcher's settings at their defaults
* rather than starting a crawl process.
*/
public function __construct()
{
}
/**
* selectCurrentServerAndUpdateIfNeeded counts a send and
* clears what would have been sent.
*
* @param bool $at_least_current_server whether a send must
* happen, as for the real method
*/
public function selectCurrentServerAndUpdateIfNeeded(
$at_least_current_server)
{
$this->sends++;
$this->found_sites[self::SEEN_URLS] = [];
}
};
$fetcher->crawl_type = Fetcher::WEB_CRAWL;
$fetcher->to_crawl = [["https://www.example.com/", 1, 0, 0]];
$fetcher->to_crawl_again = [];
$fetcher->archive_iterator = null;
$fetcher->which_fetcher = self::TEST_FETCHER;
$fetcher->schedules_folder = self::SCHEDULES;
$fetcher->channel = 0;
$fetcher->crawl_time = self::TEST_CRAWL_TIME;
$download = function ($count) use ($fetcher) {
$fetcher->found_sites[Fetcher::SEEN_URLS] = [[]];
$fetcher->updateFoundSites(array_fill(0, $count, []));
return $fetcher->sends;
};
$first = C\NUM_MULTI_CURL_PAGES;
$this->assertEqual($download($first), 1,
"the first send comes after one download of $first urls");
$this->assertEqual($download($first), 1,
"the next waits for more than $first more");
$this->assertEqual($download($first), 2,
"the second send comes after " . (2 * $first) . " more");
$download(2 * $first);
$this->assertEqual($download(2 * $first), 3,
"the third send comes after " . (4 * $first) . " more");
$this->assertEqual($fetcher->send_after, 8 * $first,
"the gap keeps doubling");
$download(8 * $first);
$this->assertEqual($fetcher->send_after,
Fetcher::SEEN_URLS_BEFORE_SEND,
"the gap stops at SEEN_URLS_BEFORE_SEND for a web crawl");
$download(Fetcher::SEEN_URLS_BEFORE_SEND);
$this->assertEqual($fetcher->sends, 5,
"a web crawl then sends every " .
Fetcher::SEEN_URLS_BEFORE_SEND . " downloads");
}
/**
* longHeldScheduleIsAnsweredTestCase checks that a web crawl's fetcher
* that has held its schedule for ANSWER_SCHEDULE_AFTER seconds sends
* what it has after its next download, though it has downloaded fewer
* urls than a send waits for. In the crawl of 18 October one fetcher
* held schedule 415 for 683 seconds on pages taking 4 seconds each,
* and the other fetchers slept until it came back. A schedule given
* just now waits for its downloads as before. The fetcher is made
* without its constructor, and its send is replaced with a count.
*/
public function longHeldScheduleIsAnsweredTestCase()
{
$fetcher = new class extends Fetcher {
/**
* sends counts the calls a real fetcher would make to send.
* @var int
*/
public $sends = 0;
/**
* __construct leaves the fetcher's settings at their defaults
* rather than starting a crawl process.
*/
public function __construct()
{
}
/**
* selectCurrentServerAndUpdateIfNeeded counts a send and
* clears what would have been sent.
*
* @param bool $at_least_current_server whether a send must
* happen, as for the real method
*/
public function selectCurrentServerAndUpdateIfNeeded(
$at_least_current_server)
{
$this->sends++;
$this->found_sites[self::SEEN_URLS] = [];
}
};
$fetcher->crawl_type = Fetcher::WEB_CRAWL;
$fetcher->to_crawl = [["https://www.example.com/", 1, 0, 0]];
$fetcher->to_crawl_again = [];
$fetcher->archive_iterator = null;
$fetcher->which_fetcher = self::TEST_FETCHER;
$fetcher->schedules_folder = self::SCHEDULES;
$fetcher->channel = 0;
$fetcher->crawl_time = self::TEST_CRAWL_TIME;
$fetcher->send_after = Fetcher::SEEN_URLS_BEFORE_SEND;
$fetcher->sequence_number = 415;
$fetcher->answered_sequence_number = 414;
$fetcher->schedule_received_time = time();
$fetcher->found_sites[Fetcher::SEEN_URLS] = [[]];
$fetcher->updateFoundSites(array_fill(0, 1, []));
$this->assertEqual($fetcher->sends, 0,
"a schedule given just now is not sent after one download");
$fetcher->schedule_received_time = time() - self::LONGEST_HOLD;
$fetcher->updateFoundSites(array_fill(0, 1, []));
$this->assertEqual($fetcher->sends, 1,
"a schedule held " . self::LONGEST_HOLD . " seconds is sent");
}
/**
* stoppedFetcherGoesOnWithoutItsPageTestCase checks what a fetcher
* restarted after it stopped while processing a page does with its
* schedule. In the crawl of 18 October a fetcher stopped on
* 2024LobbyGroupByLobbyist.xlsx three times: each restart dropped the
* schedule, the queue server waited out its timeout, then sent it to
* the next fetcher, which stopped on the same page. The restarted
* fetcher here must go on with the schedule's urls not yet sent back,
* under the same sequence number, without the page that stopped it.
* A schedule past the time the queue server sends it elsewhere is left
* alone. The fetchers are made without their constructors.
*/
public function stoppedFetcherGoesOnWithoutItsPageTestCase()
{
$make_fetcher = function () {
$fetcher = new class extends Fetcher {
/**
* __construct leaves the fetcher's settings at their
* defaults rather than starting a crawl process.
*/
public function __construct()
{
}
};
$fetcher->which_fetcher = self::TEST_FETCHER;
$fetcher->schedules_folder = self::SCHEDULES;
$fetcher->channel = 0;
$fetcher->crawl_time = self::TEST_CRAWL_TIME;
$fetcher->to_crawl = [];
$fetcher->to_crawl_again = [];
return $fetcher;
};
$stopping_url = "https://www.ethics.texas.gov/data/search/lobby/" .
"2024/2024LobbyGroupByLobbyist.xlsx";
$before = $make_fetcher();
$before->sequence_number = 860;
$before->schedule_resend_time = time() + C\ONE_MINUTE;
$before->to_crawl = [["https://www.example.com/a###1.2.3.4", 1, 0, 0],
["$stopping_url###5.6.7.8", 1, 0, 0],
["https://www.example.com/b", 1, 0, 0]];
$before->saveScheduleLeft();
file_put_contents($before->pageInProcessName(), $stopping_url);
$after = $make_fetcher();
$after->resumeScheduleLeft();
$this->assertEqual([$after->sequence_number,
array_column($after->to_crawl, 0)], [860,
["https://www.example.com/a###1.2.3.4",
"https://www.example.com/b"]],
"the restarted fetcher goes on with the schedule less the page");
$before->schedule_resend_time = time() - 1;
$before->saveScheduleLeft();
$late = $make_fetcher();
$late->resumeScheduleLeft();
$this->assertEqual($late->to_crawl, [],
"a schedule past its resend time is left to another fetcher");
}
/**
* pageWithNoAnswerIsSentBackTestCase checks that a page that gets no
* answer in both of its tries in a batch is sent back to the queue
* server to be fetched again, rather than dropped, as a seed page once
* was. The page's url is on a port of this machine nothing listens on,
* so its download fails at once. The fetcher downloads twice: first
* the batch, after which the page waits for its second try, then the
* pages that failed. Its send is replaced with nothing.
*/
public function pageWithNoAnswerIsSentBackTestCase()
{
$fetcher = new class extends Fetcher {
/**
* __construct leaves the fetcher's settings at their defaults
* rather than starting a crawl process.
*/
public function __construct()
{
}
/**
* selectCurrentServerAndUpdateIfNeeded sends nothing, so what
* would be sent stays to be read.
*
* @param bool $at_least_current_server whether a send must
* happen, as for the real method
*/
public function selectCurrentServerAndUpdateIfNeeded(
$at_least_current_server)
{
}
};
$url = "http://127.0.0.1:" . self::CLOSED_PORT . "/seed.html";
$fetcher->crawl_type = Fetcher::WEB_CRAWL;
$fetcher->queue_servers = ["http://localhost/"];
$fetcher->to_crawl = [[$url, 1, 0, 0]];
$fetcher->to_crawl_again = [];
$fetcher->found_sites = [];
$fetcher->num_multi_curl = C\NUM_MULTI_CURL_PAGES;
$fetcher->downloadPagesWebCrawl();
$this->assertEqual(count($fetcher->to_crawl_again), 1,
"after the first try the page waits for its second");
$fetcher->downloadPagesWebCrawl();
$sent_back = array_column(
$fetcher->found_sites[Fetcher::RETRY_URLS][0] ?? [], 0);
$this->assertEqual($sent_back, [$url],
"after the second the page is sent back to the queue server");
}
/**
* challengedHomePageIsIndexedByUrlWordsTestCase checks how a page that
* is a bot-protection challenge is processed. The page is the one
* https://www.pepsi.com/ sent on 21 October 2026, a 403 from Imperva
* Incapsula with a robots meta tag of NOINDEX, NOFOLLOW. In a crawl
* that allows landing pages, the home page is only sent back to be
* fetched again while the Scheduler has tries left. On the Scheduler's
* last try, which the schedule lists in LAST_TRY_URLS, the home page
* is indexed by the words of its url and still sent back once. A page
* below the home page, or the home page in a crawl that follows
* robots.txt, is only sent back, even on its last try. A home page
* that answers with no text, as a page drawn by scripts does, keeps
* its title and is described by the words of its url.
*/
public function challengedHomePageIsIndexedByUrlWordsTestCase()
{
$challenge = '<html style="height:100%"><head><META NAME="ROBOTS" ' .
'CONTENT="NOINDEX, NOFOLLOW"><script type="text/javascript" ' .
'src="/_Incapsula_Resource?SWJIYLWA=719d34d31c8e3a6e6fffd425f7e' .
'032f3"></script></head><body style="margin:0px;height:100%">' .
'<iframe id="main-iframe" src="/_Incapsula_Resource?CWUDNSAI=' .
'23" frameborder=0 width="100%" height="100%">Request ' .
'unsuccessful. Incapsula incident ID: 361000420216756332-' .
'189006456860248749</iframe></body></html>';
$process = function ($robots_txt, $url, $last_try = false,
$page = null, $code = 403) use ($challenge) {
return $this->processOnePage($robots_txt, [
Fetcher::URL => $url, Fetcher::HTTP_CODE => $code,
Fetcher::TYPE => "text/html",
Fetcher::PAGE => $page ?? $challenge], $last_try);
};
$home = "https://www.pepsi.com/";
[$summaries, $sent_back] = $process(C\ALLOW_LANDING_ROBOTS, $home);
$this->assertEqual([count($summaries), $sent_back], [0, [$home]],
"before the last try the home page is only sent back");
[$summaries, $sent_back] = $process(C\ALLOW_LANDING_ROBOTS, $home,
true);
$this->assertEqual([count($summaries),
$summaries[0][Fetcher::DESCRIPTION] ?? "",
$summaries[0][Fetcher::TITLE] ?? ""],
[1, "pepsi www.pepsi.com", "www.pepsi.com"],
"on the last try the home page is indexed by the words of its " .
"url");
$this->assertEqual($sent_back, [$home],
"on the last try the home page is also sent back once");
[$summaries, $sent_back] = $process(C\ALLOW_LANDING_ROBOTS,
$home . "about/", true);
$this->assertEqual([count($summaries), $sent_back],
[0, [$home . "about/"]],
"a page below the home page is only sent back");
[$summaries, $sent_back] = $process(C\ALWAYS_FOLLOW_ROBOTS, $home,
true);
$this->assertEqual([count($summaries), $sent_back], [0, [$home]],
"in a crawl that follows robots.txt the home page is only " .
"sent back");
[$summaries, ] = $process(C\ALLOW_LANDING_ROBOTS,
"https://coke.com/", false, "<html><head><title>Coke</title>" .
"</head><body><div id='app'></div></body></html>", 200);
$this->assertEqual([$summaries[0][Fetcher::DESCRIPTION] ?? "",
$summaries[0][Fetcher::TITLE] ?? ""], ["coke coke.com", "Coke"],
"a home page that gave no text keeps its title and is " .
"described by the words of its url");
}
/**
* redirectKeepsItsUrlsTestCase checks the record processFetchPages
* makes for a page that answers with a redirect. Its LOCATION field
* lists the page's url and then the url it redirects to, made whole
* from a relative Location header. The record is described as the
* page's url, an arrow and the target. The target is also a link of
* the record, so it gets fetched.
*/
public function redirectKeepsItsUrlsTestCase()
{
$url = "https://coke.com/";
$target = "https://coke.com/us/en";
[$summaries, ] = $this->processOnePage(C\ALWAYS_FOLLOW_ROBOTS, [
Fetcher::URL => $url, Fetcher::HTTP_CODE => 301,
Fetcher::TYPE => "text/html", Fetcher::PAGE => "",
Fetcher::LOCATION => ["/us/en"]]);
$this->assertEqual($summaries[0][Fetcher::LOCATION] ?? null,
[$url, $target], "the redirect lists the page and its target");
$this->assertEqual([$summaries[0][Fetcher::DESCRIPTION] ?? "",
isset($summaries[0][Fetcher::LINKS][$target])],
["$url => $target", true],
"the record names the target and links to it");
}
/**
* linkGetsNoQuestionAnswersOfPageBeforeItTestCase checks the question
* answers compressAndUnsetSeenUrls packs with each site it sends. A page
* whose title and summary hold a question and its answer gets them. A
* link found on an earlier page, packed after that page, gets none of
* them: each site gets the answers of its own text only.
*/
public function linkGetsNoQuestionAnswersOfPageBeforeItTestCase()
{
$fetcher = new Fetcher();
$fetcher->queue_servers = ["http://localhost/"];
$fetcher->crawl_stat_info = ["NUM_SEEN_SITES" => 0];
$fetcher->found_sites[Fetcher::SEEN_URLS] = [[
Fetcher::URL => "https://www.example.com/bob",
Fetcher::TITLE => "Who is Bob?",
Fetcher::DESCRIPTION => "Bob is a cat. The cat sat on the mat.",
Fetcher::LANG => "en", Fetcher::TIMESTAMP => self::TEST_CRAWL_TIME,
Fetcher::LINKS => [], Fetcher::HASH => "bob"]];
$fetcher->found_sites[Fetcher::LINK_SEEN_URLS][0] = [[
Fetcher::URL => "url|https://www.example.com/cats|text|cats|" .
"iref|https://www.example.com/", Fetcher::TITLE => "cats",
Fetcher::DESCRIPTION => "cats", Fetcher::TYPE => "link",
Fetcher::LANG => "en", Fetcher::TIMESTAMP => self::TEST_CRAWL_TIME,
Fetcher::HASH => "cats"]];
$packed = $fetcher->compressAndUnsetSeenUrls(0);
$sites = [];
for ($offset = 0; $offset < strlen($packed);
$offset += 4 + $length) {
$length = L\unpackInt(substr($packed, $offset, 4));
$sites[] = L\unpackSeenSite(substr($packed, $offset + 4,
$length));
}
$this->assertEqual(count($sites), 2, "both sites are packed");
$this->assertTrue(!empty($sites[0][Fetcher::QUESTION_ANSWERS]),
"the page gets the answers of its text");
$this->assertTrue(empty($sites[1][Fetcher::QUESTION_ANSWERS]),
"the link gets no answers");
}
/**
* processOnePage runs processFetchPages on one downloaded page with a
* fetcher made for the purpose, and gives what came back. The fetcher
* is numbered TEST_FETCHER, so the crawl statistics file it writes is
* one tearDown removes. challengedHomePageIsIndexedByUrlWordsTestCase
* and redirectKeepsItsUrlsTestCase use it.
*
* @param int $robots_txt the crawl's robots.txt mode, such as
* C\ALLOW_LANDING_ROBOTS
* @param array $site the downloaded page's fields; the encoding,
* weight, depth and address are filled in where not given
* @param bool $last_try whether the schedule names the page as on the
* Scheduler's last try
* @return array the page summaries processFetchPages gave, and the
* urls the fetcher would send back to be fetched again
*/
public function processOnePage($robots_txt, $site, $last_try = false)
{
$fetcher = new Fetcher();
$fetcher->which_fetcher = self::TEST_FETCHER;
$fetcher->schedules_folder = self::SCHEDULES;
$fetcher->crawl_stat_info = ["NUM_SUMMARIZED_PAGES" => 0,
"NUM_CACHED_PAGES" => 0];
$fetcher->crawl_stat_filename = self::SCHEDULES . "/" .
self::TEST_FETCHER . "-0-CrawlStat" . self::TEST_CRAWL_TIME .
".txt";
$fetcher->queue_servers = ["http://localhost/"];
$fetcher->robots_txt = $robots_txt;
if ($last_try) {
$fetcher->last_try_urls = [$site[Fetcher::URL] => true];
}
$summaries = $fetcher->processFetchPages([$site + [
Fetcher::ENCODING => "UTF-8", Fetcher::WEIGHT => 1,
Fetcher::DOC_DEPTH => 1,
Fetcher::IP_ADDRESSES => ["45.223.17.132"]]]);
$sent_back = array_column(
$fetcher->found_sites[Fetcher::RETRY_URLS][0] ?? [], 0);
return [$summaries, $sent_back];
}
/**
* ratePerHourStartsAtTheFetchersStartTestCase checks the per-hour
* figures a fetcher logs and sends, such as its downloads per hour.
* The first figure divides by the time since the fetcher was made,
* where it divided by the hours since 1970 and came out near 0. Until
* the window moves again the figure is the total over the time since
* then. Once it has moved, the figure is the amount since the total it
* kept, over the time since the move before. The fetcher
* is made without its constructor, which would start a crawl process.
*/
public function ratePerHourStartsAtTheFetchersStartTestCase()
{
$fetcher = (new \ReflectionClass(Fetcher::class))
->newInstanceWithoutConstructor();
$start = self::TEST_CRAWL_TIME;
$fetcher->rate_start = $start;
$this->assertEqual($fetcher->ratePerHour("DOWNLOADS", 500,
$start + C\ONE_HOUR / 2), 1000,
"500 pages in the first half hour are 1000 an hour");
$this->assertEqual($fetcher->ratePerHour("DOWNLOADS", 800,
$start + C\ONE_HOUR), 800,
"800 pages in the first hour are 800 an hour");
$this->assertEqual($fetcher->ratePerHour("DOWNLOADS", 2000,
$start + 3 * C\ONE_HOUR), 2000 / 3,
"within the window the figure counts from the start");
$this->assertEqual($fetcher->ratePerHour("DOWNLOADS", 2600,
$start + 4 * C\ONE_HOUR), 2100 / 3.5,
"a moved window counts from its last move before");
$this->assertEqual($fetcher->ratePerHour("UPLOADS", 50,
$start + 4 * C\ONE_HOUR), 12.5,
"each figure has its own window");
}
/**
* uploadPartSizeSettlesOnLargestThatWorksTestCase checks how a fetcher
* picks the part size of its uploads to each queue server. Fetchers
* used to keep 2,000,000 byte parts unless a server reported a smaller
* limit, and dropped to 1,000,000 bytes for good after one failed first
* part, so a 76 megabyte upload took 38 posts. Now a fetcher starts at
* DEFAULT_POST_MAX_SIZE, takes the limit a server reports, and after a
* size fails searches between the largest size that went through and
* the smallest that failed.
*/
public function uploadPartSizeSettlesOnLargestThatWorksTestCase()
{
$fetcher = new Fetcher();
$server = "https://queue.example.com/";
$other = "https://other.example.com/";
$this->assertEqual($fetcher->postSize($server),
Fetcher::DEFAULT_POST_MAX_SIZE, "a new server gets the default");
$fetcher->noteServerPostLimit($server, 9000000);
$this->assertEqual($fetcher->postSize($server), 9000000,
"the size grows to the limit the server reports");
$fetcher->notePostResult($server, 9000000, false);
$this->assertEqual($fetcher->postSize($server), 4500000,
"with nothing through yet, a failed size is halved");
$fetcher->notePostResult($server, 4500000, true);
$this->assertEqual($fetcher->postSize($server), 6750000,
"the next size is halfway between what worked and what failed");
$fetcher->notePostResult($server, 6750000, false);
$fetcher->notePostResult($server, 5625000, true);
$fetcher->notePostResult($server, 6187500, false);
$fetcher->notePostResult($server, 5906250, true);
$fetcher->notePostResult($server, 6046875, false);
$this->assertEqual($fetcher->postSize($server), 5906250,
"the search stops at the largest size that went through");
$fetcher->noteServerPostLimit($server, 9000000);
$this->assertEqual($fetcher->postSize($server), 5906250,
"a reported limit does not raise a size past one that failed");
$this->assertEqual($fetcher->postSize($other),
Fetcher::DEFAULT_POST_MAX_SIZE, "each server has its own size");
$fetcher->noteServerPostLimit($server, 5000000);
$this->assertEqual($fetcher->postSize($server), 5000000,
"a lower reported limit lowers the size");
$fetcher->notePostResult($other, 1000000, false);
$fetcher->notePostResult($other, 600000, false);
$this->assertEqual($fetcher->postSize($other),
Fetcher::MIN_POST_SIZE, "the size stays at MIN_POST_SIZE or more");
}
/**
* hostWithManySlowPagesGetsFastProcessorTestCase checks that once a
* host has given a fetcher SLOW_PAGES_BEFORE_FAST slow pages within
* an hour, its pages get the fast variant of their processor, an
* HtmlProcessor whose summary comes from the ScrapeSummarizer, until
* the hour has passed. Other hosts keep the processor the crawl asks
* for. The fetcher is made without its constructor, which would start
* a crawl process.
*/
public function hostWithManySlowPagesGetsFastProcessorTestCase()
{
$fetcher = (new \ReflectionClass(Fetcher::class))
->newInstanceWithoutConstructor();
$fetcher->page_processors = ["text/html" => "HtmlProcessor"];
$fetcher->plugin_processors = [];
$fetcher->plugin_hash = "fast processor case";
$fetcher->max_description_len = 2000;
$fetcher->summarizer_option = Fetcher::CENTROID_WEIGHTED_SUMMARIZER;
$start = self::TEST_CRAWL_TIME;
for ($i = 1; $i < Fetcher::SLOW_PAGES_BEFORE_FAST; $i++) {
$fetcher->noteSlowPage("slow.example.com", $start + $i);
}
$this->assertTrue(!$fetcher->usesFastProcessor("slow.example.com",
$start + 60), "one slow page short of the limit is not enough");
$this->assertEqual($fetcher->noteSlowPage("slow.example.com",
$start + 61), Fetcher::SLOW_PAGES_BEFORE_FAST,
"the slow pages of the hour are counted");
$this->assertTrue($fetcher->usesFastProcessor("slow.example.com",
$start + 62), "the host then gets the fast processor");
$this->assertTrue(!$fetcher->usesFastProcessor("other.example.com",
$start + 62), "another host does not");
$this->assertTrue(!$fetcher->usesFastProcessor("slow.example.com",
$start + 1 + C\ONE_HOUR), "once the hour has passed, it does not");
$this->assertEqual(get_class($fetcher->pageProcessor("text/html",
true)->summarizer), C\NS_SUMMARIZERS . "ScrapeSummarizer",
"the fast processor summarizes with the ScrapeSummarizer");
$this->assertEqual(get_class($fetcher->pageProcessor("text/html")
->summarizer), C\NS_SUMMARIZERS . "CentroidWeightedSummarizer",
"the usual processor keeps the crawl's summarizer");
}
}
- Yioop Repo - Seekquarry