<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see
* <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @package seek_quarry\tests
*/
namespace seekquarry\yioop\tests;
use seekquarry\atto as A;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\controllers\FetchController;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\Model;
/**
* FetchControllerTest checks how FetchController keeps the robot table,
* the list of the fetchers that spoke with a machine lately, which the
* queue server reads for the most recent fetcher and the fetchers'
* download rates.
*
* @author Chris Pollett
*/
class FetchControllerTest extends UnitTest
{
/**
* FOLDER is the folder, under tests/test_files, that the cases keep
* their robot table in.
*/
const FOLDER = C\TEST_DIR . "/test_files/fetch_controller_test";
/**
* setUp makes FOLDER.
*/
public function setUp()
{
L\makePath(self::FOLDER);
}
/**
* tearDown deletes FOLDER with the robot table in it.
*/
public function tearDown()
{
(new Model())->db->unlinkRecursive(self::FOLDER);
}
/**
* emptyRobotTableIsStartedAgainTestCase checks that recordRobot puts a
* fetcher's entry in a robot table that cannot be read back. On the
* crawl of 15 October a write of the table failed with no space left
* on the device and left it empty; FetchController then never wrote
* the table again, and the queue server found no fetcher from then on.
*/
public function emptyRobotTableIsStartedAgainTestCase()
{
$table = self::FOLDER . "/0-RobotTable.txt";
file_put_contents($table, "");
$this->assertTrue(FetchController::recordRobot($table, "0-0-Yioop",
["1.2.3.4", "/", 1000, 17000]), "the table is opened");
$this->assertEqual(unserialize(file_get_contents($table)),
["0-0-Yioop" => ["1.2.3.4", "/", 1000, 17000]],
"the empty table now holds the fetcher's entry");
file_put_contents($table, "a:2:{s:9:\"0-0-Yioop\";a:4:{i:0;s:7:");
FetchController::recordRobot($table, "1-0-Yioop",
["1.2.3.4", "/", 1001, 9000]);
$this->assertEqual(unserialize(file_get_contents($table)),
["1-0-Yioop" => ["1.2.3.4", "/", 1001, 9000]],
"a table cut off part way is started again");
$this->assertEqual(fileperms($table) & 0004, 0004,
"the table can be read by the crawl processes");
}
/**
* robotEntryKeepsItsRateTestCase checks that recordRobot keeps the
* other fetchers' entries, and that a request with no rate, given as
* -1, keeps the rate the fetcher sent before.
*/
public function robotEntryKeepsItsRateTestCase()
{
$table = self::FOLDER . "/0-RobotTable.txt";
FetchController::recordRobot($table, "0-0-Yioop",
["1.2.3.4", "/", 1000, 17000]);
FetchController::recordRobot($table, "1-0-Yioop",
["1.2.3.5", "/", 1005, 9000]);
FetchController::recordRobot($table, "0-0-Yioop",
["1.2.3.4", "/", 1010, -1]);
$this->assertEqual(unserialize(file_get_contents($table)),
["0-0-Yioop" => ["1.2.3.4", "/", 1010, 17000],
"1-0-Yioop" => ["1.2.3.5", "/", 1005, 9000]],
"each fetcher has its entry, and the rate is kept");
}
/**
* uploadPartsAreSavedInFolderTestCase checks savePart. The first part
* of an upload starts its file again, so a fetcher that restarts an
* upload with a new part size does not leave the old parts in front of
* the new ones; before, every part was added to the end of the file.
* A hash with a path in it cannot name a file outside the folder.
*/
public function uploadPartsAreSavedInFolderTestCase()
{
$file = FetchController::savePart(self::FOLDER, "abc123", "old1", 1);
FetchController::savePart(self::FOLDER, "abc123", "old2", 2);
FetchController::savePart(self::FOLDER, "abc123", "new1", 1);
FetchController::savePart(self::FOLDER, "abc123", "new2", 2);
$this->assertEqual(file_get_contents($file), "new1new2",
"a restarted upload holds only its new parts");
$file = FetchController::savePart(self::FOLDER, "../../x/y", "z", 1);
$this->assertEqual($file, self::FOLDER . "/xy",
"the file stays in the folder");
}
/**
* postMaxSizeUnderOwnWebServerTestCase checks the largest post
* FetchController tells fetchers. Under Yioop's own web server, index.php
* used to raise MAX_REQUEST_LEN to the Git push size limit, which by
* default has none, so MAX_REQUEST_LEN could be PHP_INT_MAX. The
* controller then told fetchers they could post that much, and a
* fetcher sent each upload, tens of megabytes, as one post. The size
* now follows post_max_size, raised to the web server's default, less
* POST_HEADER_ROOM, whatever MAX_REQUEST_LEN is.
*/
public function postMaxSizeUnderOwnWebServerTestCase()
{
$controller = new FetchController();
$php_post_max = L\metricToInt(ini_get("post_max_size"));
$this->assertEqual($controller->postMaxSize(), $php_post_max,
"under another web server it is PHP's post_max_size");
$default_max = A\WebSite::defaultServerGlobals()['MAX_REQUEST_LEN'];
$controller->web_site = (object)["default_server_globals" =>
["MAX_REQUEST_LEN" => PHP_INT_MAX]];
$this->assertEqual($controller->postMaxSize(),
max($php_post_max, $default_max) -
FetchController::POST_HEADER_ROOM,
"a request limit raised for Git pushes is not given to fetchers");
$controller->web_site->default_server_globals['MAX_REQUEST_LEN'] =
$default_max - 1000000;
$this->assertEqual($controller->postMaxSize(),
$default_max - 1000000 - FetchController::POST_HEADER_ROOM,
"the size stays under the web server's request limit");
}
}
- Yioop Repo - Seekquarry