<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\controllers;
use seekquarry\atto as A;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\CrawlDaemon;
use seekquarry\yioop\library\storage_formats\MessagesBundle;
use seekquarry\yioop\library\classifiers\Classifier;
/**
* This class handles data coming to a queue_server from a fetcher
* Basically, it receives the data from the fetcher and saves it into
* various files for later processing by the queue server.
* This class can also be used by a fetcher to get status information.
*
* @author Chris Pollett
*/
class FetchController extends Controller implements CrawlConstants
{
/**
* These are the activities supported by this controller
* @var array
*/
public $activities = ["schedule", "archiveSchedule", "update", "crawlTime"];
/**
* File used to store info about the status of a queue server's
* active crawl. Default to channel 0 but might change in
* @see processRequest
* @var string
*/
public $crawl_status_file_name =
C\SCHEDULES_DIR . "/0-" . self::crawl_status_file;
/**
* Number of seconds that must elapse after last call before doing
* cron activities (mainly check liveness of fetchers which should be
* alive)
*/
const CRON_INTERVAL = 300;
/**
* POST_HEADER_ROOM is the number of bytes taken off the web server's
* request limit, for the request line and headers, when FetchController
* tells a fetcher the largest post it may send.
*/
const POST_HEADER_ROOM = 65536;
/**
* Checks that the request seems to be coming from a legitimate fetcher then
* determines which activity the fetcher is requesting and calls that
* activity for processing.
*/
public function processRequest()
{
// to allow the calculation of longer archive schedules
if (!$this->web_site->isCli()) {
ini_set('max_execution_time', 60);
}
$data = [];
/* do a quick test to see if this is a request seems like
from a legitimate machine
*/
if (!$this->checkRequest()) {
$this->web_site->header("HTTP/1.0 400 Bad Request");
$_REQUEST['p'] = "400";
$_REQUEST['c'] = "static";
\seekquarry\yioop\bootstrap($this->web_site, false);
return;
}
$activity = $_REQUEST['a'];
$channel = $this->getChannel();
$this->crawl_status_file_name =
C\SCHEDULES_DIR . "/{$channel}-" . self::crawl_status_file;
$robot_table_name = C\WORK_DIRECTORY . "/{$channel}-" .
self::robot_table_name;
if (isset($_REQUEST['robot_instance']) &&
isset($_REQUEST['machine_uri'])) {
$downloads_per_hour = isset($_REQUEST['downloads_per_hour']) ?
$this->clean($_REQUEST['downloads_per_hour'], "int") : -1;
self::recordRobot($robot_table_name,
$this->clean($_REQUEST['robot_instance'], "string") ?? 0,
[L\remoteAddress(),
$this->clean($_REQUEST['machine_uri'], "string"), time(),
$downloads_per_hour]);
}
if (in_array($activity, $this->activities)) {
$this->call($activity);
}
}
/**
* recordRobot puts a fetcher's entry in a channel's robot table: its
* address, its machine uri, the time of its request and the pages it
* downloaded per hour. A request that sends no rate, given as -1, keeps
* the rate the fetcher sent before. The table is read, changed and
* written under an exclusive lock, so two requests at once do not drop
* each other's entry. A table that cannot be read back, such as one
* left empty by a write that failed when the disk was full, is started
* again rather than left as it is, and the file is made readable by the
* queue server and the other crawl processes.
*
* @param string $robot_table_name path of the channel's robot table
* @param string $robot_instance the fetcher's name, which keys its entry
* @param array $entry the fetcher's address, machine uri, request time
* and downloads per hour, or -1 for the last one
* @return bool whether the table could be opened
*/
public static function recordRobot($robot_table_name, $robot_instance,
$entry)
{
$handle = @fopen($robot_table_name, "c+");
if ($handle === false) {
return false;
}
flock($handle, LOCK_EX);
$robot_table = @unserialize(stream_get_contents($handle));
$robot_table = is_array($robot_table) ? $robot_table : [];
if ($entry[3] < 0) {
$entry[3] = $robot_table[$robot_instance][3] ?? 0;
}
$robot_table[$robot_instance] = $entry;
ftruncate($handle, 0);
rewind($handle);
fwrite($handle, serialize($robot_table));
fflush($handle);
flock($handle, LOCK_UN);
fclose($handle);
L\setWorldPermissions($robot_table_name);
return true;
}
/**
* Returns the channel used by the given uploaded data
*
* @return int channel used
*/
public function getChannel()
{
$channel = 0;
if (!empty($_REQUEST['robot_instance'])) {
$instance_parts = explode("-", $_REQUEST['robot_instance']);
$channel = (empty($instance_parts[1])) ? 0 : $instance_parts[1];
}
return $channel;
}
/**
* Checks if there is a schedule of sites to crawl available and
* if so present it to the requesting fetcher, and then delete it.
*/
public function schedule()
{
$view = "fetch";
// set up query
$data = [];
if (isset($_REQUEST['crawl_time'])) {;
$crawl_time = substr($this->clean($_REQUEST['crawl_time'], 'int'),
0, C\TIMESTAMP_LEN);
} else {
$crawl_time = 0;
}
$info = [];
$info[self::STATUS] = self::NO_DATA_STATE;
$data['MESSAGE'] = base64_encode(serialize($info)) . "\n";
$message = "";
if ($crawl_time > 0) {
$messages_folder = C\SCHEDULES_DIR . "/" .
self::messages_data_base_name . $crawl_time;
$messages_bundle = new MessagesBundle($messages_folder);
$messages_bundle->initWindows();
// get a schedule if exists, else false
$message = $messages_bundle->extractSendWindow();
if (!$message) {
$message = $data['MESSAGE'];
}
/* this gets a ready for further processing file from receive window
if exists, and moves it to other folder for further processing
*/
$packed_data = $messages_bundle->extractReceiveWindow();
if (!empty($packed_data)) {
list($byte_counts, $upload_data) = unserialize(
$packed_data);
$this->handleUploadedData($byte_counts,
$upload_data, $messages_bundle);
}
}
$data['MESSAGE'] = $message;
$this->displayView($view, $data);
}
/**
* Checks to see whether there are more pages to extract from the current
* archive, and if so returns the next batch to the requesting fetcher. The
* iteration progress is automatically saved on each call to nextPages, so
* that the next fetcher will get the next batch of pages. If there is no
* current archive to iterate over, or the iterator has reached the end of
* the archive then indicate that there is no more data by setting the
* status to NO_DATA_STATE.
*/
public function archiveSchedule()
{
$view = "fetch";
$request_start = time();
if (isset($_REQUEST['crawl_time'])) {;
$crawl_time = substr($this->clean($_REQUEST['crawl_time'], 'int'),
0, C\TIMESTAMP_LEN);
} else {
$crawl_time = 0;
}
$channel = $this->getChannel();
$messages_filename = CrawlDaemon::getMessageFileName("NameServer",
$channel);
$lock_filename = CrawlDaemon::getLockFileName("NameServer",
$channel);
if ($crawl_time > 0 && file_exists($messages_filename)) {
$fetch_pages = true;
$info = unserialize(file_get_contents($messages_filename));
if ($info[self::STATUS] == 'STOP_CRAWL') {
/* The stop crawl message gets created by the admin_controller
when the "stop crawl" button is pressed.*/
if (file_exists($messages_filename)) {
unlink($messages_filename);
}
if (file_exists($lock_filename)) {
unlink($lock_filename);
}
$fetch_pages = false;
$info = [];
}
} else {
$fetch_pages = false;
$info = [];
}
$pages = [];
$got_lock = true;
if (file_exists($lock_filename)) {
$lock_time = unserialize(file_get_contents($lock_filename));
if ($request_start - $lock_time < ini_get('max_execution_time')){
$got_lock = false;
}
}
$chunk = false;
$archive_iterator = null;
if ($fetch_pages && $got_lock) {
file_put_contents($lock_filename, serialize($request_start));
if ($info[self::ARC_DIR] == "MIX" ||
file_exists($info[self::ARC_DIR])) {
$iterate_timestamp = $info[self::CRAWL_INDEX];
$result_timestamp = $crawl_time;
$result_dir = C\SCHEDULES_DIR .
"/" . self::name_archive_iterator . $crawl_time;
$arctype = $info[self::ARC_TYPE];
$iterator_name = C\NS_ARCHIVE . $arctype . "Iterator";
try {
if ($info[self::ARC_DIR] == "MIX") {
//recrawl of crawl mix case
$archive_iterator = new $iterator_name(
$iterate_timestamp, $result_timestamp);
} else {
//any other archive crawl except web archive recrawls
$archive_iterator = new $iterator_name(
$iterate_timestamp, $info[self::ARC_DIR],
$result_timestamp, $result_dir);
}
} catch (\Exception $e) {
$info['ARCHIVE_BUNDLE_ERROR'] =
"Invalid bundle iterator: '{$iterator_name}' \n".
$e->getMessage();
}
}
$pages = false;
if ($archive_iterator && !$archive_iterator->end_of_iterator) {
if (L\generalIsA($archive_iterator,
C\NS_ARCHIVE . "TextArchiveBundleIterator")) {
$pages = $archive_iterator->nextChunk();
$chunk = true;
} else {
$pages = $archive_iterator->nextPages(
C\ARCHIVE_BATCH_SIZE);
}
}
if (file_exists($lock_filename)) {
unlink($lock_filename);
}
}
if (!empty($archive_iterator) && $archive_iterator->end_of_iterator) {
$info[self::END_ITERATOR] = true;
}
if ((!empty($chunk) && !isset($pages)) || (!empty($pages))) {
$pages_string = L\webencode(gzcompress(serialize($pages)));
} else {
$info[self::STATUS] = self::NO_DATA_STATE;
$info[self::POST_MAX_SIZE] = $this->postMaxSize();
$pages = [];
$pages_string = L\webencode(gzcompress(serialize($pages)));
}
$info[self::DATA] = $pages_string;
$info_string = serialize($info);
$data['MESSAGE'] = $info_string;
$this->displayView($view, $data);
}
/**
* Saves one part of a fetcher's upload to the file in $folder named by
* the upload's hash, where the parts are put together. The name is
* kept to the characters a hash has, so a post cannot name a file
* outside $folder. The first part starts the file afresh, so an upload
* a fetcher restarts with a new part size does not add to the parts it
* sent before.
*
* @param string $folder the folder the upload files are kept in
* @param string $hash_data the hash of the whole upload
* @param string $part the part's bytes
* @param int $current_part the number of the part, starting at 1
* @return string the name of the file the part was saved to
*/
public static function savePart($folder, $hash_data, $part,
$current_part)
{
$filename = $folder . "/" . preg_replace('/[^A-Za-z0-9_\-]/', "",
$hash_data);
file_put_contents($filename, $part,
(intval($current_part) == 1) ? 0 : FILE_APPEND);
return $filename;
}
/**
* Gives the largest post in bytes this web app takes, which it tells
* fetchers so they can size their upload parts. Under another web
* server this is PHP's post_max_size. Under Yioop's own web server,
* index.php sets MAX_REQUEST_LEN from post_max_size, raised to the web
* server's own default, so the size given is post_max_size raised to
* that default and kept under MAX_REQUEST_LEN, less POST_HEADER_ROOM
* for the request line and headers. A post_max_size of 0, no limit,
* gives the web server's default, so one post does not carry a whole
* upload of tens of megabytes.
*
* @return int the largest post in bytes
*/
public function postMaxSize()
{
$php_post_max = L\metricToInt(ini_get("post_max_size"));
$max_request_len = $this->web_site->default_server_globals[
'MAX_REQUEST_LEN'] ?? 0;
if ($max_request_len <= 0) {
return $php_post_max;
}
$default_max_request = A\WebSite::defaultServerGlobals()[
'MAX_REQUEST_LEN'];
$post_max = min($max_request_len,
max($php_post_max, $default_max_request));
return max($post_max - self::POST_HEADER_ROOM,
self::POST_HEADER_ROOM);
}
/**
* Processes Robot, To Crawl, and Index data sent from a fetcher
* Acknowledge to the fetcher if this data was received okay.
*/
public function update()
{
$view = "fetch";
$info_flag = false;
$logging = "";
$necessary_fields = ['byte_counts', 'current_part', 'hash_data',
'hash_part', 'num_parts', 'part', 'sequence_number', 'diff_origin'];
$part_flag = true;
$missing = "";
$channel = $this->getChannel();
foreach ($necessary_fields as $field) {
if (!isset($_REQUEST[$field])) {
$part_flag = false;
$missing = $field;
}
}
$crawl_type = $this->clean($_REQUEST['crawl_type'] ?? "", 'string');
$arc_type = $this->clean($_REQUEST['arc_type'] ?? "", 'string');
$crawl_time = substr($this->clean($_REQUEST['crawl_time'] ?? "0",'int'),
0, C\TIMESTAMP_LEN);
if ($part_flag &&
L\crawlHash($_REQUEST['part']) == $_REQUEST['hash_part']) {
$upload = false;
if (!file_exists(C\TEMP_DIR)) {
mkdir(C\TEMP_DIR);
L\setWorldPermissionsRecursive(C\TEMP_DIR . "/");
}
$filename = self::savePart(C\TEMP_DIR, $_REQUEST['hash_data'],
$_REQUEST['part'], $_REQUEST['current_part']);
L\setWorldPermissions($filename);
if (intval($_REQUEST['num_parts']) > 1) {
$info_flag = true;
if ($_REQUEST['num_parts'] == $_REQUEST['current_part']) {
$upload = true;
}
} else if (intval($_REQUEST['num_parts']) == 1) {
$info_flag = true;
$upload = true;
}
if ($upload) {
$messages_folder = C\SCHEDULES_DIR . "/" .
self::messages_data_base_name . $crawl_time;
$messages_bundle = new MessagesBundle($messages_folder);
$messages_bundle->initWindows();
$diff_origin = filter_var($_REQUEST['diff_origin'],
FILTER_VALIDATE_BOOLEAN);
$sequence_number = intval($_REQUEST['sequence_number']);
$byte_counts = [];
if (!empty($_REQUEST['byte_counts'])) {
$byte_counts = unserialize(
L\webdecode($_REQUEST['byte_counts']));
}
$schedule_time = $byte_counts["SCHEDULE_TIME"] ?? false;
$upload_old = false;
if (file_exists($this->crawl_status_file_name)) {
$crawl_status = unserialize(file_get_contents(
$this->crawl_status_file_name));
if (!empty($crawl_status['REPEAT_TIME']) &&
($schedule_time &&
$crawl_status['REPEAT_TIME'] > $schedule_time)) {
$upload_old = true;
}
}
if ($upload_old) {
$logging = "DoubleIndexArchive has switched index!\n" .
"Discarding old index upload data!";
unlink($filename);
} else {
$logging = "... Data received successfully\n";
$upload_data = file_get_contents($filename);
unlink($filename);
if (!$diff_origin && ($crawl_type != self::ARCHIVE_CRAWL ||
$arc_type == "WebArchiveBundle" || $arc_type == "") &&
!empty($messages_bundle->send[$sequence_number])) {
$messages_bundle->updateReceiveWindow($sequence_number,
serialize([$byte_counts, $upload_data]));
$packed_data = $messages_bundle->extractReceiveWindow();
if (!empty($packed_data)) {
list($byte_counts, $upload_data) = unserialize(
$packed_data);
} else {
$upload_data = "";
$byte_counts = [];
}
} else if (!$diff_origin) {
/* The schedule this data answers has already left
the send window: an earlier send of the fetcher
answered it, or it timed out because the
fetcher's request waited too long to reach here.
The data is still this crawl's, so it is taken
as it is rather than dropped. */
$logging .= "... Schedule $sequence_number was " .
"answered or timed out; data taken anyway\n";
}
$this->handleUploadedData($byte_counts,
$upload_data, $messages_bundle);
}
} else {
$logging = "...".(
$_REQUEST['current_part']/$_REQUEST['num_parts']).
" of data uploaded.";
}
}
$info =[];
$info['REPEAT_TIME'] = $crawl_status['REPEAT_TIME'] ?? 0;
$info['SLEEP_START'] = $crawl_status['SLEEP_START'] ?? "00:00";
$info['SLEEP_DURATION'] = $crawl_status['SLEEP_DURATION'] ?? 0;
if ($logging != "") {
$info[self::LOGGING] = $logging;
}
if ($info_flag == true) {
$info[self::STATUS] = self::CONTINUE_STATE;
} else {
$info[self::STATUS] = self::REDO_STATE;
if (!$part_flag) {
$info[self::SUMMARY] = "Missing request field: $missing.";
} else {
$info[self::SUMMARY] = "Hash of uploaded data was:".
L\crawlHash($_REQUEST['part']).". Sent checksum was:".
$_REQUEST['hash_part'];
}
}
$info[self::MEMORY_USAGE] = memory_get_peak_usage();
$info[self::POST_MAX_SIZE] = $this->postMaxSize();
if (file_exists($this->crawl_status_file_name)) {
$change = false;
$crawl_status = unserialize(file_get_contents(
$this->crawl_status_file_name));
//want to make sure $crawl_status has data if will change
if (!empty($crawl_status['CRAWL_TIME'])) {
if (isset($_REQUEST['fetcher_peak_memory'])) {
if (!isset($crawl_status['FETCHER_MEMORY']) ||
$_REQUEST['fetcher_peak_memory'] >
$crawl_status['FETCHER_PEAK_MEMORY']
) {
$crawl_status['FETCHER_PEAK_MEMORY'] =
$_REQUEST['fetcher_peak_memory'];
$change = true;
}
}
if (!isset($crawl_status['WEBAPP_PEAK_MEMORY']) ||
$info[self::MEMORY_USAGE] >
$crawl_status['WEBAPP_PEAK_MEMORY']) {
$crawl_status['WEBAPP_PEAK_MEMORY'] =
$info[self::MEMORY_USAGE];
$change = true;
}
}
if (!isset($crawl_status[self::CRAWL_TIME])) {
$network_filename = C\SCHEDULES_DIR .
"/{$channel}-" . self::network_status_file;
if (file_exists($network_filename)) {
$net_info = unserialize(file_get_contents(
$network_filename));
$info[self::CRAWL_TIME] = $net_info[self::CRAWL_TIME] ?? 0;
$info[self::SLEEP_START] = $net_info[self::SLEEP_START] ??
"00:00";
$info[self::SLEEP_DURATION] =
$net_info[self::SLEEP_DURATION] ?? "-1";
} else {
$info[self::CRAWL_TIME] = 0;
$info[self::SLEEP_START] = $net_info[self::SLEEP_START] ??
"00:00";
$info[self::SLEEP_DURATION] =
$net_info[self::SLEEP_DURATION] ?? "-1";
}
} else {
$info[self::CRAWL_TIME] = $crawl_status['CRAWL_TIME'];
$info[self::SLEEP_START] = $crawl_status["SLEEP_START"] ??
"00:00";
$info[self::SLEEP_DURATION] =
$crawl_status["SLEEP_DURATION"] ?? "-1";
}
if ($change == true) {
file_put_contents($this->crawl_status_file_name,
serialize($crawl_status), LOCK_EX);
}
} else {
$info[self::CRAWL_TIME] = 0;
}
$info[self::MEMORY_USAGE] = memory_get_peak_usage();
$data = [];
$data['MESSAGE'] = serialize($info);
$this->displayView($view, $data);
}
/**
* After robot, schedule, and index data have been uploaded and reassembled
* as one big data file/string, this function splits that string into
* each of these data types and then save the result into the appropriate
* schedule sub-folder of the $messages_bundle. Any temporary files used
* during uploading are then deleted.
*
* @param array $byte_counts associative array giving the number of
* bytes of each data category (ROBOT, CACHE_PAGE_VALIDATION,
* SCHEDULE, INDEX) within $uploaded
* @param string &$uploaded reassembled upload payload, sliced
* according to $byte_counts and dispatched to $messages_bundle
* @param MessagesBundle $messages_bundle used to store the messages
* contained in the upload so that they can be later processed
* @return string $logging diagnostic info to be sent to fetcher about
* what was done
*/
public function handleUploadedData($byte_counts, &$uploaded,
$messages_bundle)
{
$logging = "... Data upload complete";
$address = strtr(L\remoteAddress(), ["." => "-", ":" => "_"]);
$time = time();
$day = floor($time/C\ONE_DAY);
$robot_data = "";
$cache_page_validation_data = "";
$schedule_data = "";
$index_data = "";
if (isset($byte_counts["TOTAL"]) &&
$byte_counts["TOTAL"] > 0) {
$pos = 0;
$robot_data = substr($uploaded, $pos, $byte_counts["ROBOT"]);
$pos += $byte_counts["ROBOT"];
$cache_page_validation_data = substr($uploaded, $pos,
$byte_counts["CACHE_PAGE_VALIDATION"]);
$pos += $byte_counts["CACHE_PAGE_VALIDATION"];
$schedule_data =
substr($uploaded, $pos, $byte_counts["SCHEDULE"]);
$pos += $byte_counts["SCHEDULE"];
$index_data =
substr($uploaded, $pos);
}
$added_data = false;
if (strlen($robot_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::ROBOTS_FOLDER,
$robot_data);
$added_data = true;
}
if (C\USE_ETAG_EXPIRES && strlen($cache_page_validation_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::ETAG_EXPIRES_FOLDER,
$cache_page_validation_data);
$added_data = true;
}
if (strlen($schedule_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::SCHEDULES_FOLDER,
$schedule_data);
$added_data = true;
}
if (strlen($index_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::INDEX_DATA_FOLDER,
$index_data);
$added_data = true;
}
if ($added_data) {
/* Time on $this->crawl_status_file_name used to determine if crawl
stalled. We just uploaded data so not stalled
*/
touch($this->crawl_status_file_name);
}
return $logging;
}
/**
* Checks for the crawl time according either to crawl_status.txt or to
* NetworkStatus.txt, and presents it to the requesting fetcher, along
* with a list of available queue servers.
*/
public function crawlTime()
{
$info = [];
$info[self::STATUS] = self::CONTINUE_STATE;
$view = "fetch";
$cron_model = $this->model("cron");
$channel = $this->getChannel();
if (isset($_REQUEST['crawl_time'])) {;
$prev_crawl_time = substr(
$this->clean($_REQUEST['crawl_time'], 'int'), 0,
C\TIMESTAMP_LEN);
} else {
$prev_crawl_time = 0;
}
$cron_time = $cron_model->getCronTime("fetcher_restart");
$delta = time() - $cron_time;
if ($delta > self::CRON_INTERVAL) {
$cron_model->updateCronTime("fetcher_restart");
$this->model("machine")->restartCrashedFetchers();
} else if ($delta == 0) {
$cron_model->updateCronTime("fetcher_restart");
}
$network_filename = C\SCHEDULES_DIR .
"/{$channel}-" . self::network_status_file;
if (file_exists($this->crawl_status_file_name)) {
$crawl_status = unserialize(file_get_contents(
$this->crawl_status_file_name));
$crawl_time = (isset($crawl_status["CRAWL_TIME"])) ?
$crawl_status["CRAWL_TIME"] : 0;
$sleep_start = $crawl_status["SLEEP_START"] ?? "00:00";
$sleep_duration = $crawl_status["SLEEP_DURATION"] ?? "-1";
} else if (file_exists($network_filename)) {
$net_status = unserialize(file_get_contents($network_filename));
$crawl_time = $net_status[self::CRAWL_TIME] ?? 0;
$sleep_start = $net_status[self::SLEEP_START] ?? "00:00";
$sleep_duration = $net_status[self::SLEEP_DURATION] ?? "-1";
} else {
$crawl_time = 0;
$sleep_start = "00:00";
$sleep_duration = "-1";
}
$info[self::CRAWL_TIME] = $crawl_time;
$info[self::SLEEP_START] = $sleep_start;
$info[self::SLEEP_DURATION] = $sleep_duration;
$status_filename = CrawlDaemon::getMessageFileName("NameServer",
$channel);
if ($crawl_time != 0 && file_exists($status_filename)) {
$status = unserialize(file_get_contents($status_filename));
if ($status[self::STATUS] == 'STOP_CRAWL') {
$info[self::STATUS] == 'STOP_CRAWL';
$info[self::CRAWL_TIME] = 0;
} else {
/* this is supposed to slow down fetchers
if the indexer has a lot to still index
$mult_factor * C\MINIMUM_FETCH_LOOP_TIME
gets bigger up to a maximum of C\PROCESS_TIMEOUT/2
C\PROCESS_TIMEOUT also determines if Yioop
thinks the indexes is dead, so we don't what the
MINIMUM_FETCH_LOOP_TIME to be too big. Note:
If the name server isn't being used for crawling,
the number of $mult_factors below will always be 1,
so this will always set the fetch loop time to
C\MINIMUM_FETCH_LOOP_TIME
*/
$tmp_base_dir = C\SCHEDULES_DIR . "/".
self::index_data_base_name . $crawl_time;
$tmp_dirs = glob($tmp_base_dir . '/*', GLOB_ONLYDIR);
$mult_factor = max(1, count($tmp_dirs));
$info[self::MINIMUM_FETCH_LOOP_TIME] = max(min(
$mult_factor * C\MINIMUM_FETCH_LOOP_TIME,
C\PROCESS_TIMEOUT/2), C\MINIMUM_FETCH_LOOP_TIME);
}
if ($status[self::STATUS] != 'STOP_CRAWL' &&
$crawl_time != $prev_crawl_time) {
$to_copy_fields = [self::ALLOWED_SITES, self::ARC_DIR,
self::ARC_TYPE, self::CRAWL_INDEX, self::CRAWL_TYPE,
self::DISALLOWED_SITES, self::SITE_COOKIES,
self::INDEXED_FILE_TYPES,
self::PROXY_SERVERS, self::RESTRICT_SITES_BY_URL,
self::SUMMARIZER_OPTION, self::TOR_PROXY
];
foreach ($to_copy_fields as $field) {
if (isset($status[$field])) {
$info[$field] = $status[$field];
}
}
/*
When initiating a new crawl AND there are active
classifiers (an array of class labels), then augment the
info with compressed, serialized versions of each active
classifier so that each fetcher can reconstruct the same
classifiers.
*/
$classifier_array = [];
if (isset($status[self::ACTIVE_CLASSIFIERS])) {
$classifier_array = array_merge(
$status[self::ACTIVE_CLASSIFIERS]);
$info[self::ACTIVE_CLASSIFIERS] =
$status[self::ACTIVE_CLASSIFIERS];
}
if (isset($status[self::ACTIVE_RANKERS])) {
$classifier_array = array_merge($classifier_array,
$status[self::ACTIVE_RANKERS]);
$info[self::ACTIVE_RANKERS] =
$status[self::ACTIVE_RANKERS];
}
if ($classifier_array != []) {
$classifiers_data = Classifier::loadClassifiersData(
$classifier_array);
$info[self::ACTIVE_CLASSIFIERS_DATA] = $classifiers_data;
}
}
}
$info[self::SCRAPERS] = base64_encode(
serialize($this->model("scraper")->getAllScrapers()));
$info[self::QUEUE_SERVERS] =
$this->model("machine")->getQueueServerUrls(0, $channel);
$info[self::SAVED_CRAWL_TIMES] = $this->getCrawlTimes();
$info[self::POST_MAX_SIZE] = $this->postMaxSize();
/* A fetcher that has no Web Bot Auth key pair asks for the name
server's, so every fetcher of a crawl signs with the key the
name server publishes. It is sent only on a request that came
by https, so the private key never crosses in the clear. */
if (!empty($_REQUEST['web_bot_auth']) && !empty($_SERVER['HTTPS']) &&
$_SERVER['HTTPS'] != 'off' &&
L\FetchUrl::webBotAuthAgent() !== "" &&
function_exists("sodium_crypto_sign_keypair")) {
$info[self::WEB_BOT_AUTH_KEY] = base64_encode(
L\FetchUrl::webBotAuthKeyPair());
}
if (count($info[self::QUEUE_SERVERS]) == 0 && $channel == 0) {
$info[self::QUEUE_SERVERS] = [C\p('NAME_SERVER')];
}
$data = [];
$data['MESSAGE'] = serialize($info);
$this->displayView($view, $data);
}
/**
* Gets a list of all the timestamps of previously stored crawls
*
* This could probably be moved to crawl model. It is a little lighter
* than getCrawlList and should be only used with a name server so leaving
* it here so it won't be confused.
*
* @return array list of timestamps
*/
public function getCrawlTimes()
{
$list = [];
$dirs = glob(C\CACHE_DIR . '/*');
foreach ($dirs as $dir) {
if (strlen($pre_timestamp = strstr($dir,
self::index_data_base_name)) > 0) {
$list[] = substr($pre_timestamp,
strlen(self::index_data_base_name));
}
if (strlen($pre_timestamp = strstr($dir,
self::network_base_name)) > 0) {
$tmp = substr($pre_timestamp,
strlen(self::network_base_name), -4);
if (is_numeric($tmp)) {
$list[] = $tmp;
}
}
}
$list = array_unique($list);
return $list;
}
}
- Yioop Repo - Seekquarry