- Yioop Repo - Seekquarry

<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\media_jobs;
/* MediaUpdater's stop check is a tick function, which PHP calls only in
   code compiled with this line, so a stop can end this job midway. */
declare(ticks=1000);

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\models\CrawlModel;
use seekquarry\yioop\models\MachineModel;

/**
 * CrawlSpeedJob keeps a history of how fast each running crawl goes. Once
 * each CRAWL_SPEED_SAMPLE_INTERVAL (the seconds between two rows, ten
 * minutes unless a site sets another) it reads the status of the crawl on
 * each channel, the same status the Crawl Status screen shows, and logs a
 * line for each crawl running to CrawlSpeed.log in the log folder: the
 * time, the crawl's timestamp, the channel, the Visited Urls/Hour, the
 * visited urls count and the total urls seen. With no crawl running it
 * does nothing. CrawlModel::crawlSpeedHistory reads a crawl's lines back
 * for its Crawl Speed Data page in Manage Crawls. The job runs on the name
 * server only, which reads the status from the queue servers, so it works
 * when the name server runs no queue server itself.
 *
 * @author Chris Pollett
 */
class CrawlSpeedJob extends MediaJob
{
    /**
     * $update_time is the time, in seconds since the epoch, the job last
     * logged lines
     * @var int
     */
    public $update_time;
    /**
     * $crawl_model reads the crawl status with the rate the screen shows
     * @var object
     */
    public $crawl_model;
    /**
     * $machine_model gives the urls of the queue servers whose status is
     * read
     * @var object
     */
    public $machine_model;
    /**
     * init makes the models the job uses. A machine other than the name
     * server asks the name server for tasks and is given none, so only the
     * name server logs lines.
     */
    public function init()
    {
        $this->update_time = 0;
        $this->crawl_model = new CrawlModel();
        $this->machine_model = new MachineModel();
    }
    /**
     * checkPrerequisites says if the time between rows has passed since the
     * job last logged lines. That time is CRAWL_SPEED_SAMPLE_INTERVAL (the
     * seconds between two rows, ten minutes unless a site sets another).
     *
     * @return bool true if lines are to be logged now
     */
    public function checkPrerequisites()
    {
        $time = time();
        if ($time - $this->update_time >= C\CRAWL_SPEED_SAMPLE_INTERVAL) {
            $this->update_time = $time;
            return true;
        }
        $this->worked_this_tick = false;
        return false;
    }
    /**
     * nondistributedTasks logs the lines when MediaUpdater runs all jobs on
     * the name server
     */
    public function nondistributedTasks()
    {
        $this->logSpeedRows();
    }
    /**
     * prepareTasks logs the lines when MediaUpdater shares jobs among
     * machines. The name server runs it, and no other machine has a part
     * in this job.
     */
    public function prepareTasks()
    {
        $this->logSpeedRows();
    }
    /**
     * logSpeedRows reads the status of the crawls on each channel from the
     * queue servers and logs a line to CrawlSpeed.log in the log folder for
     * each crawl running, its values separated by tabs. A channel whose
     * status has no crawl time, or that is stopping, gets no line. MediaUpdater
     * runs its jobs one after another, so a long job before this one delays
     * a line; the time in each line shows when it was taken.
     */
    public function logSpeedRows()
    {
        $time = time();
        $statuses = $this->crawl_model->crawlStatus(
            $this->machine_model->getQueueServerUrls());
        $lines = "";
        foreach ($statuses as $status) {
            $description = $status['DESCRIPTION'] ?? "";
            if (empty($status['CRAWL_TIME']) ||
                str_starts_with($description, "SHUTDOWN")) {
                continue;
            }
            $lines .= implode("\t", [$time, $status['CRAWL_TIME'],
                $status['VISITED_URLS_COUNT_PER_HOUR'] ?? 0,
                $status['VISITED_URLS_COUNT'] ?? 0,
                ($status['COUNT'] ?? 0) + ($status['ACTIVE_COUNT'] ?? 0),
                $status['INDEXER_BACKLOG'] ?? 0,
                $status['FETCHER_DOWNLOADS_PER_HOUR'] ?? 0]) .
                "\n";
        }
        if ($lines == "") {
            L\crawlLog("No crawl running, so no speed line logged");
            return;
        }
        file_put_contents(C\LOG_DIR . "/" . self::crawl_speed_log, $lines,
            FILE_APPEND | LOCK_EX);
        L\crawlLog("Logged crawl speed lines:\n" . $lines);
    }
}