<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\executables;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\CrawlDaemon;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\MediaConstants;
use seekquarry\yioop\library\media_jobs\MediaJob;
use seekquarry\yioop\library\WikiParser;

if (php_sapi_name() != 'cli' ||
    defined("seekquarry\\yioop\\configs\\IS_OWN_WEB_SERVER")) {
    echo "BAD REQUEST";
    exit();
}
/** We do want logging, but crawl model and others will try to turn off
 * if we don't set this
 */
$_SERVER["NO_LOGGING"] = false;
/** To guess language based on page encoding */
require_once __DIR__."/../library/LocaleFunctions.php";
/** General utility functions used by mail and other media jobs */
require_once __DIR__."/../library/Utility.php";
if (!C\PROFILE) {
    echo "Please configure the search engine instance by visiting" .
        "its web interface on localhost.\n";
    exit();
}
ini_set("memory_limit", C\MEDIA_UPDATER_MEMORY_LIMIT);
/*
 * We'll set up multi-byte string handling to use UTF-8
 */
mb_internal_encoding("UTF-8");
mb_regex_encoding("UTF-8");
/**
 * Separate process/command-line script which can be used to update
 * news sources for Yioop and also handle other kinds of activities such as
 * video conversion. This is as an alternative to using the web app
 * for updating. Makes use of the web-apps code.
 *
 * @author Chris Pollett
 */
class MediaUpdater implements CrawlConstants
{
    /**
     * Shortest sleep at the bottom of the loop when a job did
     * work last iteration -- low so the next iteration starts
     * quickly and back-to-back ticks don't bleed the dead-air
     * that motivated the adaptive-sleep design.
     */
    const MIN_IDLE_LOOP_SLEEP = 1;
    /**
     * Cap on the back-off sleep when every job sat idle. With
     * MIN_IDLE_LOOP_SLEEP=1 and BACKOFF_GROWTH=2 the schedule
     * is 1, 2, 4, 8, 16, 30, 30, ... so a freshly-quiet
     * updater is responsive for about 5 iterations (~30s) and
     * then settles to one wake per half-minute when there is
     * really nothing to do.
     */
    const MAX_IDLE_LOOP_SLEEP = 30;
    /**
     * Multiplier applied to the idle sleep on each consecutive
     * no-work iteration.
     */
    const BACKOFF_GROWTH = 2;
    /**
     * The last time feeds were checked for updates
     * @var int
     */
    public $update_time;
    /**
     * If true then it is assumed that mail should be
     * sent using a media updater rather than from within the web app
     *
     * @var bool
     */
    public $mail_mode;
    /**
     * Controls whether media updating should be viewed as only occurring
     * on the name server or should it be viewed as a distributed process
     * amongst all machines in this Yioop instance
     * @var string
     */
    public $media_mode;
    /**
     * List of job this media updater performs
     * @var array
     */
    public $jobs;
    /**
     * Sets up the field variables so that media updating can begin
     */
    public function __construct()
    {
        global $argv;
        $this->update_time = 0;
        $this->media_mode = "name_server";
        $this->jobs = [];
    }
    /**
     * This is the function that should be called to get the MediaUpdater to
     * start to start updating. Calls init to handle the command-line
     * arguments then enters news_updaters main loop
     */
    public function start()
    {
        global $argv;
        CrawlDaemon::init($argv, "MediaUpdater");
        L\crawlLog("\n\nInitialize logger..", "MediaUpdater", true);
        $this->installStopSignalHandler();
        $this->loop();
    }
    /**
     * Installs a handler so a stop signal ends the updater promptly,
     * in the middle of a job if need be, rather than only at the gap
     * between jobs. MediaUpdater's work is safe to cut off this way:
     * an interrupted mail batch is gathered again and an interrupted
     * transcription is re-run on the next pass. Where the signal
     * functions are not present, for example on Windows, this does
     * nothing and the updater stops at the next gap between jobs as
     * before.
     */
    public function installStopSignalHandler()
    {
        if (!function_exists("pcntl_async_signals") ||
            !function_exists("pcntl_signal") || !defined("SIGTERM")) {
            return;
        }
        pcntl_async_signals(true);
        $stop = function ($signal_number) {
            L\crawlLog("MediaUpdater received a stop signal; ending now");
            exit();
        };
        pcntl_signal(SIGTERM, $stop);
        if (defined("SIGINT")) {
            pcntl_signal(SIGINT, $stop);
        }
    }
    /**
     * Main loop for the news updater. Sleeps adaptively: a
     * short sleep when any job did work last iteration, an
     * exponentially-growing sleep (capped at
     * MAX_IDLE_LOOP_SLEEP) when every job sat idle. The idle
     * backoff makes a no-clones-pending updater quiet (one
     * stat per half-minute) without giving up snappy response
     * the moment work appears.
     */
    public function loop()
    {
        L\crawlLog("In Media Update Loop");
        L\crawlLog("PHP Version in use:  " . phpversion());
        L\crawlLog("Max memory can use:  " . ini_get('memory_limit'));
        $info[self::STATUS] = self::CONTINUE_STATE;
        $local_archives = [""];
        $updater_start_time = date("r", time());
        $idle_sleep = self::MIN_IDLE_LOOP_SLEEP;
        while (CrawlDaemon::processHandler()) {
            L\crawlLog("MediaUpdater was started: " . $updater_start_time);
            $start_time = microtime(true);
            $this->getUpdateProperties();
            $any_worked = false;
            if (!empty($this->jobs)) {
                foreach ($this->jobs as $job_name => $job) {
                    if (!CrawlDaemon::processHandler(true)) {
                        L\crawlLog("Stop requested; skipping the rest " .
                            "of this MediaUpdater pass");
                        break;
                    }
                    $job->run();
                    if (!empty($job->worked_this_tick)) {
                        $any_worked = true;
                    }
                }
            }
            if ($any_worked) {
                $idle_sleep = self::MIN_IDLE_LOOP_SLEEP;
            } else {
                $idle_sleep = min(self::MAX_IDLE_LOOP_SLEEP,
                    max(self::MIN_IDLE_LOOP_SLEEP,
                    $idle_sleep * self::BACKOFF_GROWTH));
            }
            $elapsed = L\changeInMicrotime($start_time);
            $sleep_time = max(0, ceil($idle_sleep - $elapsed));
            if ($sleep_time > 0) {
                L\crawlLog("Ensure minimum loop time by sleeping...".
                    $sleep_time . ($any_worked ?
                    " (job worked)" : " (idle backoff)"));
                L\interruptibleSleep($sleep_time, 1, function () {
                    return !CrawlDaemon::processHandler(true);
                });
            }
        } //end while
        L\crawlLog("Media Updater shutting down!!");
    }
    /**
     * Makes a request to the name server to find out if we are running
     * as a media updater just on the name server or on both the name server
     * as well as all other machines in the Yioop instance
     */
    public function getUpdateProperties()
    {
        L\crawlLog("Checking Name Server for Media Updater properties...");
        $properties = MediaJob::execNameServer("getUpdateProperties");
        if ($properties) {
            if (isset($properties['MEDIA_MODE'])) {
                $this->media_mode = $properties['MEDIA_MODE'];
                L\crawlLog("...Setting media mode to: " .
                    $properties['MEDIA_MODE']);
            }
            if (isset($properties['SEND_MAIL_MEDIA_UPDATER'])) {
                $this->mail_mode = (
                    $properties['SEND_MAIL_MEDIA_UPDATER']== "true") ?
                    true : false;
                L\crawlLog("...Setting mail mode to: " .
                    (($this->mail_mode) ? "true" : "false"));
            }
            if (isset($properties['JOBS_LIST'])) {
                $this->loadJobs($properties['JOBS_LIST']);
            }
        }
        L\crawlLog("Done checking Name Server for Media Updater properties");
    }
    /**
     * Given a list of MediaUpdate jobs, updates $this->jobs to contain
     * instantiated objects of the the corresponding jobs, requiring
     * classes that have not been loaded yet as needed.
     *
     * @param array $jobs_list list of MediaUpdater jobs
     */
    public function loadJobs($jobs_list)
    {
        L\crawlLog("Updating list of jobs...");
        $job_path = C\BASE_DIR . "/library/media_jobs/";
        $app_job_path = C\APP_DIR . "/library/media_jobs/";
        $len_path = strlen($job_path);
        $job_files = [];
        foreach ($jobs_list as $job => $enabled) {
            if (!$enabled) {
                continue;
            }
            if (file_exists($app_job_path . "{$job}Job.php")) {
                $job_files[$job] = $app_job_path . "{$job}Job.php";
            } else if (file_exists($job_path . "{$job}Job.php")) {
                $job_files[$job] = $job_path . "{$job}Job.php";
            }
        }
        foreach ($this->jobs as $job_name => $job) {
            if (empty($job_files[$job_name])) {
                L\crawlLog("...Removing $job_name from list of jobs");
                unset($this->jobs[$job_name]);
            }
        }
        foreach ($job_files as $job_name => $job_file) {
            if (!empty($this->jobs[$job_name])) {
                continue;
            }
            L\crawlLog("...Adding $job_name to list of jobs");
            require_once $job_file;
            $job_class_name = C\NS_JOBS . $job_name . "Job";
            if ($job_class_name != C\NS_JOBS . "MediaJob") {
                $job = new $job_class_name($this);
                $this->jobs[$job_name] = $job;
                L\crawlLog("... done.");
            }
        }
    }
}
/*
 * Instantiate and run the MediaUpdater program
 */
$media_updater =  new MediaUpdater();
$media_updater->start();
