<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\media_jobs;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\processors\PageProcessor;
use seekquarry\yioop\models\CrawlModel;
use seekquarry\yioop\models\ImpressionModel;
use seekquarry\yioop\models\MachineModel;
use seekquarry\yioop\models\PhraseModel;

/**
 * A media job used to periodically calculate summary statistics about
 * group, thread, page, and query impressions.
 */
class AnalyticsJob extends MediaJob
{
    /**
     * Time in current epoch when analytics last updated
     * @var int
     */
    public $update_time;
    /**
     * Used to get statistics from DBMS about wiki and thread views
     *
     * @var object
     */
    public $impression_model;
    /**
     * Used to get crawl statistics about the number of various HTTP response
     * requests seen during a crawl
     *
     * @var object
     */
    public $phrase_model;
    /**
     * Used to determine which queue servers are available and which might
     * have information about a crawl
     *
     * @var object
     */
    public $machine_model;
    /**
     * Used to get crawl seed info
     *
     * @var object
     */
    public $crawl_model;
    /**
     * For size and time distributions the number of times the minimal
     * recorded interval (DOWNLOAD_SIZE_INTERVAL for size) to check for
     * pages with that size/download time
     */
    const NUM_TIMES_INTERVAL = 50;
    /**
     * While computing the statistics page, number of seconds until a
     * page refresh and save of progress so far occurs
     */
    const STATISTIC_REFRESH_RATE = C\ANALYTICS_UPDATE_INTERVAL/2;
    /**
     * Initializes the time of last analytics update
     */
    public function init()
    {
        $this->update_time = 0;
        $this->name_server_does_client_tasks = true;
        $this->name_server_does_client_tasks_only = true;
        $this->impression_model = new ImpressionModel();
        $this->phrase_model = new PhraseModel();
        $this->machine_model = new MachineModel();
        $this->crawl_model = new CrawlModel();
        PageProcessor::initializeIndexedFileTypes();
    }
    /**
     * Only update if its been more than an hour since the last update
     *
     * @return bool whether its been an hour since the last update
     */
    public function checkPrerequisites()
    {
        $time = time();
        $something_updated = false;
        $delta = $time - $this->update_time;
        if ($delta > C\ANALYTICS_UPDATE_INTERVAL) {
            $this->update_time = $time;
            L\crawlLog("Performing analytics update");
            return true;
        }
        L\crawlLog("Time since last update not exceeded, skipping analytics".
            " update");
        return false;
    }
    /**
     * For now analytics update is only done on name server as Yioop
     * currently only supports one DBMS at a time.
     */
    public function nondistributedTasks()
    {
        $this->doTasks([]);
    }
    /**
     * Calls ImpressionModel to actually calculate various impression totals
     * since the last update
     *
     * @param array $tasks array of info that came from getTasks
     * (in this nothing)
     */
    public function doTasks($tasks)
    {
        $this->computeCrawlStatistics();
        $this->impression_model->computeStatistics();
    }
    /**
     * Runs the queries necessary to determine httpd code distribution,
     * filetype distribution, num hosts, language distribution,
     * os distribution, server distribution, site distribution, file size
     * distribution, download time distribution, etc for a web crawl
     * for which statistics have been requested but not yet computed.
     * If these queries take too long it saves partial results and returns.
     *
     * @return mixed the populated stats $data array on completion;
     *      void on early-out (no pending request or unprocessable
     *      request file)
     */
    public function computeCrawlStatistics()
    {
        $stats_requests =
            glob(C\CACHE_DIR . "/pre_" . self::statistics_base_name . "*.txt");
        if (empty($stats_requests)) {
            L\crawlLog("No statistics for particular crawls requested");
            return;
        }
        $pre_stats_file = $stats_requests[0];
        $stats_file = str_replace("pre_", "", $pre_stats_file);
        $data = unserialize(file_get_contents($pre_stats_file));
        if (empty($data["TIMESTAMP"])) {
            L\crawlLog("Request " . $pre_stats_file . " could not be ".
                "processed. Deleting request file.");
            unlink($pre_stats_file);
            return;
        }
        L\crawlLog("Starting to compute statistics for timestamp index ".
            $data["TIMESTAMP"]);
        $machine_urls = $this->machine_model->getQueueServerUrls();
        $seed_info = $this->crawl_model->getCrawlSeedInfo($data["TIMESTAMP"],
            $machine_urls);
        $num_machines = count($machine_urls);
        if ($num_machines <  1 || ($num_machines ==  1 &&
            UrlParser::isLocalhostUrl($machine_urls[0]))) {
            $machine_urls = [];
        }
        $queries = [
            "CODE" => [100, 101, 102, 103, 122, 200, 201, 202, 203, 204,
                205, 206, 207, 208, 226, 301, 302, 303, 304, 305, 306, 307,
                308, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410,
                411, 412, 413, 414, 415, 416, 417, 418, 420, 422, 423, 424,
                425, 426, 428, 429, 431, 444, 449, 450, 499, 500, 501, 502,
                503, 504, 505, 506, 507, 508, 509, 510, 511, 598, 599],
            "FILETYPE" => PageProcessor::$indexed_file_types,
            "HOST" => ["all"],
            "LANG" => [ 'aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar',
                'as', 'av', 'ay', 'az', 'ba', 'be', 'bg', 'bh', 'bi', 'bm',
                'bn', 'bo', 'br', 'bs', 'ca', 'ce', 'ch', 'co', 'cr', 'cs',
                'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee', 'el', 'en',
                'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr',
                'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv', 'ha', 'he', 'hi',
                'ho', 'hr', 'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig',
                'ii', 'ik', 'in', 'io', 'is', 'it', 'iu', 'iw', 'ja', 'ji',
                'jv', 'jw', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn',
                'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb', 'lg',
                'li', 'ln', 'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk',
                'ml', 'mn', 'mo', 'mr', 'ms', 'mt', 'my', 'na', 'nb', 'nd',
                'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv', 'ny', 'oc', 'oj',
                'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps', 'pt', 'qu', 'rm',
                'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'sh',
                'si', 'sk', 'sl', 'sm', 'sn', 'so', 'sq', 'sr', 'ss', 'st',
                'su', 'sv', 'sw', 'ta', 'te', 'tg', 'th', 'ti', 'tk', 'tl',
                'tn', 'to', 'tr', 'ts', 'tt', 'tw', 'ty', 'ug', 'uk', 'ur',
                'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', 'yo', 'za',
                'zh', 'zu'],
            "MEDIA" => ["image", "text", "video"],
            "OS" => ["asianux", "centos", "clearos", "debian", "fedora",
                "freebsd", "gentoo", "linux", "netware", "solaris", "sunos",
                "ubuntu", "unix"],
            "SERVER" => ["aolserver", "apache", "bigip", "boa", "caudium",
                "cherokee", "gws", "goahead-webs", "httpd", "iis",
                "ibm_http_server", "jetty", "lighttpd", "litespeed",
                "microsoft-iis", "nginx", "resin", "server", "sun-java-system",
                "thttpd", "tux", "virtuoso", "webrick", "yaws", "yts",
                "zeus", "zope"],
            "SITE" => [".aero", ".asia", ".biz", ".cat", ".com", ".coop",
                ".edu", ".gov", ".info", ".int", ".jobs", ".mil", ".mobi",
                ".museum", ".name", ".net", ".org", ".pro", ".tel", ".travel",
                ".xxx", ".ac", ".ad", ".ae", ".af", ".ag", ".ai", ".al", ".am",
                ".ao", ".aq", ".ar", ".as", ".at", ".au", ".aw", ".ax", ".az",
                ".ba", ".bb", ".bd", ".be", ".bf", ".bg", ".bh", ".bi", ".bj",
                ".bm", ".bn", ".bo", ".br", ".bs", ".bt", ".bw", ".by", ".bz",
                ".ca", ".cc", ".cd", ".cf", ".cg", ".ch", ".ci", ".ck", ".cl",
                ".cm", ".cn", ".co", ".cr", ".cu", ".cv", ".cx", ".cy", ".cz",
                ".de", ".dj", ".dk", ".dm", ".do", ".dz", ".ec", ".ee", ".eg",
                ".er", ".es", ".et", ".eu", ".fi", ".fj", ".fk", ".fm", ".fo",
                ".fr", ".ga", ".gd", ".ge", ".gf", ".gg", ".gh", ".gi", ".gl",
                ".gm", ".gn", ".gp", ".gq", ".gr", ".gs", ".gt", ".gu", ".gw",
                ".gy", ".hk", ".hm", ".hn", ".hr", ".ht", ".hu", ".id", ".ie",
                ".il", ".im", ".in", ".io", ".iq", ".ir", ".is", ".it", ".je",
                ".jm", ".jo", ".jp", ".ke", ".kg", ".kh", ".ki", ".km", ".kn",
                ".kp", ".kr", ".kw", ".ky", ".kz", ".la", ".lb", ".lc", ".li",
                ".lk", ".lr", ".ls", ".lt", ".lu", ".lv", ".ly", ".ma", ".mc",
                ".md", ".me", ".mg", ".mh", ".mk", ".ml", ".mm", ".mn", ".mo",
                ".mp", ".mq", ".mr", ".ms", ".mt", ".mu", ".mv", ".mw", ".mx",
                ".my", ".mz", ".na", ".nc", ".ne", ".nf", ".ng", ".ni", ".nl",
                ".no", ".np", ".nr", ".nu", ".nz", ".om", ".pa", ".pe", ".pf",
                ".pg", ".ph", ".pk", ".pl", ".pm", ".pn", ".pr", ".ps", ".pt",
                ".pw", ".py", ".qa", ".re", ".ro", ".rs", ".ru", ".rw", ".sa",
                ".sb", ".sc", ".sd", ".se", ".sg", ".sh", ".si", ".sk", ".sl",
                ".sm", ".sn", ".so", ".sr", ".ss", ".st", ".sv", ".sy", ".sz",
                ".tc", ".td", ".tf", ".tg", ".th", ".tj", ".tk", ".tl", ".tm",
                ".tn", ".to", ".tr", ".tt", ".tv", ".tw", ".tz", ".ua", ".ug",
                ".uk", ".us", ".uy", ".uz", ".va", ".vc", ".ve", ".vg", ".vi",
                ".vn", ".vu", ".wf", ".ws", ".ye", ".za", ".zm", ".zw" ],
        ];
        for ($i = 0; $i <= self::NUM_TIMES_INTERVAL; $i++) {
            $queries["SIZE"][] = $i * C\DOWNLOAD_SIZE_INTERVAL;
        }
        for ($i = 0; $i <= self::NUM_TIMES_INTERVAL; $i++) {
            $queries["TIME"][] = $i * C\DOWNLOAD_TIME_INTERVAL;
        }
        for ($i = 0; $i <= self::NUM_TIMES_INTERVAL; $i++) {
            $queries["DNS"][] = $i * C\DOWNLOAD_TIME_INTERVAL;
        }
        for ($i = 0; $i <=max(
            $seed_info["general"]['max_links_to_extract'] ?? 0,
            C\MAX_LINKS_TO_EXTRACT); $i++) {
            $queries["NUMLINKS"][] = $i;
        }
        $date = date("Y");
        for ($i = 1969; $i <= $date; $i++) {
            $queries["MODIFIED"][] = $i;
        }
        $sort_fields = ["CODE", "FILETYPE", "LANG", "MEDIA", "OS",
            "SERVER", "SITE"];
        $time = time();
        if (isset($data["UNFINISHED"])) {
            unset($data["UNFINISHED"]);
        }
        foreach ($queries as $group_description => $query_group) {
            $total = 0;
            foreach ($query_group as $query) {
                L\crawlTimeoutLog("Processing crawl statistics about ".
                    "$group_description $query");
                //avoid casting float to int by making a string
                $group_description = "$group_description";
                $query = "$query";
                if (isset($data["SEEN"][$group_description][$query])) {
                    if (isset($data[$group_description]["DATA"][$query])) {
                        $total += $data[$group_description]["DATA"][$query];
                    }
                    continue;
                }
                $count =
                    $this->countQuery(strtolower($group_description)
                        .":".$query, $data["TIMESTAMP"], $machine_urls);
                $data["SEEN"][$group_description][$query] = true;
                if ($count >= 0) {
                    $data[$group_description]["DATA"][$query] = $count;
                    $total += $count;
                }
                if (time() - $time > self::STATISTIC_REFRESH_RATE) {
                    $data["UNFINISHED"] = true;
                    break 2;
                }
            }
            if (isset($data[$group_description]["DATA"])) {
                if (in_array($group_description, $sort_fields)) {
                    arsort($data[$group_description]["DATA"]);
                }
                $data[$group_description]["TOTAL"] = $total;
            }
        }
        $data["OS"]["DATA"]["windows"] = 0;
        if (isset($data["SERVER"]["DATA"]["iis"])) {
            $data["OS"]["DATA"]["windows"] = $data["SERVER"]["DATA"]["iis"];
        }
        if (isset($data["SERVER"]["DATA"]["microsoft-iis"])) {
            $data["OS"]["DATA"]["windows"] +=
                $data["SERVER"]["DATA"]["microsoft-iis"];
        }
        arsort($data["OS"]["DATA"]);
        if (empty($data["UNFINISHED"])) {
            unlink($pre_stats_file);
            file_put_contents($stats_file, serialize($data));
            chmod($stats_file, 0777);
            L\crawlLog("Done computing crawl statistics in " . $stats_file);
        } else {
            file_put_contents($pre_stats_file, serialize($data));
            chmod($pre_stats_file, 0777);
            L\crawlLog("Saving partial crawl statistics in " . $pre_stats_file);
        }
        return $data;
    }
    /**
     * Performs the provided $query of a web crawl (potentially distributed
     * across queue servers). Returns the count of the number of results that
     * would be returned by that query.
     *
     * @param string $query to use and count the results of
     * @param string $index_timestamp timestamp of index to compute query
     *      count for
     * @param array $machine_urls queue servers on which the count is to be
     *      computed
     * @return int number of results that would be returned by the given query
     */
    public function countQuery($query, $index_timestamp, $machine_urls)
    {
        $results = $this->phrase_model->getPhrasePageResults(
            "$query i:$index_timestamp", 0,
            1, true, null, false, 0, $machine_urls);
        return (isset($results["TOTAL_ROWS"])) ? $results["TOTAL_ROWS"] : -1;
    }
}
