<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Parth Patel (modfied to better use xpaths Chris Pollett)
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\media_jobs;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\models\GroupModel;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\UrlParser;

/**
* A media job to periodically update descriptions of Wiki resources
* using Description Search Sources
*/
class DescriptionUpdateJob extends MediaJob
{
    /**
     * Time in current epoch when description last updated
     * @var int
     */
    public $update_time;
    /**
     * Resource and thumb folders for all the wiki pages that have resources
     * that need descriptions
     * @var array
     */
    public $page_id_thumb_folder_paths;
    /**
     * Datasource object used to run db queries related to fes items
     * @var object
     */
    public $db;
    /**
     * File to tell DescriptionUpdateJob that a wiki resource needs a
     * description
     */
    const NEEDS_DESCRIPTION_FILE = C\APP_DIR .
        "/resources/needs_descriptions.txt";
    /**
     * Initializes the last update time to far in the past so, description will
     * get immediately updated. Sets up connection to DB to fetch description
     * search sources
     */
    public function init()
    {
        $this->update_time = 0;
        $this->name_server_does_client_tasks = true;
        $this->name_server_does_client_tasks_only = true;
        $db_class = C\NS_DATASOURCES . ucfirst(C\p('DBMS')). "Manager";
        $this->db = new $db_class();
        $this->db->connect();
        C\nsconddefine("DESCRIPTION_UPDATE_INTERVAL", C\ONE_HOUR);
    }
    /**
     * Only update if its been more than a day since the last update
     * and there are resources requiring description update
     *
     * @return bool whether its been a daysince the last update
     */
    public function checkPrerequisites()
    {
        $time = time();
        $delta = $time - $this->update_time;
        if ($delta > C\DESCRIPTION_UPDATE_INTERVAL &&
            file_exists(self::NEEDS_DESCRIPTION_FILE) &&
            filesize(self::NEEDS_DESCRIPTION_FILE) > 0) {
            $this->update_time = $time;
            L\crawlLog("---- Performing resources description update ----");
            return true;
        }
        L\crawlLog("---- Time since last update not exceeded, " .
            "skipping description update ----");
        return false;
    }
    /**
     * Get the description search sources from the local database and use
     * those to run the same task as in the distributed setting
     */
    public function nondistributedTasks()
    {
        $db = $this->db;
        $sql = "SELECT * FROM MEDIA_SOURCE WHERE TYPE='description_source'";
        $result = $db->execute($sql);
        $sources = [];
        while ($source = $db->fetchArray($result)) {
            $this->parseDescriptionAuxInfo($source);
            $sources[] = $source;
        }
        $this->tasks = $sources;
        $this->doTasks($sources);
    }
    /**
     * Parses out the components of the auxiliary field of a description
     * source.
     *
     * @param array &$source associative array of data about one
     *  particular description
     */
    public static function parseDescriptionAuxInfo(&$source)
    {
        $aux_parts = explode("###", html_entity_decode(
            $source['AUX_INFO'], ENT_QUOTES));
        list($source['AUX_INFO'], $source['ITEM_XPATH'],
            $source['TITLE_XPATH'],$source['URL_XPATH'],
            , , , $source['TEST_DATA']) = $aux_parts;
    }
    /**
     * For each resource requiring description update, use the description
     * search sources to find information
     *
     * @param array $tasks array of description sources
     */
    public function doTasks($tasks)
    {
        $this->page_id_thumb_folder_paths = explode("\n",
            file_get_contents(self::NEEDS_DESCRIPTION_FILE));
        $this->page_id_thumb_folder_paths = array_unique(
            $this->page_id_thumb_folder_paths);
        if (!is_array($tasks) || !is_array($this->page_id_thumb_folder_paths)) {
            L\crawlLog(
                "---- This media updater is NOT responsible for " .
                "any description update! ----");
            return;
        }
        L\crawlLog("---- This media updater is responsible for " .
            "the description updates ----");
        $page_id_thumb_folder_paths = $this->page_id_thumb_folder_paths;
        foreach ($page_id_thumb_folder_paths as $page_id_thumb_folder_path) {
            $time = time();
            if ($time - $this->update_time >= C\ONE_HOUR) {
                L\crawlLog("---- Runtime limit exceeded, saving the current " .
                    "state and yielding the processor ----");
                file_put_contents(self::NEEDS_DESCRIPTION_FILE, implode(PHP_EOL,
                    $this->page_id_thumb_folder_paths));
                return;
            }
            $this->updateResourcesDescription($tasks,
                $page_id_thumb_folder_path);
            array_shift($this->page_id_thumb_folder_paths);
        }
        file_put_contents(self::NEEDS_DESCRIPTION_FILE, "");
    }
    /**
     * Updates/finds descriptions for resources listed in a
     * needs_description.txt in a wiki pages thumb subfolder.
     * It does this by iterating over all configured description search sources
     * a until a match is found. It then saves the description in file at given
     * resource thumb folder path
     *
     * @param array $sources associative array containing details of all search
     *      sources
     * @param string $page_id_thumb_folder_path colon-separated
     *  "page_id:thumb_folder_path" identifying the wiki page resource
     *  whose needs_description.txt should be processed
     * @param boolean $test_mode used to return string in test mode
     * @return string if $test_mode true
     */
    public function updateResourcesDescription($sources,
        $page_id_thumb_folder_path = "", $test_mode = false)
    {
        $page_name = "";
        $thumb_folder_path = "";
        $page_path_parts = explode(":", $page_id_thumb_folder_path, 2);
        if (count($page_path_parts) == 2) {
            list($page_id, $thumb_folder_path) = $page_path_parts;
            $group_model = new GroupModel();
            $page_info = $group_model->getPageInfoByPageId($page_id);
            $page_name = $page_info['PAGE_NAME'] ?? "";
        } else if (!$test_mode) {
            return false;
        }
        if (!$test_mode && !file_exists($thumb_folder_path)) {
            return false;
        }
        $test_results = "";
        $log_function = function ($msg, $log_tag = "div class='source-test'")
            use (&$test_results, $test_mode) {
            $close_tag= preg_split("/\s+/",$log_tag)[0];
            if ($test_mode) {
                $test_results .=
                    "<$log_tag style='overflow-x: scroll;'>$msg</$close_tag>\n";
            } else {
                L\crawlLog($msg);
            }
        };
        $file_path = $thumb_folder_path . "/needs_description.txt";
        if (!$test_mode) {
            $log_function("---- Processing file $file_path ----");
        }
        $resource_details = !$test_mode ?
            (file_exists($file_path) ? file($file_path) : [] ) :
            explode("\n", $sources[0]['TEST_DATA']);
        $i = 1;
        $resource_details = array_filter($resource_details);
        $resource_details_copy = $resource_details;
        foreach ($resource_details as $resource_detail) {
            $time = time();
            if (!$test_mode && $time - $this->update_time >= C\ONE_HOUR) {
                file_put_contents($file_path, implode(PHP_EOL,
                    $resource_details_copy));
                return true;
            }
            $log_function("Processing $i - $resource_detail", "h3");
            array_shift($resource_details_copy);
            $resource_detail = trim($resource_detail);
            $resource_name = trim(preg_replace('/\s+/', ' ', $resource_detail));
            $resource_name = pathinfo($resource_name)['filename'];
            $resource_name = preg_replace('/\s+/', '%20', $resource_name);
            $max_score = 0;
            $details_page_url = "";
            $found_details = false;
            $details = "";
            foreach ($sources as $source) {
                $source_name = $source['NAME'];
                $log_function("***Seeing if search source <b>$source_name</b>".
                    " applies or not***", "pre");
                if ($additional_search_items =
                    $this->matchResourceSourcePathTerms(
                    $page_name . "/". $thumb_folder_path . "/" .
                    $resource_detail, $source['CATEGORY'])) {
                    $log_function("*** Using search source <b>$source_name" .
                        "</b> to find description ***", "p");
                    $query = (is_string($additional_search_items)) ?
                        urlencode($additional_search_items) . "%20" .
                        $resource_name : $resource_name;
                    $search_page_url = $source['SOURCE_URL'] . $query;
                    $log_function(" Search Page URL - $search_page_url", "pre");
                    $search_page = FetchUrl::getPage($search_page_url);
                    if (empty($search_page)) {
                        $log_function("<span class='red'>No search results".
                        "found for $resource_name</span>", "p");
                        continue;
                    }
                    set_error_handler(null);
                    $dom = L\getDomFromString($search_page);
                    $dom_xpath = new \DOMXPath($dom);
                    $items = @$dom_xpath->evaluate($source['ITEM_XPATH']);
                    restore_error_handler();
                    foreach ($items as $item) {
                        $processed_result = $this->processItem($item,
                            $resource_name, $source, $dom, $test_mode);
                        $test_results .= $processed_result[2];
                        if ($processed_result[0] > $max_score) {
                            $max_score = $processed_result[0];
                            $details_page_url = $processed_result[1];
                        }
                    }
                    if (!empty($details_page_url)) {
                        $log_function("<b>Selected Details Page URL - " .
                            "$details_page_url</b>", "pre");
                        $details_page = FetchUrl::getPage($details_page_url);
                        if (empty($details_page)) {
                            $log_function("<span class='red'>Details page".
                            " not available</span>", "p");
                            continue;
                        }
                        list($details, $test_info) = $this->getDetails(
                            $details_page, $source, $test_mode);
                        $test_results .= $test_info;
                    }
                } else {
                    $log_function("*** ...does not***", "pre");
                }
            }
            if (!empty($details)) {
                $found_details = true;
                if ($test_mode) {
                    $log_function("*** Found below details ***",
                        "p");
                    $log_function("$details", "pre");
                } else {
                    file_put_contents($thumb_folder_path .
                        "/$resource_detail.txt", $details);
                }
            }
            if (!$found_details && !$test_mode) {
                file_put_contents($thumb_folder_path .
                    "/$resource_detail.txt", "Description search sources".
                        " failed to find description.");
            }
            $i++;
        }
        if (!$test_mode) {
            file_put_contents($file_path, "");
        }
        return $test_mode ? $test_results : true;
    }
    /**
     * Checks if the terms wiki page name followed by a path to a wiki resource
     * contain the terms in a description search source string which would
     * trigger that search source to get used
     * @param string $page_name_resource_path path to check terms against
     *  search source trigger terms. The mimetype of the resource is also
     *  added to the list of terms to check
     * @param string $source_regex a regex used on the $page_name_resource_path
     *  to determine if a lookup should be done for the resource
     * @return string|bool additional search terms or true if there was a match,
     *  null otherwise
     */
    public function matchResourceSourcePathTerms($page_name_resource_path,
        $source_regex)
    {
        $page_name_resource_path = preg_replace("/\_/u", " ",
            $page_name_resource_path);
        if(empty($source_regex)) {
            return false;
        }
        if ($source_regex[0] != $source_regex[strlen($source_regex) - 1]) {
            $source_regex = "/$source_regex/";
        }
        if (preg_match($source_regex, $page_name_resource_path, $matches)) {
            return ($matches[1] ?? true);
        }
        return false;
    }
    /**
     * Processes $item, a DOMElement representing a search result for
     * a description for the wiki resource $name, extracting a title and
     * url. Form the title a match score with $name is obtained. This score
     * and url as well as in test mode log messages are returned.
     *
     * @param $item DOMNode representing one possible description search result
     * @param $name the wiki resource name we are trying to get a description
     *  of
     * @param $source the source associative array with information about how
     *  to extract description from the current dom document and dom  node.
     * @param $dom DOMDocument of whole document node is from, used in
     *  creating DOMXpath object for quering $item.
     * @param bool $test_mode when true, descriptive HTML is accumulated
     *  into $test_results instead of being written to the crawl log
     * @return array $score, $url, $test_results $score of $item as a likely
     *  source for a description for the wiki resource $name, $url
     *  that $item point to with more information, $test_results log messages
     *  if in test mode.
     */
    public function processItem($item, $name, $source, $dom, $test_mode = false)
    {
        if (!$item->hasChildNodes()) {
            return [0, null, ""];
        }
        $test_results = "";
        $log_function = function ($msg, $log_tag = "div class='source-test'")
            use (&$test_results, $test_mode) {
            $close_tag = preg_split("/\s+/",$log_tag)[0];
            if ($test_mode) {
                $test_results .=
                    "<$log_tag style='overflow-x: scroll;'>$msg</$close_tag>\n";
            } else {
                L\crawlLog($msg);
            }
        };
        $log_function("*** Processing item ***", "p");
        $dom_xpath = new \DOMXPath($dom);
        $title = "";
        if ($source['TITLE_XPATH'][0] == "/") {
            $source['TITLE_XPATH'] = "." . $source['TITLE_XPATH'];
        }
        $title_nodes = @$dom_xpath->evaluate($source['TITLE_XPATH'], $item);
        if (!empty($title_nodes) && !empty($title_nodes->item(0))) {
            $title = trim(mb_strtolower($title_nodes->item(0)->textContent));
            similar_text($name, $title, $score);
        }
        if ($source['URL_XPATH'][0] == "/") {
            $source['URL_XPATH'] = "." . $source['URL_XPATH'];
        }
        $url_nodes = @$dom_xpath->evaluate($source['URL_XPATH'], $item);
        $url_parts = parse_url($source['SOURCE_URL']);
        $base_url = $url_parts['scheme']."://".$url_parts['host'];
        if (!empty($url_nodes) && !empty($url_nodes->item(0))) {
            $url = $url_nodes->item(0)->textContent;
            $url = UrlParser::canonicalLink($url, $base_url);
        }
        if (!empty($title) && !empty($url)) {
            $log_function(" <b>Title:</b> $title", "pre");
            $log_function(" <b>URL:</b> $url", "pre");
            $log_function(" <b>Title Match Percentage:</b> $score", "pre");
            return [$score, $url, $test_results];
        }
        return [0, null, $test_results];
    }
    /**
     * Fetches the details on the url page using the xpaths
     * values configured in search source
     *
     * @param $page string the html string of the details page
     * @param $source array search source details
     * @param bool $test_mode when true, descriptive HTML is accumulated
     *  into the test_results section of the output instead of being
     *  written to the crawl log
     * @return string details found using xpaths
     */
    public function getDetails($page, $source, $test_mode = false)
    {
        $test_results = "";
        $log_function = function ($msg, $log_tag = "div class='source-test'")
            use (&$test_results, $test_mode) {
            $close_tag= preg_split("/\s+/",$log_tag)[0];
            if ($test_mode) {
                $test_results .=
                    "<$log_tag style='overflow-x: scroll;'>$msg</$close_tag>\n";
            } else {
                L\crawlLog($msg);
            }
        };
        set_error_handler(null);
        $dom = L\getDomFromString($page);
        $details = "";
        restore_error_handler();
        if (empty($dom)) {
            $log_function("<span class='red'>Error creating DOM</span>",
                "pre");
            return $details;
        }
        $detail_items = array_filter(explode("\n", $source['AUX_INFO']));
        $details = "";
        foreach ($detail_items as $detail_item) {
            $info_found = false;
            $sub_details = "";
            list($detail_name, $detail_xpath, ) = explode("|", $detail_item);
            $detail_name = trim($detail_name);
            $dom_xpath = new \DOMXPath($dom);
            $detail_nodes = @$dom_xpath->evaluate($detail_xpath);
            if (!empty($detail_nodes)) {
                if (is_string($detail_nodes)) {
                    $sub_details .= trim(preg_replace('/\s+/', ' ',
                        $detail_nodes)) . "\n";
                    $info_found = true;
                } else {
                    foreach ($detail_nodes as $detail_node) {
                        if (!empty($detail_node->textContent)) {
                            $entry_text = trim(preg_replace('/\s+/', ' ',
                                $detail_node->textContent)) . "\n";
                            if (!str_contains($sub_details, $entry_text)) {
                                $sub_details .= $entry_text;
                                $info_found = true;
                            }
                        }
                    }
                }
            }
            if (!$info_found) {
                $log_function("<span class='red'>Could not fetch value".
                    " for <b>$detail_name</b></span>", "pre");
            } else {
                $details .= "= $detail_name =\n" . $sub_details . "\n";
            }
        }
        return [wordwrap($details), $test_results];
    }
}
