<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\archive_bundle_iterators;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\BZip2BlockIterator;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\WikiParser;

/** For crawlHash*/
require_once __DIR__."/../Utility.php";
/**
 * Used to iterate through a collection of .xml.bz2  media wiki files
 * stored in a WebArchiveBundle folder. Here these media wiki files contain the
 * kinds of documents used by wikipedia. Iteration would be
 * for the purpose making an index of these records
 *
 * @author Chris Pollett
 * @see WebArchiveBundle
 */
class MediaWikiArchiveBundleIterator extends TextArchiveBundleIterator
{
    /**
     * Used to define the styles we put on cache wiki pages
     */
    const WIKI_PAGE_STYLES = <<<EOD
<style>
table.wikitable
{
    background:white;
    border:1px #aaa solid;
    border-collapse: scollapse
    margin:1em 0;
}
table.wikitable > tr > th,table.wikitable > tr > td,
table.wikitable > * > tr > th,table.wikitable > * > tr > td
{
    border:1px #aaa solid;
    padding:0.2em;
}
table.wikitable > tr > th,
table.wikitable > * > tr > th
{
    text-align:center;
    background:white;
    font-weight:bold
}
table.wikitable > caption
{
    font-weight:bold;
}
</style>
EOD;
    /**
     * Used to hold a WikiParser object that will be used for parsing
     * @var object
     */
    public $parser;
    /**
     * Base url the dump's own rules build their links against, such as the
     * target of an image or a redirect.
     * @var string
     */
    public $dump_base;
    /**
     * Url of the page now being read, used by the rule for the "other uses"
     * hatnote to point at that page's disambiguation page.
     * @var string
     */
    public $current_url;
    /**
     * Title of the page now being read, shown as the link text of the
     * "other uses" hatnote.
     * @var string
     */
    public $current_title;
    /**
     * Creates a media wiki archive iterator with the given parameters.
     *
     * @param string $iterate_timestamp timestamp of the arc archive bundle to
     *     iterate  over the pages of
     * @param string $iterate_dir folder of files to iterate over
     * @param string $result_timestamp timestamp of the arc archive bundle
     *     results are being stored in
     * @param string $result_dir where to write last position checkpoints to
     */
    public function __construct($iterate_timestamp, $iterate_dir,
            $result_timestamp, $result_dir)
    {
        $ini = [ 'compression' => 'bzip2',
            'file_extension' => 'bz2',
            'encoding' => 'UTF-8',
            'start_delimiter' => '@page@'];
        parent::__construct($iterate_timestamp, $iterate_dir,
            $result_timestamp, $result_dir, $ini);
        $this->switch_partition_callback_name = "readMediaWikiHeader";
    }
    /**
     * Estimates the important of the site according to the weighting of
     * the particular archive iterator
     * @param $site an associative array containing info about a web page
     * @return int a 4-bit number based on the log_2 size - 10 of the wiki
     *     entry (@see nextPage).
     */
    public function weight(&$site)
    {
        return min($site[self::WEIGHT] ?? 1, 15);
    }
    /**
     * Reads the siteinfo tag of the mediawiki xml file and extract data that
     * will be used in constructing page summaries.
     *
     * @return bool false when the siteinfo tag could not be located in
     *      the archive (also marks the iterator finished); void on the
     *      success path with $this->header populated
     */
    public function readMediaWikiHeader()
    {
        $this->header = [];
        $site_info = $this->getNextTagData("siteinfo");
        $found_lang =
            preg_match('/lang\=\"(.*)\"/', $this->remainder, $matches);
        if ($found_lang) {
            $this->header['lang'] = $matches[1];
        }
        if ($site_info === false) {
            $this->bz2_iterator = null;
            return false;
        }
        $dom = new \DOMDocument();
        @$dom->loadXML($site_info);
        $this->header['sitename'] = $this->getTextContent($dom,
            "/siteinfo/sitename");
        $pre_host_name =
            $this->getTextContent($dom, "/siteinfo/base");
        $this->header['base_address'] = substr($pre_host_name, 0,
            strrpos($pre_host_name, "/") + 1);
        set_error_handler(null);
        $url_parts = @parse_url($this->header['base_address']);
        restore_error_handler();
        $this->header['ip_address'] = gethostbyname($url_parts['host']);
        return true;
    }
    /**
     * Adds this iterator's own rules to the parser so the dump-only markup
     * an archived wiki uses is turned into html during the parse and
     * reaches the reader as html. The rules cover image and file links, the
     * redirect forms, the "other uses" hatnote, and the many templates a
     * dump carries that this reader does not render and so drops. Each rule
     * scans the text itself rather than leaning on a regular expression,
     * and nested braces or brackets are handled by the scan, so no
     * flattening pass is needed ahead of it.
     *
     * @param string $base_address base url the rules build their links on
     */
    public function initializeSubstitutions($base_address)
    {
        $this->dump_base = $base_address;
        $this->parser = new WikiParser($base_address);
        $this->parser->addRule([$this, "ruleImage"]);
        $this->parser->addRule([$this, "ruleRedirect"]);
        $this->parser->addRule([$this, "ruleRedirectLine"]);
        $this->parser->addRule([$this, "ruleOtherUses"]);
        $this->parser->addRule([$this, "ruleStripTemplate"]);
    }
    /**
     * Finds the offset just past the close that matches an opening bracket
     * pair at a given spot, counting deeper pairs so a nested construct does
     * not end the outer one early. It lets a rule take in a whole image
     * link, redirect, or template even when another sits inside it.
     *
     * @param string $text the text being read
     * @param int $index the offset of the opening pair
     * @param string $open the opening pair, such as "[[" or "{{"
     * @param string $close the closing pair, such as "]]" or "}}"
     * @return int|bool the offset just past the matching close, or false
     *      when there is none
     */
    protected function findMatchingClose($text, $index, $open, $close)
    {
        $depth = 0;
        $length = strlen($text);
        $open_length = strlen($open);
        $close_length = strlen($close);
        while ($index < $length) {
            if (substr_compare($text, $open, $index, $open_length) === 0) {
                $depth++;
                $index += $open_length;
                continue;
            }
            if (substr_compare($text, $close, $index, $close_length)
                === 0) {
                $depth--;
                $index += $close_length;
                if ($depth === 0) {
                    return $index;
                }
                continue;
            }
            $index++;
        }
        return false;
    }
    /**
     * Reports whether a bar-separated part of an image link sets layout or
     * size rather than being the caption, so the caption can be told apart
     * from directions such as thumb, right, or a pixel width.
     *
     * @param string $part one bar-separated part of an image link
     * @return bool true when the part sets layout rather than caption
     */
    protected function isImageLayoutPart($part)
    {
        $part = strtolower(trim($part));
        $words = ["thumb", "thumbnail", "frame", "frameless", "border",
            "right", "left", "center", "centre", "none", "upright", "live",
            "baseline", "sub", "super", "top", "text-top", "middle",
            "bottom", "text-bottom"];
        if (in_array($part, $words)) {
            return true;
        }
        $settings = ["alt=", "link=", "lang=", "upright=", "class=",
            "page="];
        foreach ($settings as $setting) {
            if (strncmp($part, $setting, strlen($setting)) === 0) {
                return true;
            }
        }
        $units = ["px", "em", "ex", "pt", "in", "cm"];
        foreach ($units as $unit) {
            $unit_at = strlen($part) - strlen($unit);
            if ($unit_at > 0 && substr($part, $unit_at) === $unit) {
                $head = str_replace("x", "", substr($part, 0, $unit_at));
                if ($head !== "" && ctype_digit($head)) {
                    return true;
                }
            }
        }
        return false;
    }
    /**
     * Rule that turns an image or file link into a plain link to the file
     * page followed by the caption, dropping the layout directions a wiki
     * uses to place and size a picture the reader is not shown.
     *
     * @param string $text the text being read
     * @param int $index the offset to try the rule at
     * @return array|null the html and the offset past the link, or null
     */
    public function ruleImage($text, $index)
    {
        $prefixes = ["[[Image:", "[[File:"];
        $found = false;
        foreach ($prefixes as $prefix) {
            if (substr_compare($text, $prefix, $index,
                strlen($prefix)) === 0) {
                $found = true;
                break;
            }
        }
        if (!$found) {
            return null;
        }
        $end = $this->findMatchingClose($text, $index, "[[", "]]");
        if ($end === false) {
            return null;
        }
        $inner = substr($text, $index + 2, $end - $index - 4);
        $colon = strpos($inner, ":");
        $after = substr($inner, $colon + 1);
        $parts = explode("|", $after);
        $name = trim(array_shift($parts));
        $caption = "";
        foreach ($parts as $part) {
            if (!$this->isImageLayoutPart($part)) {
                $caption = trim($part);
            }
        }
        $label = ($caption === "") ? $name : $caption;
        $html = "(<a href=\"" . $this->dump_base . "File:" .
            htmlspecialchars($name) . "\">Image:" .
            htmlspecialchars($label) . "</a>)";
        return ["html" => $html, "next" => $end];
    }
    /**
     * Rule that turns a redirect template into an indented hatnote. A
     * three-part redirect names a topic, a phrase, and a target page; a
     * shorter one names only a topic and points at its disambiguation page.
     *
     * @param string $text the text being read
     * @param int $index the offset to try the rule at
     * @return array|null the html and the offset past the template, or null
     */
    public function ruleRedirect($text, $index)
    {
        if (strncasecmp(substr($text, $index, 10), "{{Redirect", 10)
            !== 0) {
            return null;
        }
        $end = $this->findMatchingClose($text, $index, "{{", "}}");
        if ($end === false) {
            return null;
        }
        $inner = substr($text, $index + 2, $end - $index - 4);
        $parts = explode("|", $inner);
        $name = strtolower(trim($parts[0]));
        if ($name !== "redirect" && $name !== "redirect2") {
            return null;
        }
        array_shift($parts);
        if (count($parts) >= 3) {
            $topic = htmlspecialchars(trim($parts[0]));
            $phrase = htmlspecialchars(trim($parts[1]));
            $target = trim($parts[2]);
            $html = "<div class='indent'>\"" . $topic . "\". (" . $phrase .
                " &rarr;<a href=\"" . $this->dump_base .
                htmlspecialchars($target) . "\">" .
                htmlspecialchars($target) . "</a>)</div>";
        } else if (count($parts) >= 1) {
            $safe = htmlspecialchars(trim($parts[0]));
            $html = "<div class='indent'>\"" . $safe . "\". (<a href=\"" .
                $this->dump_base . $safe . "_(disambiguation)\">" . $safe .
                "???</a>)</div>";
        } else {
            return null;
        }
        return ["html" => $html, "next" => $end];
    }
    /**
     * Rule that turns a redirect line, the first line of a page that only
     * forwards to another, into a link to that other page.
     *
     * @param string $text the text being read
     * @param int $index the offset to try the rule at
     * @return array|null the html and the offset past the line, or null
     */
    public function ruleRedirectLine($text, $index)
    {
        if ($index > 0 && $text[$index - 1] !== "\n") {
            return null;
        }
        if (strncasecmp(substr($text, $index, 9), "#REDIRECT", 9) !== 0) {
            return null;
        }
        $open = strpos($text, "[[", $index);
        if ($open === false) {
            return null;
        }
        $end = $this->findMatchingClose($text, $open, "[[", "]]");
        if ($end === false) {
            return null;
        }
        $target = substr($text, $open + 2, $end - $open - 4);
        $bar = strpos($target, "|");
        if ($bar !== false) {
            $target = substr($target, 0, $bar);
        }
        $target = trim($target);
        $safe = htmlspecialchars($target);
        $href = htmlspecialchars(str_replace(" ", "_", $target));
        $html = "<a href='" . $this->dump_base . $href . "'>" . $safe .
            "</a>";
        return ["html" => $html, "next" => $end];
    }
    /**
     * Rule that turns the "other uses" hatnote into a link to the current
     * page's disambiguation page, since that hatnote sends a reader who
     * wanted a different topic of the same name to the list of topics.
     *
     * @param string $text the text being read
     * @param int $index the offset to try the rule at
     * @return array|null the html and the offset past the hatnote, or null
     */
    public function ruleOtherUses($text, $index)
    {
        $tag = "{{Other uses}}";
        if (strncasecmp(substr($text, $index, strlen($tag)), $tag,
            strlen($tag)) !== 0) {
            return null;
        }
        $html = "<div class='indent'>\"\". (<a href='" .
            $this->current_url . "_(disambiguation)'>" .
            htmlspecialchars($this->current_title) . "</a>)</div>";
        return ["html" => $html, "next" => $index + strlen($tag)];
    }
    /**
     * Rule that drops any remaining template. A dump carries many templates,
     * such as infoboxes and maintenance notes, that this reader does not
     * render; run after the rules that do handle a template, this one takes
     * in whatever is left, nested braces and all, and emits nothing.
     *
     * @param string $text the text being read
     * @param int $index the offset to try the rule at
     * @return array|null the empty html and the offset past the template,
     *      or null
     */
    public function ruleStripTemplate($text, $index)
    {
        if (substr_compare($text, "{{", $index, 2) !== 0) {
            return null;
        }
        $end = $this->findMatchingClose($text, $index, "{{", "}}");
        if ($end === false) {
            return null;
        }
        return ["html" => "", "next" => $end];
    }
    /**
     * Restores the internal state from the file iterate_status.txt in the
     * result dir such that the next call to nextPages will pick up from just
     * after the last checkpoint. We also reset up our regex substitutions
     *
     * @return array the data serialized when saveCheckpoint was called
     */
    public function restoreCheckPoint()
    {
        $info = parent::restoreCheckPoint();
        if (!$this->iterate_dir) { // do on client not name server
            $this->initializeSubstitutions($this->header['base_address']);
        }
        return $info;
    }
    /**
     * Gets the text content of the first dom node satisfying the
     * xpath expression $path in the dom document $dom
     *
     * @param object $dom DOMDocument to get the text from
     * @param $path xpath expression to find node with text
     *
     * @return string text content of the given node if it exists
     */
    public function getTextContent($dom, $path)
    {
        $xpath = new \DOMXPath($dom);
        $objects = $xpath->evaluate($path);
        if ($objects  && is_object($objects) && $objects->item(0) != null ) {
            return $objects->item(0)->textContent;
        }
        return "";
    }
    /**
     * Gets the next doc from the iterator
     * @param bool $no_process do not do any processing on page data
     * @return array associative array for doc or string if no_process true
     */
    public function nextPage($no_process = false)
    {
        static $minimal_regexes = false;
        static $first_call = true;
        if ($first_call) {
            if (!isset($this->header['base_address'])) {
                $this->header['base_address'] = "";
            }
            $this->initializeSubstitutions($this->header['base_address']);
        }
        $page_info = $this->getNextTagData("page");
        if ($no_process) {
            return $page_info;
        }
        if( empty($page_info)) {
            return [];
        }
        $dom = new \DOMDocument();
        set_error_handler(null);
        @$dom->loadXML($page_info);
        restore_error_handler();
        $site = [];
        $pre_url = $this->getTextContent($dom, "/page/title");
        $pre_url = str_replace(" ", "_", $pre_url);
        $site[self::URL] = $this->header['base_address'].$pre_url;
        $this->current_url = $site[self::URL];
        $this->current_title = $pre_url;
        $site[self::IP_ADDRESSES] = [$this->header['ip_address']];
        $pre_timestamp = $this->getTextContent($dom,
            "/page/revision/timestamp");
        $site[self::MODIFIED] = date("U", strtotime($pre_timestamp));
        $site[self::TIMESTAMP] = time();
        $site[self::TYPE] = "text/html";
        $site[self::HEADER] = "MediawikiBundleIterator extractor";
        $site[self::HTTP_CODE] = 200;
        $site[self::ENCODING] = "UTF-8";
        $site[self::SERVER] = "unknown";
        $site[self::SERVER_VERSION] = "unknown";
        $site[self::OPERATING_SYSTEM] = "unknown";
        $site[self::PAGE] = "<html lang='".$this->header['lang']."' >\n".
            "<head><title>$pre_url</title>\n".
            self::WIKI_PAGE_STYLES . "\n</head>\n".
            "<body><h1>$pre_url</h1>\n";
        $pre_page = $this->getTextContent($dom, "/page/revision/text");
        $current_hash = L\crawlHash($pre_page);
        if ($first_call) {
            $this->saveCheckPoint(); //ensure we remember to advance one on fail
            $first_call = false;
        }
        $pre_page = $this->parser->parse($pre_page, false, true);
        $site[self::PAGE] .= $pre_page;
        $site[self::PAGE] .= "\n</body>\n</html>";
        $site[self::HASH] = FetchUrl::computePageHash($site[self::PAGE],
            "text");
        $site[self::WEIGHT] = ceil(max(
            log(strlen($site[self::PAGE]) + 1, 2) - 10, 1));
        $site[self::DOC_DEPTH] = 1;
        return $site;
    }
}
