<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\archive_bundle_iterators;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\BZip2BlockIterator;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\WikiParser;
/** For crawlHash*/
require_once __DIR__."/../Utility.php";
/**
* Used to iterate through a collection of .xml.bz2 media wiki files
* stored in a WebArchiveBundle folder. Here these media wiki files contain the
* kinds of documents used by wikipedia. Iteration would be
* for the purpose making an index of these records
*
* @author Chris Pollett
* @see WebArchiveBundle
*/
class MediaWikiArchiveBundleIterator extends TextArchiveBundleIterator
{
/**
* Used to define the styles we put on cache wiki pages
*/
const WIKI_PAGE_STYLES = <<<EOD
<style>
table.wikitable
{
background:white;
border:1px #aaa solid;
border-collapse: scollapse
margin:1em 0;
}
table.wikitable > tr > th,table.wikitable > tr > td,
table.wikitable > * > tr > th,table.wikitable > * > tr > td
{
border:1px #aaa solid;
padding:0.2em;
}
table.wikitable > tr > th,
table.wikitable > * > tr > th
{
text-align:center;
background:white;
font-weight:bold
}
table.wikitable > caption
{
font-weight:bold;
}
</style>
EOD;
/**
* Used to hold a WikiParser object that will be used for parsing
* @var object
*/
public $parser;
/**
* Base url the dump's own rules build their links against, such as the
* target of an image or a redirect.
* @var string
*/
public $dump_base;
/**
* Url of the page now being read, used by the rule for the "other uses"
* hatnote to point at that page's disambiguation page.
* @var string
*/
public $current_url;
/**
* Title of the page now being read, shown as the link text of the
* "other uses" hatnote.
* @var string
*/
public $current_title;
/**
* Creates a media wiki archive iterator with the given parameters.
*
* @param string $iterate_timestamp timestamp of the arc archive bundle to
* iterate over the pages of
* @param string $iterate_dir folder of files to iterate over
* @param string $result_timestamp timestamp of the arc archive bundle
* results are being stored in
* @param string $result_dir where to write last position checkpoints to
*/
public function __construct($iterate_timestamp, $iterate_dir,
$result_timestamp, $result_dir)
{
$ini = [ 'compression' => 'bzip2',
'file_extension' => 'bz2',
'encoding' => 'UTF-8',
'start_delimiter' => '@page@'];
parent::__construct($iterate_timestamp, $iterate_dir,
$result_timestamp, $result_dir, $ini);
$this->switch_partition_callback_name = "readMediaWikiHeader";
}
/**
* Estimates the important of the site according to the weighting of
* the particular archive iterator
* @param $site an associative array containing info about a web page
* @return int a 4-bit number based on the log_2 size - 10 of the wiki
* entry (@see nextPage).
*/
public function weight(&$site)
{
return min($site[self::WEIGHT] ?? 1, 15);
}
/**
* Reads the siteinfo tag of the mediawiki xml file and extract data that
* will be used in constructing page summaries.
*
* @return bool false when the siteinfo tag could not be located in
* the archive (also marks the iterator finished); void on the
* success path with $this->header populated
*/
public function readMediaWikiHeader()
{
$this->header = [];
$site_info = $this->getNextTagData("siteinfo");
$found_lang =
preg_match('/lang\=\"(.*)\"/', $this->remainder, $matches);
if ($found_lang) {
$this->header['lang'] = $matches[1];
}
if ($site_info === false) {
$this->bz2_iterator = null;
return false;
}
$dom = new \DOMDocument();
@$dom->loadXML($site_info);
$this->header['sitename'] = $this->getTextContent($dom,
"/siteinfo/sitename");
$pre_host_name =
$this->getTextContent($dom, "/siteinfo/base");
$this->header['base_address'] = substr($pre_host_name, 0,
strrpos($pre_host_name, "/") + 1);
set_error_handler(null);
$url_parts = @parse_url($this->header['base_address']);
restore_error_handler();
$this->header['ip_address'] = gethostbyname($url_parts['host']);
return true;
}
/**
* Adds this iterator's own rules to the parser so the dump-only markup
* an archived wiki uses is turned into html during the parse and
* reaches the reader as html. The rules cover image and file links, the
* redirect forms, the "other uses" hatnote, and the many templates a
* dump carries that this reader does not render and so drops. Each rule
* scans the text itself rather than leaning on a regular expression,
* and nested braces or brackets are handled by the scan, so no
* flattening pass is needed ahead of it.
*
* @param string $base_address base url the rules build their links on
*/
public function initializeSubstitutions($base_address)
{
$this->dump_base = $base_address;
$this->parser = new WikiParser($base_address);
$this->parser->addRule([$this, "ruleImage"]);
$this->parser->addRule([$this, "ruleRedirect"]);
$this->parser->addRule([$this, "ruleRedirectLine"]);
$this->parser->addRule([$this, "ruleOtherUses"]);
$this->parser->addRule([$this, "ruleStripTemplate"]);
}
/**
* Finds the offset just past the close that matches an opening bracket
* pair at a given spot, counting deeper pairs so a nested construct does
* not end the outer one early. It lets a rule take in a whole image
* link, redirect, or template even when another sits inside it.
*
* @param string $text the text being read
* @param int $index the offset of the opening pair
* @param string $open the opening pair, such as "[[" or "{{"
* @param string $close the closing pair, such as "]]" or "}}"
* @return int|bool the offset just past the matching close, or false
* when there is none
*/
protected function findMatchingClose($text, $index, $open, $close)
{
$depth = 0;
$length = strlen($text);
$open_length = strlen($open);
$close_length = strlen($close);
while ($index < $length) {
if (substr_compare($text, $open, $index, $open_length) === 0) {
$depth++;
$index += $open_length;
continue;
}
if (substr_compare($text, $close, $index, $close_length)
=== 0) {
$depth--;
$index += $close_length;
if ($depth === 0) {
return $index;
}
continue;
}
$index++;
}
return false;
}
/**
* Reports whether a bar-separated part of an image link sets layout or
* size rather than being the caption, so the caption can be told apart
* from directions such as thumb, right, or a pixel width.
*
* @param string $part one bar-separated part of an image link
* @return bool true when the part sets layout rather than caption
*/
protected function isImageLayoutPart($part)
{
$part = strtolower(trim($part));
$words = ["thumb", "thumbnail", "frame", "frameless", "border",
"right", "left", "center", "centre", "none", "upright", "live",
"baseline", "sub", "super", "top", "text-top", "middle",
"bottom", "text-bottom"];
if (in_array($part, $words)) {
return true;
}
$settings = ["alt=", "link=", "lang=", "upright=", "class=",
"page="];
foreach ($settings as $setting) {
if (strncmp($part, $setting, strlen($setting)) === 0) {
return true;
}
}
$units = ["px", "em", "ex", "pt", "in", "cm"];
foreach ($units as $unit) {
$unit_at = strlen($part) - strlen($unit);
if ($unit_at > 0 && substr($part, $unit_at) === $unit) {
$head = str_replace("x", "", substr($part, 0, $unit_at));
if ($head !== "" && ctype_digit($head)) {
return true;
}
}
}
return false;
}
/**
* Rule that turns an image or file link into a plain link to the file
* page followed by the caption, dropping the layout directions a wiki
* uses to place and size a picture the reader is not shown.
*
* @param string $text the text being read
* @param int $index the offset to try the rule at
* @return array|null the html and the offset past the link, or null
*/
public function ruleImage($text, $index)
{
$prefixes = ["[[Image:", "[[File:"];
$found = false;
foreach ($prefixes as $prefix) {
if (substr_compare($text, $prefix, $index,
strlen($prefix)) === 0) {
$found = true;
break;
}
}
if (!$found) {
return null;
}
$end = $this->findMatchingClose($text, $index, "[[", "]]");
if ($end === false) {
return null;
}
$inner = substr($text, $index + 2, $end - $index - 4);
$colon = strpos($inner, ":");
$after = substr($inner, $colon + 1);
$parts = explode("|", $after);
$name = trim(array_shift($parts));
$caption = "";
foreach ($parts as $part) {
if (!$this->isImageLayoutPart($part)) {
$caption = trim($part);
}
}
$label = ($caption === "") ? $name : $caption;
$html = "(<a href=\"" . $this->dump_base . "File:" .
htmlspecialchars($name) . "\">Image:" .
htmlspecialchars($label) . "</a>)";
return ["html" => $html, "next" => $end];
}
/**
* Rule that turns a redirect template into an indented hatnote. A
* three-part redirect names a topic, a phrase, and a target page; a
* shorter one names only a topic and points at its disambiguation page.
*
* @param string $text the text being read
* @param int $index the offset to try the rule at
* @return array|null the html and the offset past the template, or null
*/
public function ruleRedirect($text, $index)
{
if (strncasecmp(substr($text, $index, 10), "{{Redirect", 10)
!== 0) {
return null;
}
$end = $this->findMatchingClose($text, $index, "{{", "}}");
if ($end === false) {
return null;
}
$inner = substr($text, $index + 2, $end - $index - 4);
$parts = explode("|", $inner);
$name = strtolower(trim($parts[0]));
if ($name !== "redirect" && $name !== "redirect2") {
return null;
}
array_shift($parts);
if (count($parts) >= 3) {
$topic = htmlspecialchars(trim($parts[0]));
$phrase = htmlspecialchars(trim($parts[1]));
$target = trim($parts[2]);
$html = "<div class='indent'>\"" . $topic . "\". (" . $phrase .
" →<a href=\"" . $this->dump_base .
htmlspecialchars($target) . "\">" .
htmlspecialchars($target) . "</a>)</div>";
} else if (count($parts) >= 1) {
$safe = htmlspecialchars(trim($parts[0]));
$html = "<div class='indent'>\"" . $safe . "\". (<a href=\"" .
$this->dump_base . $safe . "_(disambiguation)\">" . $safe .
"???</a>)</div>";
} else {
return null;
}
return ["html" => $html, "next" => $end];
}
/**
* Rule that turns a redirect line, the first line of a page that only
* forwards to another, into a link to that other page.
*
* @param string $text the text being read
* @param int $index the offset to try the rule at
* @return array|null the html and the offset past the line, or null
*/
public function ruleRedirectLine($text, $index)
{
if ($index > 0 && $text[$index - 1] !== "\n") {
return null;
}
if (strncasecmp(substr($text, $index, 9), "#REDIRECT", 9) !== 0) {
return null;
}
$open = strpos($text, "[[", $index);
if ($open === false) {
return null;
}
$end = $this->findMatchingClose($text, $open, "[[", "]]");
if ($end === false) {
return null;
}
$target = substr($text, $open + 2, $end - $open - 4);
$bar = strpos($target, "|");
if ($bar !== false) {
$target = substr($target, 0, $bar);
}
$target = trim($target);
$safe = htmlspecialchars($target);
$href = htmlspecialchars(str_replace(" ", "_", $target));
$html = "<a href='" . $this->dump_base . $href . "'>" . $safe .
"</a>";
return ["html" => $html, "next" => $end];
}
/**
* Rule that turns the "other uses" hatnote into a link to the current
* page's disambiguation page, since that hatnote sends a reader who
* wanted a different topic of the same name to the list of topics.
*
* @param string $text the text being read
* @param int $index the offset to try the rule at
* @return array|null the html and the offset past the hatnote, or null
*/
public function ruleOtherUses($text, $index)
{
$tag = "{{Other uses}}";
if (strncasecmp(substr($text, $index, strlen($tag)), $tag,
strlen($tag)) !== 0) {
return null;
}
$html = "<div class='indent'>\"\". (<a href='" .
$this->current_url . "_(disambiguation)'>" .
htmlspecialchars($this->current_title) . "</a>)</div>";
return ["html" => $html, "next" => $index + strlen($tag)];
}
/**
* Rule that drops any remaining template. A dump carries many templates,
* such as infoboxes and maintenance notes, that this reader does not
* render; run after the rules that do handle a template, this one takes
* in whatever is left, nested braces and all, and emits nothing.
*
* @param string $text the text being read
* @param int $index the offset to try the rule at
* @return array|null the empty html and the offset past the template,
* or null
*/
public function ruleStripTemplate($text, $index)
{
if (substr_compare($text, "{{", $index, 2) !== 0) {
return null;
}
$end = $this->findMatchingClose($text, $index, "{{", "}}");
if ($end === false) {
return null;
}
return ["html" => "", "next" => $end];
}
/**
* Restores the internal state from the file iterate_status.txt in the
* result dir such that the next call to nextPages will pick up from just
* after the last checkpoint. We also reset up our regex substitutions
*
* @return array the data serialized when saveCheckpoint was called
*/
public function restoreCheckPoint()
{
$info = parent::restoreCheckPoint();
if (!$this->iterate_dir) { // do on client not name server
$this->initializeSubstitutions($this->header['base_address']);
}
return $info;
}
/**
* Gets the text content of the first dom node satisfying the
* xpath expression $path in the dom document $dom
*
* @param object $dom DOMDocument to get the text from
* @param $path xpath expression to find node with text
*
* @return string text content of the given node if it exists
*/
public function getTextContent($dom, $path)
{
$xpath = new \DOMXPath($dom);
$objects = $xpath->evaluate($path);
if ($objects && is_object($objects) && $objects->item(0) != null ) {
return $objects->item(0)->textContent;
}
return "";
}
/**
* Gets the next doc from the iterator
* @param bool $no_process do not do any processing on page data
* @return array associative array for doc or string if no_process true
*/
public function nextPage($no_process = false)
{
static $minimal_regexes = false;
static $first_call = true;
if ($first_call) {
if (!isset($this->header['base_address'])) {
$this->header['base_address'] = "";
}
$this->initializeSubstitutions($this->header['base_address']);
}
$page_info = $this->getNextTagData("page");
if ($no_process) {
return $page_info;
}
if( empty($page_info)) {
return [];
}
$dom = new \DOMDocument();
set_error_handler(null);
@$dom->loadXML($page_info);
restore_error_handler();
$site = [];
$pre_url = $this->getTextContent($dom, "/page/title");
$pre_url = str_replace(" ", "_", $pre_url);
$site[self::URL] = $this->header['base_address'].$pre_url;
$this->current_url = $site[self::URL];
$this->current_title = $pre_url;
$site[self::IP_ADDRESSES] = [$this->header['ip_address']];
$pre_timestamp = $this->getTextContent($dom,
"/page/revision/timestamp");
$site[self::MODIFIED] = date("U", strtotime($pre_timestamp));
$site[self::TIMESTAMP] = time();
$site[self::TYPE] = "text/html";
$site[self::HEADER] = "MediawikiBundleIterator extractor";
$site[self::HTTP_CODE] = 200;
$site[self::ENCODING] = "UTF-8";
$site[self::SERVER] = "unknown";
$site[self::SERVER_VERSION] = "unknown";
$site[self::OPERATING_SYSTEM] = "unknown";
$site[self::PAGE] = "<html lang='".$this->header['lang']."' >\n".
"<head><title>$pre_url</title>\n".
self::WIKI_PAGE_STYLES . "\n</head>\n".
"<body><h1>$pre_url</h1>\n";
$pre_page = $this->getTextContent($dom, "/page/revision/text");
$current_hash = L\crawlHash($pre_page);
if ($first_call) {
$this->saveCheckPoint(); //ensure we remember to advance one on fail
$first_call = false;
}
$pre_page = $this->parser->parse($pre_page, false, true);
$site[self::PAGE] .= $pre_page;
$site[self::PAGE] .= "\n</body>\n</html>";
$site[self::HASH] = FetchUrl::computePageHash($site[self::PAGE],
"text");
$site[self::WEIGHT] = ceil(max(
log(strlen($site[self::PAGE]) + 1, 2) - 10, 1));
$site[self::DOC_DEPTH] = 1;
return $site;
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software