<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\processors;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\UrlParser;
/**
* SitemapProcessor makes the crawl summary of a sitemap or a sitemap index.
* The summary's title is the sitemap's url, its description the words of
* that url, and its links the urls the sitemap's loc elements list.
* XmlProcessor hands it documents whose root element is urlset or
* sitemapindex.
*
* @author Chris Pollett
*/
class SitemapProcessor extends TextProcessor
{
/**
* process makes the summary of the sitemap $page fetched from $url. It
* parses the part of $page entriesPage gives for max_links_to_extract
* entries and takes their links with links. Where some of those
* entries give no link and more entries follow, it parses every whole
* entry instead. A document that gives neither links nor a title is
* summarized as text by TextProcessor. Either summary is marked
* JUST_METAS.
*
* @param string $page the text of the sitemap
* @param string $url the url the sitemap came from, which relative
* links are resolved against
* @return array the summary of the sitemap
*/
public function process($page, $url)
{
$summary = null;
if (is_string($page)) {
$max_links = self::$max_links_to_extract;
[$entries_page, $more] = self::entriesPage($page, $max_links);
$dom = L\getDomFromString($entries_page);
if (!$dom) {
self::closeDanglingTags($entries_page);
$dom = L\getDomFromString($entries_page);
}
if ($dom !== false) {
$summary[self::TITLE] = $url;
$summary[self::DESCRIPTION] =
UrlParser::extractTextFromUrl($url);
$summary[self::LANG] = "";
$summary[self::LINKS] = self::links($dom, $url);
if ($more && count($summary[self::LINKS]) < $max_links) {
/* some entries gave no link, so later entries are
read to make up the number */
[$entries_page] = self::entriesPage($page, 0);
$summary[self::LINKS] = self::links(
L\getDomFromString($entries_page), $url);
}
if (strlen($summary[self::DESCRIPTION] . $summary[self::TITLE])
== 0 && count($summary[self::LINKS]) == 0) {
/* maybe not a sitemap, so it is read as text, which
may still give urls */
$summary = parent::process($page, $url);
}
$summary[self::JUST_METAS] = true;
} else {
$summary = parent::process($page, $url);
$summary[self::JUST_METAS] = true;
}
}
return $summary;
}
/**
* entriesPage gives the part of the sitemap $page that holds its first
* $max_entries entries, url elements in a urlset or sitemap elements in
* a sitemap index. The text is cut after the end tag of the last of
* them, and the root element's end tag is put after it, so the part
* parses as a whole sitemap. A sitemap of ten megabytes is then parsed
* only as far as the entries that can give links. A sitemap cut off
* where the fetcher stopped uncompressing it is cut back the same way,
* after its last whole entry, or after the loc element of the entry the
* cut runs through if that loc is whole, with the entry closed. So no
* url is taken from a loc the cut runs through. A $max_entries of 0 or
* less keeps every whole entry.
* A page that is not such a sitemap, or one holding no more entries
* than asked and ending in its root's end tag, is given back as it is.
* process calls it before parsing a sitemap.
*
* @param string $page the text of a sitemap
* @param int $max_entries how many entries to keep, 0 or less for all
* @return array [the text to parse, whether $page has whole entries
* after it]
*/
public static function entriesPage($page, $max_entries)
{
$entry_names = ["urlset" => "url", "sitemapindex" => "sitemap"];
$root_name = XmlProcessor::rootName($page);
$entry_name = $entry_names[$root_name] ?? "";
if ($entry_name == "") {
return [$page, false];
}
$entry_end = "</$entry_name>";
$end = 0;
$count = 0;
while (($max_entries <= 0 || $count < $max_entries) &&
($position = strpos($page, $entry_end, $end)) !== false) {
$end = $position + strlen($entry_end);
$count++;
}
$more = $end > 0 && strpos($page, $entry_end, $end) !== false;
if ($end == 0 ||
(!$more && strpos($page, "</$root_name>", $end) !== false)) {
return [$page, false];
}
$close_tags = "</$root_name>";
$loc_end = ($more) ? false : strpos($page, "</loc>", $end);
if ($loc_end !== false) {
$end = $loc_end + strlen("</loc>");
$close_tags = $entry_end . $close_tags;
}
return [substr($page, 0, $end) . $close_tags, $more];
}
/**
* links gives the urls listed by the loc elements of a parsed sitemap
* or sitemap index, each made canonical against $site. It stops after
* max_links_to_extract loc elements have given a url, counting a url
* each time it is listed, so a url listed twice uses two of the
* places. Priority and change frequency entries are not read. process
* calls it on the part of the sitemap entriesPage gives.
*
* @param \DOMDocument $dom the parsed sitemap
* @param string $site the url of the sitemap
* @return array links of the sitemap, url => "From sitemap" text
* naming the sitemap and the words of the url
*/
public static function links($dom, $site)
{
$sites = [];
$xpath = new \DOMXPath($dom);
$xpath->registerNamespace('s',
"http://www.sitemaps.org/schemas/sitemap/0.9");
$paths = [
"/s:urlset/s:url/s:loc",
"/s:sitemapindex/s:sitemap/s:loc"
];
$i = 0;
foreach ($paths as $path) {
$nodes = @$xpath->evaluate($path);
foreach ($nodes as $node) {
$node_content = (empty($node->textContent)) ?
$node->nodeValue : $node->textContent;
$url = UrlParser::canonicalLink(
$node_content, $site);
if ($url === null || $url === "" ||
UrlParser::checkRecursiveUrl($url) ||
strlen($url) >= C\MAX_URL_LEN) {
continue;
}
$sites[$url] = "From sitemap : " . $site . " .." .
UrlParser::extractTextFromUrl($url);
$i++;
if (self::$max_links_to_extract > 0 &&
$i >= self::$max_links_to_extract) {
break 2;
}
}
}
return $sites;
}
}
- Yioop Repo - Seekquarry