<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Charles Bocage (charles.bocage@sjsu.edu)
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
/**
* ScraperManager matches a web page to a scraper and applies it. A scraper
* is made for pages that one kind of software writes, such as WordPress or
* MediaWiki. Its signature tells such a page apart, and its rules pick out
* the page's main text and any extra fields. HtmlProcessor calls it for
* each page.
*
* @author Charles Bocage (charles.bocage@sjsu.edu) updated to support
* scraper priorities and extract fields Chris Pollett
*/
class ScraperManager
{
/**
* getScraper finds the scraper for a page: the one of highest priority
* whose signature holds on the page. A signature is an XPath, or else a
* regular expression matched against the page's text.
*
* @param string $page the html page to check
* @param array $scrapers an array of scrapers to check against
* @param \DOMXPath $xpath an XPath over the page with its script and
* style elements removed, as HtmlProcessor reads it, or null. An
* XPath signature is tested on it, unless the signature names a
* script or style element. Such a signature, or any signature
* when $xpath is null, is tested on the page as fetched.
* @return array an associative array of scraper properties if a matching
* scraper signature found; otherwise, the empty array
*/
public static function getScraper($page, $scrapers, $xpath = null)
{
$out_scraper = [];
$out_priority = -1;
/* The page as fetched is read at most once, and only for a
signature $xpath cannot answer. Reading it costs more than the
rest of what HtmlProcessor does with the page. */
$page_xpath = null;
foreach ($scrapers as $scraper) {
if (empty($scraper)) {
continue;
}
$signature = html_entity_decode(
$scraper['SIGNATURE'], ENT_QUOTES);
if ($xpath !== null && !preg_match('/script|style/i',
$signature)) {
$holds = self::checkSignature($page, $signature, $xpath);
} else {
$holds = self::checkSignature($page, $signature, $page_xpath);
}
if ($holds && $scraper['PRIORITY'] > $out_priority) {
$out_scraper = $scraper;
$out_priority = $scraper['PRIORITY'];
}
}
foreach ($out_scraper as $key => $value) {
$out_scraper[$key] = html_entity_decode($value, ENT_QUOTES);
}
return $out_scraper;
}
/**
* applyScraperRules uses a scraper's rules to cut a page down to its
* main text. TEXT_PATH is an XPath to the part that holds the text.
* DELETE_PATHS lists XPaths, one to a line, of parts to remove from
* it. EXTRACT_FIELDS lists summary fields to fill, one name=XPath or
* name:=value to a line.
*
* @param string $page the web page to operate on
* @param array $scraper the scraper whose rules are applied
* @param \DOMDocument $page_dom the document $page reads as, if the
* caller has it already, so $page is not read again; otherwise null
* @return array the fields extracted, and the page cut down to its
* text, or the whole page where TEXT_PATH finds nothing
*/
public static function applyScraperRules($page, $scraper, $page_dom = null)
{
$delete_paths = explode("\n", $scraper["DELETE_PATHS"]);
$dom = self::getContentByXquery($page, $scraper["TEXT_PATH"],
$page_dom);
$summary = [];
$scraper_apply_info = [];
if (!empty($dom)) {
$scraper_apply_info["GET_CONTENT_SUCCEEDED"] = true;
foreach ($delete_paths as $tag_to_remove) {
$did_remove_stuff =
self::removeContentByXquery($dom, $tag_to_remove);
if ($did_remove_stuff) {
$scraper_apply_info["TRIGGERED_REMOVALS"][] =
$tag_to_remove;
}
if (empty($dom)) {
break;
}
}
if (!empty($dom)) {
$out_page = utf8SafeSaveHtml($dom);
}
} else {
$scraper_apply_info["GET_CONTENT_SUCCEEDED"] = false;
}
$summary[L\CrawlConstants::SCRAPER_INFO] = $scraper_apply_info;
set_error_handler(null);
$extract_fields = explode("\n", $scraper["EXTRACT_FIELDS"]);
$results = false;
$dom = new \DOMDocument();
/* Most scrapers extract no fields, and reading the page for them
would cost as much as reading it for its text. */
if (trim($scraper["EXTRACT_FIELDS"]) !== "" && !empty($page) &&
@$dom->loadHTML($page)) {
foreach ($extract_fields as $extract_field) {
if (preg_match('/^([^\=\:]+)(\:?\=)(.+)$/', $extract_field,
$parts)) {
list(, $summary_field, $assign_type, $value_or_query) =
$parts;
$summary_field = trim($summary_field);
$value_or_query = trim($value_or_query);
if (defined(C\NS_LIB . "CrawlConstants::" .$summary_field)){
$summary_field = constant(
C\NS_LIB . "CrawlConstants::" .$summary_field);
}
$out_text = "";
if ($assign_type == ":=") {
$out_text = $value_or_query;
} else {
if (preg_match("@^(r(:?egex)?)/@", $value_or_query,
$field_matches)) {
$field_regex = substr($value_or_query,
strlen($field_matches[1]));
$out_text = preg_match($field_regex, $page);
} if ($xpath = new \DOMXpath($dom)) {
$results = @$xpath->query($value_or_query);
if (!empty($results) && $results->length > 0) {
$len = $results->length;
for ($i = 0; $i < $len; $i++) {
$out_text .=
$results->item($i)->textContent;
}
}
}
}
$summary[$summary_field] = trim($out_text);
}
}
}
restore_error_handler();
$out_page = empty($out_page) ? $page : $out_page;
return [$summary, $out_page];
}
/**
* checkSignature tells if a scraper's signature holds on a page.
* A signature that starts with a slash is an XPath, which holds when
* it finds at least one node. Any other signature is a regular
* expression, written without delimiters, which holds when it matches
* the page's text.
*
* @param string $page a web document to check
* @param string $signature an xpath to check against
* @param mixed &$xpath an XPath over $page from an earlier check, or
* null, in which case $page is read and the XPath is left here
* for the next check; false if $page could not be read
* @return boolean true if the given xpath return a non empty dom node list
*/
public static function checkSignature($page, $signature, &$xpath = null)
{
if ($signature[0] == '/') {
$results = false;
set_error_handler(null);
if ($xpath === null) {
$dom = new \DOMDocument();
$xpath = (!empty($page) && @$dom->loadHTML($page)) ?
new \DOMXpath($dom) : false;
}
if ($xpath !== false) {
$results = $xpath->query($signature);
}
restore_error_handler();
return !empty($results->length) && $results->length > 0;
} else {
return (mb_ereg($signature, $page) !== false);
}
}
/**
* getContentByXquery makes a small page of the first node an XPath
* finds in a page, placed in the body, with the page's head kept.
*
* @param string $page a document to apply the xpath query against
* @param string $query the xpath query to run
* @param \DOMDocument $dom the document $page reads as, if the caller
* has it already; otherwise null, and $page is read here
*
* @return \DOMDocument dom of a simplified web page containing nodes
* matching xpath query within an html body tag.
*/
public static function getContentByXquery($page, $query, $dom = null)
{
$out_dom = null;
$dom ??= L\getDomFromString($page);
if (!empty($dom->documentElement) && !empty($query)) {
// if getContentByXquery succeeds we also copy over head of doc
$head_xpath = new \DOMXPath($dom);
$head_xpath_result = $head_xpath->query("//head");
$xpath = new \DOMXPath($dom);
$xpath_result = $xpath->query($query);
if (!empty($xpath_result) && $xpath_result->length > 0) {
$out_dom = new \DOMDocument();
$out_dom->loadHTML("<html><head></head><body></body></html>");
if (!empty($head_xpath_result) &&
$head_xpath_result->length > 0) {
$head_node = $out_dom->importNode(
$head_xpath_result->item(0), true);
$out_dom->documentElement->replaceChild($head_node,
$out_dom->documentElement->childNodes->item(0));
}
$node = $out_dom->importNode($xpath_result->item(0), true);
$out_dom->documentElement->childNodes->item(1)->appendChild(
$node);
}
}
return $out_dom;
}
/**
* removeContentByXquery removes from a document all the nodes an
* XPath finds, as applyScraperRules does for each of a scraper's
* DELETE_PATHS.
*
* @param \DOMDocument $dom a document to apply the xpath query against
* @param string $query the xpath query to run
* @return bool whether anything was removed from the DOMDocument
*/
public static function removeContentByXquery($dom, $query)
{
if (empty($dom) || empty($query)) {
return false;
}
$did_remove_stuff = false;
$xpath = new \DOMXPath($dom);
$xpath_result = $xpath->query($query);
if ($xpath_result->length > 0) {
$len = $xpath_result->length;
for ($i = 0; $i < $len; $i++) {
$node = $xpath_result->item($i);
$parent = $node->parentNode;
if ($parent) {
$parent->removeChild($node);
$did_remove_stuff = true;
}
}
}
return $did_remove_stuff;
}
}
- Yioop Repo - Seekquarry