<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.orgs
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\models;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
/**
* Used to manage data related to the SCRAPER database table.
* This table is used to store web scrapers, a tool for scraping
* important content from pages which might have been generated
* by a content management system.
*
* @author Charles Bocage (changed CMS_DETECTORS to SCRAPER and
* simplified Chris Pollett)
*/
class ScraperModel extends Model
{
/**
* DEFAULT_SCRAPERS lists the scrapers a new Yioop database starts with,
* and that upgradeDatabaseVersion130 adds to an older one. Each row
* holds a scraper's name; its signature, an XPath that holds on pages
* made by the software the scraper is for; the XPath of the part of the
* page that holds its text; the XPaths of parts to delete from that
* text, one to a line; and fields to extract, one name=XPath to a line.
* A page is matched to the scraper whose signature holds on it, and that
* scraper's paths then pick out the page's text.
*/
const DEFAULT_SCRAPERS = [
["DRUPAL", "/html/head/*[contains(@href, '/sites/all/themes') or " .
"contains(@href, '/sites/default/files') or " .
"contains(@content, 'Drupal')]|" .
"//*[contains(@class, 'drupalorg-site-main')]",
"//*[@id='main']|//main",
"//*[contains(@class, 'd-animations-settings')]\n" .
"//*[contains(@class,'element-invisible')]\n" .
"//*[contains(@class,'visually-hidden')]\n" .
"//*[@id='skip-link']\n" .
"//*[contains(@class,'skip-link')]\n" .
"//*[contains(@class,'last-reply')]\n" .
"//*[contains(@class,'more-link')]\n" .
"//*[contains(@class,'breadbox')]\n" .
"//*[contains(@class,'topics')]\n" .
"//*[contains(@class,'posts')]\n" .
"//thead\n",
""],
["MEDIAWIKI", "//meta[contains(@content, 'MediaWiki')]",
"//main|//*[@id='content']",
"//*[contains(@class, 'cdx-button')]\n" .
"//*[contains(@class, 'vector-dropdown')]\n" .
"//*[contains(@class, 'vector-page-toolbar')]\n" .
"//*[contains(@class, 'vector-body-before-content')]\n" .
"//*[contains(@class, 'nmbox')]\n" .
"//*[contains(@class, 'hatnote')]\n" .
"//*[contains(@class, 'infobox')]\n" .
"//*[contains(@class, 'haudio')]\n" .
"//*[contains(@class, 'IPA')]\n" .
"//*[contains(@class, 'rt-commentedText')]\n" .
"//a[contains(., 'Full') and contains(., 'article')]\n",
""],
["VBULLETIN", "/html/head/*[contains(@href,'vbulletin')]",
"//main",
"//*[contains(@class, 'widget-tabs-nav')]\n" .
"//*[contains(@class, 'conversation-tool-wrapper')]\n" .
"//*[contains(@class, 'b-sharing-menu')]\n" .
"//*[contains(@class, 'pagenav-controls')]\n" .
"//*[contains(@class, 'pagenav-container')]\n" .
"//*[contains(@class, 'widget-header')]\n" .
"//*[contains(@class, 'b-post__footer')]\n" .
"//*[contains(@class, 'toolbar-filter-overlay')]\n" .
"//*[contains(@class,'forum-list-header')]\n" .
"//*[contains(@class,'toolbar-filter')]\n" .
"//*[contains(@class,'js-conversation-showmore')]\n" .
"//*[contains(@class,'filtered-by')]\n" .
"//*[contains(@class,'empty')]\n" .
"//*[contains(@class,'new-conversations-strip')]\n" .
"//*[contains(@class,'h-hide')]\n" .
"//*[contains(@class,'post-date')]\n" .
"//*[contains(@class,'cell-count')]\n" .
"//*[contains(@class,'topics-count')]\n" .
"//*[contains(@class,'posts-count')]\n" .
"//*[contains(@class,'list-footer')]",
""],
["VIDEO SITE",
"//meta[@property='og:type' and contains(@content, 'video')]",
"", "",
"IS_VIDEO=//meta[@property='og:type' and" .
" contains(@content, 'video')]/@content\n" .
"IS_VR=//meta[(@property='og:video:tag' ".
" (contains(@content, '360') or" .
" contains(@content, '180') or contains(@content, " .
"'VR'))]/@content\n" .
"SITE_NAME=//meta[@property='og:site_name']/@content\n" .
"HEIGHT=//meta[@property='og:video:height']/@content\n" .
"WIDTH=//meta[@property='og:video:width']/@content\n" .
"DURATION=//meta[@property='video:duration']/@content\n" .
"ALT_DURATION=//meta[@itemprop='duration']/@content\n".
"PUBDATE=//meta[@property='video:release_date']/@content\n" .
"ALT_PUBDATE=//meta[@itemprop='datePublished']/@content\n" .
"VIEW_COUNT=//meta[@itemprop='interactionCount']/@content\n" .
"THUMB_URL=//meta[@property='og:image']/@content"],
["WORDPRESS", "/html/head/*[contains(@href, 'wp-content') or ".
"contains(@href, 'wp-includes')]|" .
"//meta[@name='generator' and contains(@content, 'WordPress')]",
"//main|//*[contains(@class, 'wp-block-post-content')]|" .
"//div[starts-with(@id, 'post-') and " .
"'post-' = translate(@id, '0123456789', '') and " .
"string-length(@id) >4]|//div[contains(@class, " .
"'homepagewrapper')]|" .
"//div[contains(@class, 'homepage-row')]/parent::*|" .
"//div[contains(@class, 'archive-grid')]/parent::*",
"//*[@id='entry-comments']\n" .
"//*[contains(@class, 'sharedaddy')]\n" .
"//*[contains(@class, 'blog-subscribe')]\n" .
"//*[@id='entry-side']\n" .
"//*[contains(@class, 'signup')]\n" .
"//*[contains(@class, 'module-donate')]",
""],
["YIOOP", "//meta[@name='generator' and contains(@content,'Yioop')]",
"//div[contains(@class, 'body-container')]",
"//*[@id='message']\n" .
"//*[@id='help']\n" .
"//*[contains(@class, 'query-statistics')]\n" .
"//*[@id='suggest']\n" .
"//*[@id='spell']",
""],
["GHOST", "//meta[@name='generator' and contains(@content, 'Ghost')]",
"//article|//main",
"//*[contains(@class, 'gh-subscribe')]\n" .
"//*[contains(@class, 'gh-navigation')]\n" .
"//*[contains(@class, 'gh-footer')]\n" .
"//*[contains(@class, 'site-footer')]",
""],
["DISCOURSE",
"//meta[@name='generator' and contains(@content, 'Discourse')]",
"//*[@id='main-outlet']",
"//*[contains(@class, 'crawler-nav')]\n" .
"//*[contains(@class, 'crawler-linkback-list')]\n" .
"//*[@id='noscript-footer']",
""],
["BLOGGER",
"//meta[@name='generator' and contains(@content, 'Blogger')]",
"//*[contains(@class, 'post-body')]|//main",
"//*[contains(@class, 'post-share-buttons')]\n" .
"//*[contains(@class, 'blog-pager')]\n" .
"//*[@id='comments']",
""],
["JOOMLA",
"//meta[@name='generator' and contains(@content, 'Joomla')]",
"//*[@itemprop='articleBody']|//main",
"//*[contains(@class, 'pagenavigation')]\n" .
"//*[contains(@class, 'mod-breadcrumbs')]",
""],
["PHPBB", "//body[@id='phpbb']", "//*[@id='page-body']",
"//*[contains(@class, 'action-bar')]\n" .
"//*[contains(@class, 'jumpbox')]\n" .
"//*[contains(@class, 'post-buttons')]",
""],
["STACK EXCHANGE", "//meta[@property='og:site_name' and " .
"(contains(@content, 'Stack Overflow') or " .
"contains(@content, 'Stack Exchange'))]",
"//*[@id='mainbar']",
"//*[contains(@class, 'js-post-menu')]\n" .
"//*[contains(@class, 'js-voting-container')]\n" .
"//*[contains(@class, 'bottom-notice')]\n" .
"//*[@id='sidebar']",
""],
];
/**
* Controls which tables and the names of tables
* underlie the given model and should be used in a getRows call
*
* @param string $args it does not matter.
* @return string which table to use
*/
public function fromCallback($args = null)
{
return "SCRAPER";
}
/**
* Return the contents of the SCRAPER table
* @return array associative of rows with ID, NAME, SIGNATURE,
* PRIORITY, TEXT_PATH, DELETE_PATHS, EXTRACT_FIELDS, one for each scraper
*/
public function getAllScrapers()
{
$db = $this->db;
$sources = [];
$sql = "SELECT * FROM SCRAPER";
$result = $db->execute($sql);
if (!$result) {
return false;
}
while ($sources[] = $db->fetchArray($result)) {
}
return $sources;
}
/**
* Used to add a new scraper to Yioop
*
* @param string $name of scraper to add
* @param string $signature the xpath to query the html of a web
* document to see if a scrape rule should be applied
* @param int $priority to choose this scrape rule as opposed to other
* scrape rules
* @param string $text_path the xpath string used to find
* the main dom container for the important text in the html document
* @param string $delete_paths xpath strings of dom elements to be removed
* from the dom after the dom was restricted to just the $text_path
* content. These are used to remove extranenous info from the main
* text contents. Each xpath should be separated from each other by
* a new line.
* @param string $extract_fields a string of lines each line consists
* of a summary field name followed by = followed by
* an xpath. The intended meaning of such a line is to evaluate
* the xpath and create a new field in a document summary with
* either the concatenated, trimmed text value of the nodes of the
* results of the xpath
*/
public function add($name, $signature, $priority, $text_path,
$delete_paths, $extract_fields)
{
$db = $this->db;
$sql = "INSERT INTO SCRAPER(NAME, SIGNATURE, PRIORITY, TEXT_PATH,
DELETE_PATHS, EXTRACT_FIELDS) VALUES (?, ?, ?, ?, ?, ?)";
$db->execute($sql, [$name, $signature, $priority, $text_path,
$delete_paths, $extract_fields]);
}
/**
* Deletes the scraper with the provided id
*
* @param int $id of scraper to be deleted
*/
public function delete($id)
{
$sql = "DELETE FROM SCRAPER WHERE ID=?";
$this->db->execute($sql, [$id]);
}
/**
* Returns the scraper with the given id
* @param int $id of scraper to look up
* @return array associative array with ID, NAME, SIGNATURE,
* PRIORITY, TEXT_PATH, DELETE_PATHS, EXTRACT_FIELDS of a scraper
*/
public function get($id)
{
$db = $this->db;
$sql = "SELECT * FROM SCRAPER WHERE ID = ?";
$result = $db->execute($sql, [$id]);
if (!$result) {
return false;
}
$row = $db->fetchArray($result);
return $row;
}
/**
* Used to update the fields stored in a SCRAPER row according to
* an array holding new values
* @param array $scraper_info updated values for scraper
*/
public function update($scraper_info)
{
$id = $scraper_info['ID'];
unset($scraper_info['ID']);
$sql = "UPDATE SCRAPER SET ";
$comma = "";
$params = [];
foreach ($scraper_info as $field => $value) {
$sql .= "$comma $field=? ";
$comma = ",";
$params[] = $value;
}
$sql .= " WHERE ID=?";
$params[] = $id;
$this->db->execute($sql, $params);
}
}
- Yioop Repo - Seekquarry