<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants as CC;
use seekquarry\yioop\library\processors\HtmlProcessor;
use seekquarry\yioop\library\UnitTest;
/**
* HtmlProcessorTest checks the title and description HtmlProcessor, with
* its default ScrapeSummarizer, takes from small pages, each showing one
* way a page may carry its text.
*/
class HtmlProcessorTest extends UnitTest
{
/**
* setUp makes the processor the cases share.
*/
public function setUp()
{
$this->test_objects['PROCESSOR'] = new HtmlProcessor([], 2000);
}
/**
* tearDown has nothing to remove.
*/
public function tearDown()
{
}
/**
* Runs a page through the processor.
*
* @param string $page the page's HTML
* @return array [title, description with its spaces collapsed]
*/
protected function titleAndDescription($page)
{
$summary = $this->test_objects['PROCESSOR']->process($page,
"https://example.com/page.html");
return [$summary[CC::TITLE], trim(preg_replace('/\s+/u', ' ',
$summary[CC::DESCRIPTION]))];
}
/**
* textIsSeparatedAndDecodedTestCase checks that a line break separates
* the words on each side of it, and that entities such as è are
* read as the letters they stand for.
*/
public function textIsSeparatedAndDecodedTestCase()
{
[, $description] = $this->titleAndDescription("<html><head>" .
"<title>Contact</title></head><body><p>Hours<br>Mon to Fri</p>" .
"<p>Crème brûlée</p></body></html>");
$this->assertTrue(str_contains($description, "Hours Mon to Fri"),
"a line break separates words");
$this->assertTrue(str_contains($description, "Crème brûlée"),
"entities are decoded");
}
/**
* titleIsWholeTestCase checks that a long title is cut by characters,
* leaving valid UTF-8, and that its line breaks become single spaces.
*/
public function titleIsWholeTestCase()
{
[$title] = $this->titleAndDescription("<html><head><title>" .
str_repeat("東京の天気予報", 20) . "</title></head>" .
"<body><p>x</p></body></html>");
$this->assertTrue(mb_check_encoding($title, "UTF-8"),
"a cut title is valid UTF-8");
$this->assertEqual(mb_strlen($title), HtmlProcessor::MAX_TITLE_LEN,
"it is cut at the limit in characters");
[$title] = $this->titleAndDescription("<html><head><title>\n " .
"Annual Report\n 2025\n</title></head><body></body></html>");
$this->assertEqual($title, "Annual Report 2025",
"a title's line breaks and runs of spaces become one space");
}
/**
* pageDescriptionsAreReadTestCase checks that an Open Graph
* description, a JSON-LD headline and description, image alt text,
* and text in main and figcaption all reach the description.
*/
public function pageDescriptionsAreReadTestCase()
{
[, $description] = $this->titleAndDescription("<html><head>" .
"<title>T</title><meta property='og:description' " .
"content='Guide to the loop trail'><script " .
"type='application/ld+json'>{\"headline\": \"Council approves " .
"bike lanes\"}</script></head><body><main>Nine miles long." .
"</main><img alt='Red ceramic teapot with handle'>" .
"<img alt='logo'><figure><figcaption>Photo of the summit" .
"</figcaption></figure></body></html>");
foreach (["Guide to the loop trail", "Council approves bike lanes",
"Nine miles long", "Red ceramic teapot with handle",
"Photo of the summit"] as $text) {
$this->assertTrue(str_contains($description, $text),
"the description has: $text");
}
$this->assertFalse(str_contains($description, "logo"),
"alt text of one word is left out");
}
/**
* fullStopIsNotDoubledTestCase checks that a sentence ending in 。 gets
* no period added after it.
*/
public function fullStopIsNotDoubledTestCase()
{
[, $description] = $this->titleAndDescription("<html lang='ja'>" .
"<head><title>天気</title></head><body><p>天気の情報です。</p>" .
"</body></html>");
$this->assertFalse(str_contains($description, "。."),
"no period follows 。");
}
/**
* pagesAreConvertedFromTheirEncodingTestCase checks convertToUtf8,
* which the fetcher uses on a page that is not UTF-8: it converts from
* the declared encoding, and reads an unknown or missing one as
* Windows-1252.
*/
public function pagesAreConvertedFromTheirEncodingTestCase()
{
foreach ([["ISO-8859-1", "ISO-8859-1", "Crème brûlée"],
["Shift_JIS", "SJIS", "東京の天気"],
["KOI8-R", "KOI8-R", "Привет мир"],
["", "Windows-1252", "Café"],
["no-such-encoding", "Windows-1252", "Naïve"]] as
[$declared, $written_as, $text]) {
$bytes = mb_convert_encoding($text, $written_as, "UTF-8");
$this->assertEqual(L\convertToUtf8($bytes, $declared), $text,
"text declared as '$declared' is converted");
}
}
/**
* declaredLanguageWinsATieTestCase checks a page that says lang="en"
* and holds only a few words, laid out as google.com's home page lays
* them out: links with no space between them. Its one stopword, "in",
* is a stopword of German, English, Italian and Dutch alike, so the text
* points to none of them; the page was stored as de, the first of the
* tied locales, and dropped from every query joined with lang:en.
*/
public function declaredLanguageWinsATieTestCase()
{
$summary = $this->test_objects['PROCESSOR']->process('<!doctype ' .
'html><html lang="en"><head><meta content="text/html; ' .
'charset=UTF-8" http-equiv="Content-Type"><title>Google</title>' .
'</head><body><a href="/mail">Gmail</a><a href="/imghp">Images' .
'</a><a href="/login">Sign in</a> <a href="/advanced">' .
'Advanced search</a><a href="/ads">Advertising</a><a ' .
'href="/services">Business Solutions</a><a href="/about">About ' .
'Google</a><p>© 2026 - <a href="/privacy">Privacy</a> - ' .
'<a href="/terms">Terms</a></p><div>Google apps</div></body>' .
'</html>', "https://www.google.com/");
$this->assertTrue(str_starts_with($summary[CC::LANG], "en"),
"the declared English is kept, not " . $summary[CC::LANG]);
}
/**
* jsonLdTextIsKeptWholeTestCase checks the JSON-LD text a page carries,
* which HtmlProcessor puts at the start of the body. A dollar amount
* in it, such as $250, must keep its digits, and the body must keep
* its attributes, since a scraper signature such as phpBB's tests the
* body's id.
*/
public function jsonLdTextIsKeptWholeTestCase()
{
$processor = new HtmlProcessor([], 2000);
$processor->scrapers = [["NAME" => "PHPBB",
"SIGNATURE" => "//body[@id='phpbb']", "TEXT_PATH" => "",
"DELETE_PATHS" => "", "EXTRACT_FIELDS" => "", "PRIORITY" => 0]];
$summary = $processor->process("<html><head><title>Clay shoot" .
"</title><script type='application/ld+json'>{\"description\": " .
"\"Tickets cost $250 each.\"}</script></head><body id='phpbb'>" .
"<p>Join us at the range.</p></body></html>",
"https://example.com/shoot.html");
$this->assertTrue(str_contains($summary[CC::DESCRIPTION],
"Tickets cost $250 each"), "the dollar amount keeps its digits");
$this->assertEqual($summary[CC::SCRAPER_LABEL] ?? "", "PHPBB",
"the body keeps the id the scraper signature tests");
}
/**
* largeInlineStyleKeepsTheTextTestCase checks that a page whose head
* holds a megabyte of inline style, as the Gatsby pages of
* corporate.charter.com do, still gives its text. A lazy pattern taking
* out style elements gave up on such a page and the processor found
* no text at all. A script element whose close tag has a space before
* its ">" is taken out too, and an element of another kind is left.
*/
public function largeInlineStyleKeepsTheTextTestCase()
{
$rule = ".a{color:red}";
$style = "<style data-href='/styles.css'>" . str_repeat($rule,
intdiv(self::LARGE_STYLE, strlen($rule))) . "</style>";
[, $description] = $this->titleAndDescription("<html><head>" .
"<title>T</title>$style</head><body><p>Spectrum expanded its " .
"fiber network in Polk County.</p></body></html>");
$this->assertTrue(str_contains($description,
"Spectrum expanded its fiber network in Polk County"),
"the text after a megabyte of style is read");
$this->assertEqual(L\removeTagSpans("a<SCRIPT x>b</script >c" .
"<style>d", "script", " ", true), "a c<style>d",
"a script with a spaced close tag goes and an element of " .
"another kind stays");
}
/**
* pageCutInsideScriptGivesNoCodeTestCase checks that a page whose
* end was cut off inside a script, as a fetcher keeps only the start
* of a long page, gives the page's text and none of the script's
* code. Pages of vx-3.com cut this way gave the summarizer a thousand
* sentences of a store's bundle list in JSON, and each took seconds.
*/
public function pageCutInsideScriptGivesNoCodeTestCase()
{
$bundles = "";
for ($i = 0; $i < 50; $i++) {
$bundles .= "\"$i\":{\"bundle_id\":50$i,\"handle\":" .
"\"rugby-bundle-$i\",\"type\":\"multi\"},";
}
[, $description] = $this->titleAndDescription("<html><head>" .
"<title>T</title></head><body><p>Redefine outdoor comfort " .
"with the sports robe.</p><script>var bundles={$bundles}");
$this->assertTrue(str_contains($description,
"Redefine outdoor comfort with the sports robe"),
"the page's text is read");
$this->assertTrue(!str_contains($description, "bundle"),
"the cut off script's code is not");
$this->assertEqual(L\removeTagSpans("a<script>b</script>c" .
"<script>d", "script"), "a c ",
"a script with no close tag goes with the rest of the page");
}
/**
* LARGE_STYLE is how many bytes of style the large inline style case
* puts in a page's head: about the megabyte the charter pages carry,
* past what pcre.backtrack_limit lets a lazy pattern read.
*/
const LARGE_STYLE = 1100000;
}
- Yioop Repo - Seekquarry