<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\library\SyntheticWeb;
use seekquarry\yioop\library\UnitTest;
/**
* Checks that the made-up web is built the same way every run and that each
* graph type links its pages in the shape it should.
*
* @author Chris Pollett
*/
class SyntheticWebTest extends UnitTest
{
/**
* The made-up web has no state to set up.
*/
public function setUp()
{
}
/**
* The made-up web has no state to tear down.
*/
public function tearDown()
{
}
/**
* A url addressed by path and one addressed by domain both read back to
* the page number they were built from.
*/
public function pageTupleReadsBackTheUrlTestCase()
{
$by_path = SyntheticWeb::urlForTuple([1, 2, 3], false);
$this->assertEqual([1, 2, 3], SyntheticWeb::pageTuple($by_path),
"a page url addressed by path reads back to its tuple");
$by_domain = SyntheticWeb::urlForTuple([1, 2, 3], true);
$this->assertEqual([1, 2, 3], SyntheticWeb::pageTuple($by_domain),
"a page url addressed by domain reads back to its tuple");
$this->assertEqual([],
SyntheticWeb::pageTuple("http://0-0.web/robots.txt"),
"a robots url on a made-up host is not a page");
}
/**
* A grid page links to the page one further along each of the two ways
* through the grid.
*/
public function gridLinksToTwoNeighboursTestCase()
{
$links = SyntheticWeb::outLinks([2, 3], "grid", 2, 1);
$this->assertEqual([[3, 3], [2, 4]], $links,
"a grid page steps one along in each component");
}
/**
* A tree page links to its children, which are the pages just past its
* number times the branching.
*/
public function treeLinksToItsChildrenTestCase()
{
$links = SyntheticWeb::outLinks([0, 1], "tree", 2, 1);
$this->assertEqual([[0, 1, 0], [0, 1, 1]], $links,
"a binary tree page links to its two children by path");
}
/**
* The same page and seed give the same links every time, so a run
* repeats exactly.
*/
public function sameSeedGivesSameLinksTestCase()
{
$once = SyntheticWeb::outLinks([500], "power_law", 6, 99, 2.0);
$twice = SyntheticWeb::outLinks([500], "power_law", 6, 99, 2.0);
$this->assertEqual($once, $twice,
"the same page and seed give the same power law links");
$other_seed = SyntheticWeb::outLinks([500], "power_law", 6, 100, 2.0);
$this->assertTrue($once !== $other_seed,
"a different seed gives different power law links");
}
/**
* A steep power law exponent makes far more links land on low numbered
* pages than an even draw would, since those are the hubs.
*/
public function powerLawFavoursLowPagesTestCase()
{
$cut = 1 << 25;
$count = 400;
$steep_low = 0;
$even_low = 0;
for ($page = 0; $page < $count; $page++) {
$steep = SyntheticWeb::outLinks([$page], "power_law", 1, 7, 3.0);
$even = SyntheticWeb::outLinks([$page], "power_law", 1, 7, 1.0);
if ($steep[0][0] < $cut) {
$steep_low++;
}
if ($even[0][0] < $cut) {
$even_low++;
}
}
$this->assertTrue($steep_low > $even_low * 2,
"a steep exponent lands far more links on low pages than an " .
"even draw does");
}
/**
* The words on a page are the same every run for the same page and
* seed, so a search can be checked against an exact page.
*/
public function sameSeedGivesSameWordsTestCase()
{
$once = SyntheticWeb::words("12", 5, 1.0, 100, 25);
$twice = SyntheticWeb::words("12", 5, 1.0, 100, 25);
$this->assertEqual($once, $twice,
"the same page and seed give the same words");
$this->assertTrue(count($once) > 0,
"a page has at least one word");
}
/**
* Every made-up word is an eight digit numbered string, so a search for
* one is an exact check.
*/
public function wordsAreNumberedStringsTestCase()
{
$words = SyntheticWeb::words("3", 1, 1.0, 50, 10);
$all_numbered = true;
foreach ($words as $word) {
if (!preg_match('/^[0-9]{8}$/', $word)) {
$all_numbered = false;
}
}
$this->assertTrue($all_numbered,
"every made-up word is an eight digit numbered string");
}
/**
* A page's html carries an anchor for each of the graph's out-links, so
* Yioop's own processing draws the link meta words from it.
*/
public function pageHtmlCarriesTheLinksTestCase()
{
$settings = ["graph_type" => "grid", "out_degree" => 5,
"seed" => 1, "power_law_exponent" => 2.0, "term_exponent" => 1.0,
"doc_length" => 20, "length_spread" => 5,
"domain_link_probability" => 0.0,
"image_link_probability" => 0.0];
$html = SyntheticWeb::pageHtml([1, 0], $settings);
$this->assertTrue(strpos($html, "http://synth.web/2/0/index.html") !==
false, "a grid page's html links one step along by path");
$this->assertTrue(strpos($html, "http://synth.web/1/1/index.html") !==
false, "a grid page's html links one step across by path");
}
/**
* The seed page, page zero, never links to an image whatever the image
* probability is, so a crawl always has a text page to start from.
*/
public function seedPageIsNeverAnImageTestCase()
{
$settings = ["graph_type" => "grid", "out_degree" => 5,
"seed" => 1, "power_law_exponent" => 2.0, "term_exponent" => 1.0,
"doc_length" => 20, "length_spread" => 5,
"domain_link_probability" => 0.0,
"image_link_probability" => 1.0];
$html = SyntheticWeb::pageHtml([0, 0], $settings);
$this->assertTrue(strpos($html, ".jpg") === false,
"the seed page links to no image even at full image chance");
}
/**
* A made-up page's address is a full IPv6 address, and the same url
* always gives the same address.
*/
public function ipAddressIsStableIpVersionSixTestCase()
{
$url = SyntheticWeb::urlForTuple([1, 0], false);
$once = SyntheticWeb::ipAddress($url);
$twice = SyntheticWeb::ipAddress($url);
$this->assertEqual($once, $twice,
"the same url gives the same address");
$this->assertTrue(
filter_var($once, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) !==
false, "the address is a valid IPv6 address");
}
/**
* treeRootIsTheBareHostTestCase checks that a tree web is crawled
* from http://synth.web/index.html, whose tuple is empty, and that
* its children are the single-number pages under it.
*/
public function treeRootIsTheBareHostTestCase()
{
$seed = SyntheticWeb::seedUrl("tree", 3, 0);
$this->assertEqual("http://synth.web/index.html", $seed,
"the tree's root is the bare host");
$this->assertEqual([], SyntheticWeb::pageTuple($seed, "tree"),
"and its tuple is empty");
$this->assertEqual([], SyntheticWeb::pageTuple(
"http://synth.web/indexXhtml", "tree"),
"a stray character in the file name is not a page");
}
/**
* settingsFromSeedInfoReadsTheCrawlSettingsTestCase checks that a
* crawl's seed info is read into the array pageHtml takes, with each
* setting under the key pageHtml uses and a default where the crawl
* did not say, and that a crawl with no graph type gives nothing.
*/
public function settingsFromSeedInfoReadsTheCrawlSettingsTestCase()
{
$settings = SyntheticWeb::settingsFromSeedInfo(['general' =>
['graph_type' => 'tree', 'out_degree' => '4',
'domain_link_probability' => '0.4', 'synthetic_seed' => '7']]);
$this->assertEqual('tree', $settings['graph_type'],
"the graph type is read");
$this->assertEqual(4, $settings['out_degree'],
"the out degree is read as a number");
$this->assertEqual(7, $settings['seed'], "the seed is read");
$this->assertEqual(9, count($settings),
"all nine settings are present, defaults filling the rest");
$this->assertEqual([], SyntheticWeb::settingsFromSeedInfo(
['general' => ['crawl_type' => 'web']]),
"a crawl with no graph type gives nothing");
}
/**
* linkScriptsGoToTheRightPlaceTestCase checks that a generated page's
* links go straight to the synthetic activity, while a stored cached
* page's links go through the cache first, and that both name the
* crawl they belong to.
*/
public function linkScriptsGoToTheRightPlaceTestCase()
{
$generated = SyntheticWeb::linkOpenScript("1788985922");
$this->assertTrue(str_starts_with($generated, "<script>"),
"the generated page's script is a script element");
$this->assertTrue(strpos($generated,
"?c=search&a=synthetic&its=1788985922&arg=") !== false,
"a generated page's link goes to the synthetic activity");
$this->assertTrue(strpos($generated, "a=cache") === false,
"and not through the cache");
$cached = SyntheticWeb::linkScriptBody("1788985922");
$this->assertTrue(strpos($cached,
"?c=search&a=cache&its=1788985922&arg=") !== false,
"a cached page's link is tried through the cache first");
}
}
- Yioop Repo - Seekquarry