<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\data_structures\BloomFilterFile;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\processors\RobotProcessor;
use seekquarry\yioop\library\UnitTest;
/**
* Used to test that the UrlParser class. For now, want to see that the
* method canonicalLink is working correctly and that
* isPathMemberRegexPaths (used in robot_processor.php) works
*
* @author Chris Pollett
*/
class UrlParserTest extends UnitTest
{
/**
* UrlParser uses static methods so doesn't do anything right now
*/
public function setUp()
{
}
/**
* UrlParser uses static methods so doesn't do anything right now
*/
public function tearDown()
{
}
/**
* Check if can go from a relative link, base link to a complete link
* in various different ways
*/
public function canonicalLinkTestCase()
{
$test_links = [
[".", "http://www.example.com/",
"http://www.example.com/", "root dir0"],
["/bob.html", "http://www.example.com/",
"http://www.example.com/bob.html", "root dir1"],
["bob.html", "http://www.example.com/",
"http://www.example.com/bob.html", "root dir2"],
["bob", "http://www.example.com/",
"http://www.example.com/bob", "root dir3"],
["bob", "http://www.example.com",
"http://www.example.com/bob", "root dir4"],
["http://print.bob.com/bob", "http://www.example.com",
"http://print.bob.com/bob", "root dir5"],
["/.", "http://www.example.com/",
"http://www.example.com/", "root dir6"],
["//slashdot.org", "http://www.slashdot.org",
"http://slashdot.org/", "slashdot dir"],
["bob", "http://www.example.com/a",
"http://www.example.com/a/bob", "sub dir1"],
["../bob", "http://www.example.com/a",
"http://www.example.com/bob", "sub dir2"],
["../../bob", "http://www.example.com/a",
null, "sub dir3"],
["./bob", "http://www.example.com/a",
"http://www.example.com/a/bob", "sub dir4"],
["bob.html?a=1", "http://www.example.com/a",
"http://www.example.com/a/bob.html?a=1", "query 1"],
["bob?a=1&b=2", "http://www.example.com/a",
"http://www.example.com/a/bob?a=1&b=2", "query 2"],
["/?a=1&b=2", "http://www.example.com/a",
"http://www.example.com/?a=1&b=2", "query 3"],
["?a=1&b=2", "http://www.example.com/a",
"http://www.example.com/a/?a=1&b=2", "query 4"],
["b/b.html?a=1&b=2", "http://www.example.com/a/c",
"http://www.example.com/a/c/b/b.html?a=1&b=2", "query 5"],
["b/b.html?a=1&b=2?c=4", "http://www.example.com/a/c",
"http://www.example.com/a/c/b/b.html?a=1&b=2?c=4", "query 6"],
["b#1", "http://www.example.com/",
"http://www.example.com/b#1", "fragment 1"],
["b?a=1#1", "http://www.example.com/",
"http://www.example.com/b?a=1#1", "fragment 2"],
["b?a=1#1#2", "http://www.example.com/",
"http://www.example.com/b?a=1#1#2", "fragment 3"],
["#a", "http://www.example.com/c:d",
"http://www.example.com/c:d#a", "fragment 4"],
];
foreach ($test_links as $test_link) {
$result = UrlParser::canonicalLink($test_link[0],
$test_link[1], false);
$this->assertEqual($result, $test_link[2], $test_link[3]);
}
}
/**
* Check is a path matches with a list of paths presumably coming from
* a robots.txt file
*/
public function isPathMemberRegexPathsTestCase()
{
$path = [];
$robot_paths = [];
$results = [];
$tests = [
["/bobby", ["/bob"], true, "Substring Positive"],
["/bobby", ["/alice", "/f/g/h/d"], false,
"Substring Negative 1"],
["/bobby/", ["/bobby/bay", "/f/g/h/d", "/yo"], false,
"Substring Negative 2"],
["/bay/bobby/", ["/bobby/", "/f/g/h/d", "/yo"], false,
"Substring Negative 3 (should match start)"],
["http://test.com/bay/bobby/",
["/bobby/", "/f/g/h/d", "/yo"], false,
"Substring Negative 4 (should match start)"],
["/a/bbbb/c/", ["/bobby/bay", "/a/*/c/", "/yo"], true,
"Star Positive 1"],
["/a/bbbb/d/", ["/bobby/bay", "/a/*/c/", "/yo"], false,
"Star Negative 1"],
["/test.html?a=b", ["/bobby/bay", "/*?", "/yo"], true,
"Star Positive 2"],
["/test.html", ["/bobby/bay", "/*.html$", "/yo"], true,
"Dollar Positive 1"],
["/test.htmlish", ["/bobby/bay", "/*.html$", "/yo"], false,
"Dollar Negative 1"],
["/test.htmlish", ["/bobby/bay", "*", "/yo"], true,
"Degenerate 1"],
["/test.html", ["/bobby/bay", "/**.html$", "/yo"], true,
"Degenerate 2"],
["/videos/search?q=Angelina+Jolie",
["/videos/search?"], true, "End With Question Regex Case 1"],
];
foreach ($tests as $test) {
list($path, $robot_paths, $result, $description) = $test;
$this->assertEqual(UrlParser::isPathMemberRegexPaths($path,
$robot_paths), $result, $description);
}
}
/**
* Tests simplifyUrl function used on SERP pages
*/
public function simplifyUrlTestCase()
{
$test_urls = [
["http://www.example.com/", 100,
"www.example.com", "HTTP Domain only"],
["https://www.example.com/", 100,
"www.example.com", "HTTPS Domain only"],
["http://www.superreallylongexample.com/", 25,
"www.superreallylonge...e.com", "Domain truncate"],
["http://www.example.com/word1/word2/word3/word4", 25,
"www.example.com/word...word4", "Path truncate"],
];
foreach ($test_urls as $test_url) {
$result = UrlParser::simplifyUrl($test_url[0], $test_url[1]);
$this->assertEqual($result, $test_url[2], $test_url[3]);
}
}
/**
* urlMemberSiteArray is a function called by both allowedToCrawlSite
* disallowedToCrawlSite to test if a url belongs to alist of
* regex's of urls or domain. This test function tests this functionality
*/
public function urlMemberSiteArrayTestCase()
{
$sites = ["http://www.example.com/",
"http://www.cs.sjsu.edu/faculty/pollett/*/*/",
"http://www.bing.com/video/search?*&*&",
"http://*.cool.*/a/*/", "domain:ucla.edu",
"domain:foodnetwork.com",
"domain:.ottawa.ca",
"domain:.ottawa2.ca",
"http://ottawa2.ca/"];
$test_urls = [
["http://www.cs.sjsu.edu/faculty/pollett/", false,
"regex url negative 1"],
["http://www.bing.com/video/search?", false,"regex url negative 2"],
["http://www.cool.edu/a", false, "regex url negative 3"],
["http://ucla.edu.com", false, "domain test negative"],
["http://www.cs.sjsu.edu/faculty/pollett/a/b/c", true,
"regex url positive 1"],
["http://www.bing.com/video/search?a&b&c", true,
"regex url positive 2"],
["http://www.cool.bob.edu/a/b/c", true, "regex url positive 3"],
["http://test.ucla.edu", true, "domain test positive"],
["https://test.ucla.edu", true, "domain https test positive"],
["gopher://test.ucla.edu", true, "domain gopher stest positive"],
["http://www.foodnetworkstore.com/small-appliances/", false,
"domain test negative"],
["http://a.ottawa.ca/", true,
"domain starting dot test positive 2"],
["http://ottawa.ca/", false, "domain starting dot test negative"],
["http://a.ottawa2.ca/", true,
"domain starting dot test positive 2"],
["http://ottawa2.ca/", true, "domain starting dot test positive 3"],
];
foreach ($test_urls as $test_url) {
$result = UrlParser::urlMemberSiteArray($test_url[0], $sites,
"s");
$this->assertEqual($result, $test_url[1], $test_url[2]);
}
}
/**
* Checks if getScheme is working okay
*/
public function getSchemeTestCase()
{
$test_links = [
["http://www.example.com/", "http", "Simple HTTP 1"],
["https://www.example.com/", "https", "Simple HTTPS 1"],
["gopher://www.example.com/", "gopher", "Simple GOPHER 1"],
["./", "http", "Simple HTTP 2"],
];
foreach ($test_links as $test_link) {
$result = UrlParser::getScheme($test_link[0]);
$this->assertEqual($result, $test_link[1], $test_link[2]);
}
}
/**
* Checks if getHost is working okay
*/
public function getHostTestCase()
{
$test_links = [
["https://somewhere.com:80/la/de/da", "https://somewhere.com:80",
"Host with por"],
["https://10.1.10.10/?dfas=aga/", "https://10.1.10.10",
"IP based host with query"],
["https://www.yioop.com/###@?woohoo", "https://www.yioop.com",
"Host with fragment"],
];
foreach ($test_links as $test_link) {
$result = UrlParser::getHost($test_link[0]);
$this->assertEqual($result, $test_link[1], $test_link[2]);
}
}
/**
* hyphenatedHostKeepsWholeWordTestCase checks that the host words of
* https://www.coca-cola.com/ keep coca-cola whole, so that a search
* for coca-cola finds it in the host, as a search for yahoo finds
* yahoo in www.yahoo.com. A url past the home page gives no words.
*/
public function hyphenatedHostKeepsWholeWordTestCase()
{
$this->assertEqual(UrlParser::getWordsInHostUrl(
"https://www.coca-cola.com/"), " coca-cola ",
"a hyphenated host part is one word");
$this->assertEqual(UrlParser::getWordsInHostUrl(
"https://news.yahoo.co.jp/"), " news yahoo co ",
"the parts between dots are words");
$this->assertEqual(UrlParser::getWordsInHostUrl(
"https://www.coca-cola.com/brands"), "",
"a page past the home page gives no host words");
}
/**
* Checks if getCompanyLevelDomain is working okay
*/
public function getCompanyLevelDomainTestCase()
{
$test_links = [
["https://www.somewhere.com/", "somewhere.com",
"US URL Test 1"],
["https://aaa.bbb.cc.edu/", "cc.edu",
"US URL Test 2"],
["https://somewhere.else.jp/", "else.jp", "Japanese URL Test 1"],
["https://somewhere.co.jp/", "somewhere.co.jp",
"Japanese URL Test 2"],
["https://somewhere.com.jp/", "somewhere.com.jp",
"Japanese URL Test 3"],
["https://somewhere.co.uk/", "somewhere.co.uk",
"UK URL Test 1"],
];
foreach ($test_links as $test_link) {
$result = UrlParser::getCompanyLevelDomain($test_link[0]);
$this->assertEqual($result, $test_link[1], $test_link[2]);
}
}
/**
* cullByDomainFilterTestCase checks that cullByDomainFilter keeps a
* URL whose domain is in a filter and culls one whose domain is not.
* The case makes its own BloomFilterFile holding yahoo.com in
* C\WORK_DIRECTORY's temp folder, so it needs no domain filter file
* to be installed.
*/
public function cullByDomainFilterTestCase()
{
$filter = new BloomFilterFile(C\WORK_DIRECTORY .
"/temp/cull_test.ftr", 100);
$filter->add("yahoo.com");
$filters = [$filter];
$test_links = [
["https://www.yahoo.com/foo", false,
"Top Ten Million Site Test 1"],
["https://aaa.bbb.ccc.yahoo.com/foo/goo", false,
"Top Ten Million Site Test 2"],
["https://pollett.org/foo/goo", false,
"Not Top Ten Million Site Test 1"],
["https://www.pollett.org/foo/goo", false,
"Not Top Ten Million Site Test 1"],
["https://aaa.bbb.ccc.pollett.org/foo/goo", true,
"Not Top Ten Million Site Test 3"],
];
foreach ($test_links as $test_link) {
$result = UrlParser::cullByDomainFilter($test_link[0], $filters);
$this->assertEqual($result, $test_link[1], $test_link[2]);
}
}
/**
* isSameMachineTestCase checks that UrlParser::isSameMachine says two
* urls name the same computer when their hosts are the same, or when
* one host is a name that resolves to the other's address. localhost
* resolves to 127.0.0.1 from this machine's own hosts file, so the case
* needs no network.
*/
public function isSameMachineTestCase()
{
$this->assertTrue(UrlParser::isSameMachine("https://173.13.143.74/",
"https://173.13.143.74/yioop/"), "the same address");
$this->assertTrue(UrlParser::isSameMachine("http://localhost/",
"https://127.0.0.1/"), "a name and the address it resolves to");
$this->assertFalse(UrlParser::isSameMachine("https://173.13.143.74/",
"https://173.13.143.75/"), "two different addresses");
$this->assertFalse(UrlParser::isSameMachine("", "https://127.0.0.1/"),
"an empty url names no machine");
}
/**
* trackingParametersAreDroppedTestCase checks that canonicalLink drops
* the parameters that only say where a visitor came from, so two links
* to one page become one url, and keeps the others in their order.
*/
public function trackingParametersAreDroppedTestCase()
{
$site = "https://www.example.com/";
$this->assertEqual(UrlParser::canonicalLink(
"https://news.example.com/a?id=7&utm_source=x&utm_medium=y",
$site), "https://news.example.com/a?id=7",
"utm parameters are dropped and others kept");
$this->assertEqual(UrlParser::canonicalLink(
"/b?fbclid=abc&page=2&gclid=z", $site),
"https://www.example.com/b?page=2",
"click ids are dropped from a relative link");
$this->assertEqual(UrlParser::canonicalLink(
"https://www.example.com/c?utm_campaign=q", $site),
"https://www.example.com/c", "a query of only tracking " .
"parameters is dropped whole");
}
/**
* starInRobotsRuleMatchesNoCharactersTestCase checks that a * in a
* robots.txt rule can stand for no characters at all, as in RFC 9309,
* the Robots Exclusion Protocol. The rules come from a robots.txt
* whose rule of a * followed by /search$ has to keep a crawler off
* /search, the site's own search page.
*/
public function starInRobotsRuleMatchesNoCharactersTestCase()
{
$processor = new RobotProcessor();
$summary = $processor->process("User-agent: *\n" .
"Disallow: */search$\nDisallow: /*/email$\n",
"https://www.example.com/robots.txt");
$disallowed = $summary[CrawlConstants::ROBOT_PATHS][
CrawlConstants::DISALLOWED_SITES];
$tests = [
["/search", $disallowed, true, "Star at start matches nothing"],
["/anime/search", $disallowed, true, "Star at start matches"],
["/searches", $disallowed, false, "Dollar still ends the rule"],
["/email", $disallowed, false, "Rule needs two slashes"],
["/ab", ["/a*b"], true, "Star between pieces matches nothing"],
["/ab", ["/a*b$"], true, "Empty star before a final dollar"],
];
foreach ($tests as [$path, $robot_paths, $result, $description]) {
$this->assertEqual(UrlParser::isPathMemberRegexPaths($path,
$robot_paths), $result, $description);
}
}
/**
* redirectFromHomePageAlsoNamesHomePageTestCase checks redirectTargets.
* The home page of www.coke.com redirects to
* https://www.coca-cola.com/?&redirect=true, so links go to that page
* and to https://www.coca-cola.com/. A redirect within one site, or
* from a page that is not a home page, leaves one link.
*/
public function redirectFromHomePageAlsoNamesHomePageTestCase()
{
$this->assertEqual(UrlParser::redirectTargets(
"https://www.coke.com/",
"https://www.coca-cola.com/?&redirect=true"),
["https://www.coca-cola.com/?&redirect=true",
"https://www.coca-cola.com/"], "a home page redirect names " .
"the other site's home page too");
$this->assertEqual(UrlParser::redirectTargets(
"https://www.coke.com/", "https://www.coca-cola.com/"),
["https://www.coca-cola.com/"], "a redirect to a home page " .
"names it once");
$this->assertEqual(UrlParser::redirectTargets(
"https://www.coke.com/drinks", "https://www.coca-cola.com/a"),
["https://www.coca-cola.com/a"], "a redirect from an inner " .
"page names only its target");
$this->assertEqual(UrlParser::redirectTargets(
"https://www.ibm.com/", "https://www.ibm.com/us-en"),
["https://www.ibm.com/us-en"], "a redirect within one site " .
"names only its target");
}
}
- Yioop Repo - Seekquarry