<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\UnitTest;
/**
* FetchUrlTest checks the FetchUrl class, which downloads pages for the
* crawler. Its cases cover page hashes, cookies, signed requests and
* bot challenges.
*
* @author Chris Pollett
*/
class FetchUrlTest extends UnitTest
{
/**
* setUp does nothing, since each case makes what it needs.
*/
public function setUp()
{
}
/**
* tearDown does nothing, since each case leaves nothing behind.
*/
public function tearDown()
{
}
/**
* computePageHashTestCase checks that a script tag does not change a
* page's hash, while text in other scripts, such as Devanagari, does.
*/
public function computePageHashTestCase()
{
$with_script = "<script>i++</script>hi";
$without_script = "hi";
$this->assertEqual(FetchUrl::computePageHash($with_script, "text"),
FetchUrl::computePageHash($without_script, "text"),
'Script tags don\'t affect page hash');
$with_unicode = "yoमहामा";
$without_unicode = "yo";
$this->assertTrue(FetchUrl::computePageHash($with_unicode) !=
FetchUrl::computePageHash($without_unicode),
'Unicode text counts in hash');
}
/**
* emptyPagesAreNotDuplicatesTestCase checks that two text pages whose
* descriptions hold no letters or digits, as www.pepsi.com's and
* www.cyon.ch's did, get different hashes when their urls are given,
* so the index does not group them as duplicates. Two pages with the
* same words still get the same hash whatever their urls.
*/
public function emptyPagesAreNotDuplicatesTestCase()
{
$this->assertTrue(FetchUrl::computePageHash("", "text",
"https://www.pepsi.com/") != FetchUrl::computePageHash(". ",
"text", "https://www.cyon.ch/"),
"pages with no words hash apart by url");
$this->assertEqual(FetchUrl::computePageHash("Same words", "text",
"https://one.example.com/"), FetchUrl::computePageHash(
"Same words", "text", "https://two.example.com/"),
"pages with the same words hash the same");
}
/**
* cookiesAreKeptAndSentTestCase checks the crawler's cookie handling:
* cookies a host sets are sent back to that host and no other, a
* cookie with Max-Age=0 is removed, and the Site Cookies option's
* cookies go to the domain named and its subdomains.
*/
public function cookiesAreKeptAndSentTestCase()
{
FetchUrl::$host_cookies = [];
FetchUrl::setSiteCookies(["example.com: euconsent=granted",
"other.org: a'b=1"]);
FetchUrl::keepCookies("https://shop.test/a", "HTTP/1.1 302 Found\r\n" .
"Set-Cookie: session=abc; Path=/; HttpOnly\r\n" .
"Set-Cookie: seen=1\r\n\r\nbody");
$this->assertEqual(FetchUrl::cookieHeaderFor("https://shop.test/b"),
"session=abc; seen=1", "a host gets back the cookies it set");
$this->assertEqual(FetchUrl::cookieHeaderFor("https://other.test/"),
"", "another host gets none of them");
FetchUrl::keepCookies("https://shop.test/c", "HTTP/1.1 200 OK\r\n" .
"Set-Cookie: seen=; Max-Age=0\r\n\r\n");
$this->assertEqual(FetchUrl::cookieHeaderFor("https://shop.test/"),
"session=abc", "a cookie with Max-Age=0 is removed");
$this->assertEqual(FetchUrl::cookieHeaderFor(
"https://www.example.com/x"), "euconsent=granted",
"a subdomain gets the option's cookies for its domain");
$this->assertEqual(FetchUrl::cookieHeaderFor("https://notexample.com/"),
"", "a domain merely ending in the same letters gets none");
$this->assertEqual(FetchUrl::cookieHeaderFor("https://other.org/"),
"a'b=1", "entities saved by the options form are decoded");
FetchUrl::setSiteCookies([]);
FetchUrl::$host_cookies = [];
}
/**
* webBotAuthSignsRequestsTestCase checks the Web Bot Auth headers and
* key directory against Cloudflare's documentation. Its example public
* key must give its example keyid, and its example request signature
* must verify over the signature base this case builds. The headers
* getPages adds for an https request, and the signed directory, must
* then verify over the same kind of base, with a key made for the case.
*/
public function webBotAuthSignsRequestsTestCase()
{
$example_x = "JrQLj5P_89iXES9-vFgrIy29clF9CC_oPPsw3c5D0bs";
$example_params = '("@authority" "signature-agent");' .
'created=1735689600;keyid="poqkLGiymh_W0uP6PZFw-dvez3QJT5So' .
'lqXBCW38r0U";alg="ed25519";expires=1735693200;nonce="e8N7S2' .
'MFd/qrd6T2R3tdfAuuANngKI7LFtKYI/vowzk4lAZYadIX6wW25MwG7DCT9R' .
'UKAJ0qVkU0mEeLElW1qg==";tag="web-bot-auth"';
$example_signature = "jdq0SqOwHdyHr9+r5jw3iYZH6aNGKijYp/EstF4RQ" .
"TQdi5N5YYKrD+mCT1HA1nZDsi6nJKuHxUi/5Syp3rLWBA==";
$this->assertEqual(FetchUrl::webBotAuthKeyId($example_x),
"poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U",
"the example key gives the example keyid");
$example_key = base64_decode(strtr($example_x, "-_", "+/") . "=");
$this->assertTrue(sodium_crypto_sign_verify_detached(
base64_decode($example_signature), "\"@authority\": " .
"example.com\n\"signature-agent\": " .
"\"https://signature-agent.test\"\n\"@signature-params\": " .
$example_params, $example_key),
"the example signature verifies over the base built here");
$key_pair = sodium_crypto_sign_keypair();
FetchUrl::$web_bot_auth_key_pair = $key_pair;
$public_key = sodium_crypto_sign_publickey($key_pair);
$headers = FetchUrl::webBotAuthHeaders(
"https://Example.com:8443/a?b=1", "https://bot.example.org");
$this->assertEqual($headers[0] ?? "",
'Signature-Agent: "https://bot.example.org"',
"the agent is sent as a quoted string");
$params = substr($headers[1] ?? "", strlen("Signature-Input: sig1="));
$signature = substr($headers[2] ?? "", strlen("Signature: sig1=:"),
-1);
$this->assertTrue(sodium_crypto_sign_verify_detached(
base64_decode($signature), "\"@authority\": example.com:8443\n" .
"\"signature-agent\": \"https://bot.example.org\"\n" .
"\"@signature-params\": $params", $public_key),
"a request's signature verifies over its authority and agent");
$this->assertEqual(FetchUrl::webBotAuthHeaders(
"http://example.com/", "https://bot.example.org"), [],
"a plain http request is not signed");
$this->assertEqual(FetchUrl::webBotAuthHeaders(
"https://example.com/", ""), [], "no agent means no signing");
[$directory_headers, $body] =
FetchUrl::webBotAuthDirectory("Bot.Example.org");
$public_x = rtrim(strtr(base64_encode($public_key), "+/", "-_"), "=");
$this->assertEqual(json_decode($body, true), ["keys" => [["kty" =>
"OKP", "crv" => "Ed25519", "x" => $public_x]]],
"the directory lists the public key");
$params = substr($directory_headers[2], strlen(
"Signature-Input: sig1="));
$signature = substr($directory_headers[1], strlen(
"Signature: sig1=:"), -1);
$this->assertTrue(sodium_crypto_sign_verify_detached(
base64_decode($signature), "\"@authority\";req: bot.example.org" .
"\n\"@signature-params\": $params", $public_key),
"the directory's signature verifies over the request authority");
FetchUrl::$web_bot_auth_key_pair = null;
$this->assertEqual(FetchUrl::webBotAuthHeaders(
"https://example.com/", "https://bot.example.org"), [],
"a fetcher not yet sent a key pair signs nothing");
}
/**
* botChallengeIsKnownTestCase checks that isBotChallenge picks out
* the page Imperva Incapsula sends while it checks a visitor, a
* script and no text, and does not pick out a site's own page that
* loads the same script beside its text.
*/
public function botChallengeIsKnownTestCase()
{
$challenge = "<html><head><meta name='robots' " .
"content='noindex,nofollow'><script src='/_Incapsula_Resource" .
"?SWJIYLWA=719d34d31c8e3a6e6fffd425f7e032f3'></script></head>" .
"<body></body></html>";
$this->assertTrue(FetchUrl::isBotChallenge($challenge),
"a page with the script and no text is a challenge");
$own_page = "<html><head><title>Pepsi</title><script " .
"src='/_Incapsula_Resource?SWJIYLWA=1'></script></head><body>" .
"<main>" . str_repeat("The official home of Pepsi. ", 20) .
"</main></body></html>";
$this->assertFalse(FetchUrl::isBotChallenge($own_page),
"a page with the script and its own text is not");
$this->assertFalse(FetchUrl::isBotChallenge("<html></html>"),
"an empty page without the script is not");
}
}
- Yioop Repo - Seekquarry