<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\HotTargets;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\Model;
/**
* HotTargetsTest checks the count of strikes a wiki page or the cached
* copies of a search index have drawn, and the judgment that a target has
* drawn enough to be turned away early.
*
* @author Chris Pollett
*/
class HotTargetsTest extends UnitTest
{
/**
* setUp points HotTargets at FOLDER, a folder in the tests folder, for
* its counts file and its marker files, so each case starts with
* neither and writes nothing of the site's.
*/
public function setUp()
{
$this->tearDown();
L\makePath(self::FOLDER);
HotTargets::$counts_folder = self::FOLDER;
HotTargets::$marker_folder = self::FOLDER . "/" .
HotTargets::MARKER_FOLDER;
}
/**
* tearDown deletes FOLDER and points HotTargets back at the site's
* folders.
*/
public function tearDown()
{
(new Model())->db->unlinkRecursive(self::FOLDER);
HotTargets::$counts_folder = C\TEMP_DIR;
HotTargets::$marker_folder = C\WORK_DIRECTORY . "/" .
HotTargets::MARKER_FOLDER;
}
/**
* FOLDER is the folder, under tests/test_files, that the cases have
* HotTargets keep its counts file and marker files in.
*/
const FOLDER = C\TEST_DIR . "/test_files/hot_targets";
/**
* pageBecomesHotAfterEnoughRefusalsTestCase checks that refusals of
* one page's history from many addresses add up on the page, that it
* is not hot before STRIKES_TO_BE_HOT refusals and is hot after, and
* that an ask for a different page or for a plain read is never
* counted.
*/
public function pageBecomesHotAfterEnoughRefusalsTestCase()
{
$key = HotTargets::keyFromRequest("/group/40?a=wiki",
["arg" => "history", "page_id" => "3971"]);
$this->assertEqual("40:3971:history", $key,
"the key names the group, page and what was asked");
$this->assertEqual("", HotTargets::keyFromRequest("/group/40?a=wiki",
["arg" => "read", "page_id" => "3971"]),
"a plain read is not something that is counted");
$now = 1000000;
for ($i = 1; $i < HotTargets::STRIKES_TO_BE_HOT; $i++) {
HotTargets::recordStrike($key, $now + $i);
}
$this->assertFalse(HotTargets::isHot($key, $now + 100),
"one short of the count, the page is not hot");
HotTargets::recordStrike($key, $now + 100);
$this->assertTrue(HotTargets::isHot($key, $now + 101),
"at the count, the page is hot");
$this->assertFalse(HotTargets::isHot(
"40:1:history", $now + 101),
"another page of the group is not hot");
}
/**
* hotPageStaysHotWhileTheWebServerRefusesItTestCase checks that a
* page that became hot stays hot well past WINDOW with no further
* refusal recorded, since the web server answers those requests and
* PHP never sees them, and cools only once HOT_FOR has passed; and
* that a page that never became hot is forgotten after WINDOW so
* old refusals do not count toward a new burst.
*/
public function hotPageStaysHotWhileTheWebServerRefusesItTestCase()
{
$key = "40:3971:source";
$now = 2000000;
for ($i = 0; $i < HotTargets::STRIKES_TO_BE_HOT; $i++) {
HotTargets::recordStrike($key, $now);
}
$this->assertTrue(HotTargets::isHot($key, $now + 1),
"the page is hot right after its refusals");
$this->assertTrue(HotTargets::isHot($key,
$now + HotTargets::WINDOW + 1),
"and is still hot past the window with nothing recorded since");
$this->assertFalse(HotTargets::isHot($key,
$now + HotTargets::HOT_FOR + 1),
"and cools once the hot span has passed");
$warm = "40:1:source";
HotTargets::recordStrike($warm, $now);
$later = $now + HotTargets::WINDOW + 1;
HotTargets::recordStrike($warm, $later);
$this->assertFalse(HotTargets::isHot($warm, $later + 1),
"a page that never became hot starts its count over after the" .
" window");
}
/**
* keyAndMarkerAgreeAcrossRequestShapesTestCase checks that the three
* shapes a scraper's request takes, a page by id in the query, a page
* by name in the path, and a path name with an id in the query, each
* key to what the web server's rewrite rule builds for it, so a
* marker PHP writes is the one Apache tests for; and that markers are
* written for hot targets only and taken away when a target cools.
*/
public function keyAndMarkerAgreeAcrossRequestShapesTestCase()
{
$shapes = [
["/wiki/group/40?a=wiki&arg=history&page_id=3920&show=1",
["arg" => "history", "page_id" => "3920"], "40-3920-history"],
["/wiki/group/2/Main?arg=source&settings=true",
["arg" => "source"], "2-Main-source"],
["/group/40/faculty-office-hours?arg=source&page_id=3920",
["arg" => "source", "page_id" => "3920"], "40-3920-source"],
];
$folder = HotTargets::$marker_folder;
foreach ($shapes as [$uri, $request, $marker]) {
$key = HotTargets::keyFromRequest($uri, $request);
HotTargets::writeMarkers([$key => ["count" =>
HotTargets::STRIKES_TO_BE_HOT, "last" => 5,
"hot_since" => 5]]);
$this->assertTrue(file_exists("$folder/$marker"),
"the marker for $uri is what the rewrite rule builds");
}
$hot = ["40:3920:history" => ["count" =>
HotTargets::STRIKES_TO_BE_HOT, "last" => 5, "hot_since" => 5]];
$cool = ["40:3920:history" => ["count" => 1, "last" => 5]];
HotTargets::writeMarkers($hot);
$this->assertTrue(file_exists($folder . "/40-3920-history"),
"a hot target has its marker file");
HotTargets::writeMarkers($cool);
$this->assertFalse(file_exists($folder . "/40-3920-history"),
"and loses it once it is no longer hot");
}
/**
* cacheMissesAcrossPagesMakeAnIndexHotTestCase checks that a request
* for a cached copy of a search result keys to the index it names and
* not to the page, so misses spread across many pages of one index
* from many addresses add up on that index; that a request with no
* index named keys to the default one; that a result's icon, which
* the search page itself asks for, and a request to another
* controller are never counted; and that the marker for the index is
* what the web server's rewrite rule builds.
*/
public function cacheMissesAcrossPagesMakeAnIndexHotTestCase()
{
$its = "1702631344";
$uri = "/s/trends?c=search&a=cache&q&arg=https%3A%2F%2Fwww.w3.org" .
"%2F&from_cache=true&its=$its";
$request = ["c" => "search", "a" => "cache",
"arg" => "https://www.w3.org/", "from_cache" => "true",
"its" => $its];
$key = HotTargets::keyFromRequest($uri, $request);
$this->assertEqual("cache:$its", $key,
"a cache request keys to the index it names");
HotTargets::writeMarkers([$key => ["count" =>
HotTargets::STRIKES_TO_BE_HOT, "last" => 5, "hot_since" => 5]]);
$this->assertTrue(file_exists(HotTargets::$marker_folder .
"/cache-$its"),
"and its marker is what the rewrite rule builds");
$no_index = $request;
unset($no_index["its"]);
$this->assertEqual("cache:0", HotTargets::keyFromRequest("/",
$no_index), "a request naming no index keys to the default");
$icon = $request + ["field" => "favicon"];
$this->assertEqual("", HotTargets::keyFromRequest("/", $icon),
"a result's icon is not counted");
$other = ["c" => "group"] + $request;
$this->assertEqual("", HotTargets::keyFromRequest("/", $other),
"a request to another controller is not counted");
$now = 3000000;
for ($i = 0; $i < HotTargets::STRIKES_TO_BE_HOT; $i++) {
$page = ["arg" => "https://example.com/$i"] + $request;
HotTargets::recordStrike(HotTargets::keyFromRequest($uri,
$page), $now + $i);
}
$this->assertTrue(HotTargets::isHot($key,
$now + HotTargets::STRIKES_TO_BE_HOT),
"misses on different pages of one index make the index hot");
}
/**
* hotSearchCoolsOneStrikeShortTestCase checks what happens when the
* search target's HOT_FOR seconds run out. It is no longer hot, but one
* further search within WINDOW seconds makes it hot again, rather than
* the SEARCHES_TO_BE_HOT a target starting afresh needs. With no search
* in WINDOW seconds it is dropped.
*/
public function hotSearchCoolsOneStrikeShortTestCase()
{
$key = HotTargets::SEARCH_KEY;
$start = time() - HotTargets::HOT_FOR - 10;
for ($i = 0; $i < HotTargets::SEARCHES_TO_BE_HOT; $i++) {
HotTargets::recordStrike($key, $start);
}
$now = time();
$this->assertFalse(HotTargets::isHot($key, $now),
"the search target is not hot once its hour is over");
HotTargets::recordStrike($key, $now);
$this->assertTrue(HotTargets::isHot($key, $now),
"one more search makes it hot again");
$cooled = HotTargets::prune([$key => ["count" => 100,
"last" => $start, "hot_since" => $start]],
$start + HotTargets::HOT_FOR + HotTargets::WINDOW + 5);
$this->assertEqual($cooled, [],
"with no search in the window after its hour it is dropped");
}
/**
* cookielessSearchesMakeSearchHotTestCase checks that a search asked
* with no session cookie counts under one key whatever its terms or
* its address, that the site's own fan-out of a query, which says
* network=false, and a request to the fetch controller are never
* counted, and that the search key turns hot at SEARCHES_TO_BE_HOT
* rather than at the page count.
*/
public function cookielessSearchesMakeSearchHotTestCase()
{
$this->assertEqual(HotTargets::SEARCH_KEY,
HotTargets::keyFromRequest("/?q=chris+pollett",
["q" => "chris pollett"]), "a search counts under the search key");
$this->assertEqual(HotTargets::SEARCH_KEY,
HotTargets::keyFromRequest("/s/trends?c=search&q=news",
["c" => "search", "q" => "news", "a" => "query"]),
"a query to the search controller by name counts too");
$this->assertEqual("", HotTargets::keyFromRequest("/",
["q" => "news", "network" => "false", "f" => "serial"]),
"the site's own fan-out of a query is not counted");
$this->assertEqual("", HotTargets::keyFromRequest("/",
["c" => "fetch", "a" => "schedule", "q" => "x"]),
"the crawl's own traffic is not counted");
$this->assertEqual("", HotTargets::keyFromRequest("/", []),
"the home page is not counted");
$now = 4000000;
for ($i = 0; $i < HotTargets::STRIKES_TO_BE_HOT; $i++) {
HotTargets::recordStrike(HotTargets::SEARCH_KEY, $now + $i);
}
$this->assertFalse(HotTargets::isHot(HotTargets::SEARCH_KEY,
$now + HotTargets::STRIKES_TO_BE_HOT),
"the page count does not make searching hot");
for ($i = HotTargets::STRIKES_TO_BE_HOT;
$i < HotTargets::SEARCHES_TO_BE_HOT; $i++) {
HotTargets::recordStrike(HotTargets::SEARCH_KEY, $now + $i);
}
$this->assertTrue(HotTargets::isHot(HotTargets::SEARCH_KEY,
$now + HotTargets::SEARCHES_TO_BE_HOT),
"the search count does");
}
}
- Yioop Repo - Seekquarry