<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\configs as C;
/**
* HotTargets keeps a count of strikes against a target, by the thing asked
* for rather than by the one who asked, and says when a target has drawn
* enough strikes in a short time that further requests for it from a
* reader with no session can be turned away before the site does any
* work. There are two kinds of target. A wiki page's history or source
* draws a strike each time a request for it is refused. The cached copies
* of the pages in one search index draw a strike each time a request with
* no session asks for a copy the index does not hold; such a miss costs a
* walk of every index, and a scraper following the links on cached pages
* makes one miss after another. A scraper that rotates its address is not
* slowed by a count kept per address, since each request arrives from a
* fresh one; what it cannot change is the page or the index it wants, so
* the count is kept there. The counts live in one small file under the
* temp directory so reading them costs a file read and no database.
* Controller::refuseForbidden and SearchController::cacheRequestAndOutput
* record strikes here, and index.php asks before starting a session.
*
* @author Chris Pollett
*/
class HotTargets
{
/**
* Name of the file, under the temp directory, holding the counts.
*/
const FILE_NAME = "HotTargets.txt";
/**
* How many strikes against one target in the window make it hot.
*/
const STRIKES_TO_BE_HOT = 20;
/**
* How long, in seconds, strikes against a target are counted toward
* its turning hot.
*/
const WINDOW = 300;
/**
* How long, in seconds, a target stays hot once it is hot, measured
* from the strike that made it so. This is much longer than
* WINDOW because once the web server refuses a hot target its requests
* never reach PHP, so no further strike is recorded and the target
* would look idle while still under attack: measured on a live site,
* a target cooled every five minutes and cost twenty full PHP requests
* to become hot again each time. A stale marker costs a reader with
* no session one refusal of a private page, or of a cached copy.
*/
const HOT_FOR = 3600;
/**
* The arguments whose refusal is counted: requests for a page's history
* or its source, the two things a private page keeps from a reader
* who may not edit it, and the two a scraper of a wiki asks for.
*/
const COUNTED_ARGS = ["history", "source"];
/**
* The key under which every search asked with no session cookie is
* counted, so a flood of such searches from many addresses adds up
* on one target. It is not a page or an index, so no marker file of
* the web server's rules matches it and the refusal is made in PHP,
* which can say why.
*/
const SEARCH_KEY = "search";
/**
* Searches with no session cookie within WINDOW seconds that make
* searching without a cookie hot. It is higher than the page count
* because many people search a site in five minutes while few ask
* for one page's history, and a person refused is told to accept the
* site's cookie and search again.
*/
const SEARCHES_TO_BE_HOT = 100;
/**
* Folder, under the work directory, holding one empty file per hot
* target, named for the target's key with its colons turned to
* hyphens, as writeMarkers names it. The web server's rewrite
* rules test for the file and refuse a request for a hot target
* without starting PHP at all, so a scraper costs the site a file
* existence check. It sits under the work directory because Yioop
* never writes into its own source tree. The .htaccess looks for it
* at work_directory beside the source, where the work directory is
* unless configured elsewhere; a site whose work directory is
* elsewhere names the folder to the web server with one SetEnvIf line.
*/
const MARKER_FOLDER = "hot";
/**
* $counts_folder is the folder the counts file is kept in, Yioop's
* temporary folder. HotTargetsTest sets it to a folder in the tests
* folder.
* @var string
*/
public static $counts_folder = C\TEMP_DIR;
/**
* $marker_folder is the folder of marker files the web server's
* rewrite rule reads, MARKER_FOLDER in the work directory.
* HotTargetsTest sets it to a folder in the tests folder.
* @var string
*/
public static $marker_folder = C\WORK_DIRECTORY . "/" .
self::MARKER_FOLDER;
/**
* keyFromRequest names the target a request is for, so the gate
* that runs before the router, the strike that runs after it, and
* the web server's rewrite rule all arrive at the same key for the
* same request. A request to the search controller for a cached copy
* of a search result keys to the index it names, as cache:N with N
* the index timestamp, or cache:0 where it names none and so reads
* the default index. A request for a result's icon also goes through
* that activity with a field named, but the search page asks for it
* rather than a scraper, so it is not counted. For a wiki page, the
* group comes from a /group/N path where there is
* one, else the query. The page is the page_id from the query where
* the request carries one, else the name from a /group/N/Page path,
* else the page_name from the query; the id is preferred because the
* rewrite rule can read it without knowing the path's shape.
* @param string $uri the request path and query as the server got it
* @param array $request the query arguments as decoded so far
* @return string the key, or an empty string where nothing is counted
*/
public static function keyFromRequest($uri, $request)
{
$controller = $request['c'] ?? "search";
if (($request['a'] ?? "") == "cache" && $controller == "search") {
return empty($request['field']) ? "cache:" .
intval($request['its'] ?? 0) : "";
}
/* A search is a query to the search controller with a term. One
the site sends to itself or to a managed machine says
network=false, and is never counted, so the site's own fan-out
of a query and its crawl traffic are left alone. */
if ($controller == "search" && !empty($request['q']) &&
in_array($request['a'] ?? "", ["", "query"]) &&
($request['network'] ?? "") !== "false") {
return self::SEARCH_KEY;
}
$arg = $request['arg'] ?? "";
$group_id = $request['group_id'] ?? 0;
$path_page = "";
$path = strtok($uri, "?");
if (preg_match('@/group/([0-9]+)(?:/([^/?]+))?@', $path, $match)) {
$group_id = $match[1];
$path_page = urldecode($match[2] ?? "");
}
$page = $request['page_id'] ?? "";
if ($page === "") {
$page = ($path_page !== "") ? $path_page :
($request['page_name'] ?? "");
}
if (!in_array($arg, self::COUNTED_ARGS)) {
return "";
}
/* The key names the group, the page and which of history or source
was asked for, so two requests for one page's history from
different addresses share a key. */
return intval($group_id) . ":" . $page . ":" . $arg;
}
/**
* recordStrike adds one strike against a target at the given time and
* writes the counts back, dropping any target whose window has passed.
* A strike is a refused request for a wiki page's history or source,
* or a request with no session for a cached copy that the index does
* not hold.
* @param string $key the target, from keyFromRequest
* @param int $now the time of the strike
* @return void nothing is handed back, the count is written
*/
public static function recordStrike($key, $now)
{
if ($key === "") {
return;
}
if (!file_exists(self::$counts_folder)) {
mkdir(self::$counts_folder, 0777, true);
}
/* The count is read, added to and written back while this
process holds the file's lock, so a strike another process
records at the same moment is not lost. */
$handle = fopen(self::$counts_folder . "/" . self::FILE_NAME, "c+");
if ($handle === false) {
return;
}
flock($handle, LOCK_EX);
$counts = json_decode(stream_get_contents($handle), true);
$counts = self::prune(is_array($counts) ? $counts : [], $now);
if (!isset($counts[$key])) {
$counts[$key] = ["count" => 0, "last" => $now];
}
$counts[$key]["count"]++;
$counts[$key]["last"] = $now;
$strikes_to_be_hot = ($key == self::SEARCH_KEY) ?
self::SEARCHES_TO_BE_HOT : self::STRIKES_TO_BE_HOT;
if ($counts[$key]["count"] >= $strikes_to_be_hot &&
!isset($counts[$key]["hot_since"])) {
$counts[$key]["hot_since"] = $now;
}
ftruncate($handle, 0);
rewind($handle);
fwrite($handle, json_encode($counts));
fflush($handle);
flock($handle, LOCK_UN);
fclose($handle);
self::writeMarkers($counts);
}
/**
* isHot says if a target turned hot, by drawing STRIKES_TO_BE_HOT
* strikes within WINDOW seconds, less than HOT_FOR seconds before
* the given time.
* @param string $key the target, from keyFromRequest
* @param int $now the time of the request being judged
* @return bool true where the target is hot
*/
public static function isHot($key, $now)
{
if ($key === "") {
return false;
}
$counts = self::prune(self::read(), $now);
return isset($counts[$key]["hot_since"]);
}
/**
* writeMarkers makes the marker folder hold exactly one file per hot
* target: a file for each target now hot, and none for any other.
* recordStrike calls this after every count so the web server's view
* follows the counts.
* @param array $counts targets to their count and last strike time,
* already pruned to the window
* @return void nothing is handed back, the folder is written
*/
public static function writeMarkers($counts)
{
$folder = self::$marker_folder;
if (!file_exists($folder)) {
mkdir($folder, 0755, true);
}
$wanted = [];
foreach ($counts as $key => $entry) {
if (isset($entry["hot_since"])) {
/* The marker file is the key with its separators turned
to hyphens and every other character that is not a
letter, digit, hyphen or underscore dropped, which the
web server's rewrite rule builds from the request with
the same substitution. */
$wanted[preg_replace('/[^A-Za-z0-9_-]/', "",
str_replace(":", "-", $key))] = true;
}
}
foreach (glob($folder . "/*") as $present) {
if (!isset($wanted[basename($present)])) {
unlink($present);
}
}
foreach ($wanted as $name => $unused) {
touch($folder . "/" . $name);
}
}
/**
* prune drops each target that is past its span: a target that was
* not hot once WINDOW seconds have passed since its last strike. A hot
* target whose HOT_FOR seconds have passed is not dropped but left one
* strike short of hot, as if that strike had come when its hour ended.
* The next ask for it within WINDOW seconds makes it hot again at
* once; with no ask in that time it is dropped as any other target.
* Dropped outright, a target that bots were still asking for went
* back to needing its whole count, and every ask until then ran in a
* web server worker: on 23 October that let 383 searches through in
* three minutes and held up the crawl's requests behind them.
* @param array $counts targets to their count and last strike time
* @param int $now the time to measure age from
* @return array the targets still within the window
*/
public static function prune($counts, $now)
{
foreach ($counts as $key => $entry) {
if (isset($entry["hot_since"]) &&
$now - $entry["hot_since"] > self::HOT_FOR) {
$strikes_to_be_hot = ($key == self::SEARCH_KEY) ?
self::SEARCHES_TO_BE_HOT : self::STRIKES_TO_BE_HOT;
$entry = ["count" => $strikes_to_be_hot - 1,
"last" => $entry["hot_since"] + self::HOT_FOR];
$counts[$key] = $entry;
}
if (!isset($entry["hot_since"]) &&
$now - $entry["last"] > self::WINDOW) {
unset($counts[$key]);
}
}
return $counts;
}
/**
* read gives the counts held in the file, or none where there is no
* file yet or it cannot be read.
* @return array targets to their count and last strike time
*/
public static function read()
{
$path = self::$counts_folder . "/" . self::FILE_NAME;
if (!file_exists($path)) {
return [];
}
$handle = fopen($path, "r");
if ($handle === false) {
return [];
}
flock($handle, LOCK_SH);
$counts = json_decode(stream_get_contents($handle), true);
flock($handle, LOCK_UN);
fclose($handle);
return is_array($counts) ? $counts : [];
}
}
- Yioop Repo - Seekquarry