<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\storage_formats\IndexShard;
/**
* Loads crawlHash, base64Hash and canonicalTerm, which make the keys this
* class looks up
*/
require_once __DIR__ . "/Utility.php";
/**
* IndexManager holds the index bundles a search has open. Each part of a
* query asks this class for a bundle by name, so each bundle is opened once
* in a process and shared, rather than opened again by each part. It also
* remembers recent dictionary lookups, by word and by page address.
*
* @author Chris Pollett
*/
class IndexManager implements CrawlConstants
{
/**
* $indexes holds the open index bundles, keyed by the name getIndex
* was asked for
* @var array
*/
public static $indexes = [];
/**
* $index_times holds, for each bundle name, the time the bundle was
* opened. It is ordered from least to most recently used, so the
* first name is the one dropped when too many bundles are open.
* @var array
*/
public static $index_times = [];
/**
* $index_stamps holds, for each bundle name, the state of its folder on
* disk when the bundle was last opened or checked, as indexStamp gives
* it, so that getIndex reopens a bundle another process has changed
* @var array
*/
public static $index_stamps = [];
/**
* $index_stamp_times holds, for each bundle name, when getIndex last
* compared the bundle's folder with its stamp, so the comparison, which
* reads files, is made at most once each STAMP_CHECK_SECONDS rather
* than on each of the many getIndex calls a search makes.
* @var array
*/
public static $index_stamp_times = [];
/**
* STAMP_CHECK_SECONDS is the number of seconds between the times
* getIndex reads a bundle's folder to see whether another process
* changed it. A search calls getIndex for each document it scores, so
* checking on each call would read the folder thousands of times a
* search.
*/
const STAMP_CHECK_SECONDS = 1;
/**
* $urls_cache holds, for each page address hash that
* lookupLatestVersionPage has looked up, the partition and posting
* of the newest copy of the page
* @var array
*/
public static $urls_cache = [];
/**
* $info_cache holds what was looked up about a word, kept under a
* hash of the query parameters.
* Kept as a class property (rather than a local static of
* getWordInfo) so the periodic memory log can report how large it has
* grown.
* @var array
*/
public static $info_cache = [];
/**
* MAX_OPEN_INDEXES is how many indexes are kept open at once. Yioop
* reads from a handful of indexes at most, and an index is held on
* disk rather than in memory, so the count that matters is small and
* fixed. The index used least recently is dropped once a sixth is
* opened.
*/
const MAX_OPEN_INDEXES = 5;
/**
* MAX_REMEMBERED_LOOKUPS is how many lookups are remembered, both for
* words and for page addresses. A lookup is a small record and a
* search makes many of them, so this is far larger than the number
* of indexes.
*/
const MAX_REMEMBERED_LOOKUPS = 1000;
/**
* getIndex gives back the open bundle for an index, opening it first
* if this process has not, or if another process has changed it on
* disk since. The name can be a crawl's timestamp, "feed" for the
* index of news and media feeds, or the path of an index's folder.
* A timestamp may name a double index, which is two bundles where
* one is served while the other is built. An index made before the
* current layout is opened as an IndexArchiveBundle, and any other as
* an IndexDocumentBundle.
*
* @param string $index_name timestamp, "feed" or path of the index
* @return object|bool the bundle, or false if there is no such index
*/
public static function getIndex($index_name)
{
$index_name = trim($index_name ?? ""); //trim to fix postgres quirkiness
$cache_dir = C\CACHE_DIR . '/';
$index_archive_full_path = $cache_dir . self::index_data_base_name .
$index_name;
$handled = false;
if ($index_name == "feed" || $index_name == self::FEED_CRAWL_TIME) {
$index_archive_name = self::feed_index_data_base_name;
$index_name = "feed";
$handled = true;
} else if (is_numeric($index_name) &&
file_exists($index_archive_full_path)) {
$index_archive_name = self::index_data_base_name . $index_name;
$handled = true;
}
if (!$handled && is_numeric($index_name) ) {
$index_name = $cache_dir . self::double_index_base_name .
$index_name;
}
if (!$handled && file_exists($index_name) ) {
$is_repeating = file_exists("$index_name/bundle0");
$serve_archive = "0";
if ($is_repeating) {
$status_file = "$index_name/status.txt";
if (file_exists($status_file)) {
$status = unserialize(file_get_contents($status_file));
$active_archive = (empty($status["swap_count"])) ? 1 :
$status["swap_count"] % 2;
$serve_archive = 1 - $active_archive;
}
}
$sub_folder = ($is_repeating) ? "/bundle$serve_archive" : "";
$is_old = ($is_repeating) ? (file_exists($index_name.
"/bundle0/summaries")) : (file_exists($index_name .
"/summaries"));
$bundle_class_name = (
$is_old) ? C\NS_LIB . "storage_formats\\IndexArchiveBundle"
: C\NS_LIB . "storage_formats\\IndexDocumentBundle";
$tmp = new $bundle_class_name($index_name . $sub_folder);
} else if (!$handled) {
return false;
}
$stamp_folder = isset($index_archive_name) ?
$cache_dir . $index_archive_name : $index_name;
$now = microtime(true);
if ($now - (self::$index_stamp_times[$index_name] ?? 0) >=
self::STAMP_CHECK_SECONDS) {
self::$index_stamp_times[$index_name] = $now;
$stamp = self::indexStamp($stamp_folder);
if (!empty(self::$indexes[$index_name]) &&
(self::$index_stamps[$index_name] ?? "") !== $stamp) {
/* Another process, such as the queue server or a remerge,
has changed the bundle on disk since it was opened here,
so the bundle kept from then, and the dictionary lookups
made with it, are out of date. */
unset(self::$indexes[$index_name],
self::$index_times[$index_name]);
self::$info_cache = [];
$reopen_start = $now;
}
AnalyticsManager::addTiming("index change checks", $now);
self::$index_stamps[$index_name] = $stamp;
}
if (empty(self::$indexes[$index_name]) ||
(!empty(self::$index_times[$index_name]) &&
($index_name == 'feed' && php_sapi_name() == 'cli') &&
(time() - self::$index_times[$index_name])
> C\MIN_QUERY_CACHE_TIME) ) {
if (!isset($tmp)) {
$index_filename = $cache_dir . $index_archive_name;
if (file_exists($index_filename)) {
$is_old = (file_exists($index_filename . "/summaries"));
$bundle_class_name = ($is_old) ?
C\NS_LIB . "storage_formats\\IndexArchiveBundle"
: C\NS_LIB .
"storage_formats\\IndexDocumentBundle";
$tmp = new $bundle_class_name($cache_dir .
$index_archive_name);
if (!$tmp) {
return false;
}
} else {
$tmp = false;
$use_name = $index_name;
$serve_archive = -1;
if (preg_match("/\-\d$/", $index_name)) {
$serve_archive = substr($index_name, -1);
$use_name = substr($index_name, 0, -2);
}
$index_archive_name = self::double_index_base_name .
$use_name;
$status_file = $cache_dir . $index_archive_name .
"/status.txt";
if ($serve_archive < 0 && file_exists($status_file)) {
$status = unserialize(file_get_contents($status_file));
$active_archive = (empty($status["swap_count"])) ? 1 :
$status["swap_count"] % 2;
$serve_archive = 1 - $active_archive;
}
$is_old = (file_exists($index_filename .
"/bundle0/summaries") ||
file_exists($index_filename . "/bundle1/summaries"));
$bundle_class_name = ($is_old) ?
C\NS_LIB . "storage_formats\\IndexArchiveBundle"
: C\NS_LIB . "storage_formats\\IndexDocumentBundle";
$tmp = new $bundle_class_name($cache_dir .
$index_archive_name . "/bundle$serve_archive");
if (!$tmp) {
$serve_archive = ($serve_archive == 0) ? 1 : 0;
$tmp = new $bundle_class_name($cache_dir .
$index_archive_name . "/bundle$serve_archive");
}
if (!$tmp) {
return false;
}
}
}
self::$indexes[$index_name] = $tmp;
if ($is_old) {
self::$indexes[$index_name]->setCurrentShard(0, true);
}
if (isset($reopen_start)) {
AnalyticsManager::addTiming(
"index reopened after a change on disk", $reopen_start);
}
self::$index_times[$index_name] = time();
while (count(self::$indexes) > self::MAX_OPEN_INDEXES) {
$least_recent = array_key_first(self::$index_times);
unset(self::$index_times[$least_recent],
self::$indexes[$least_recent]);
}
}
if (isset(self::$indexes[$index_name]) &&
isset(self::$index_times[$index_name])) {
$keep_time = self::$index_times[$index_name];
unset(self::$index_times[$index_name]);
self::$index_times[$index_name] = $keep_time;
}
return self::$indexes[$index_name];
}
/**
* indexStamp describes the state of an index bundle's folder on disk,
* so that getIndex can tell whether a bundle it opened earlier in this
* process is still current. Yioop's own web server keeps one process
* running between requests, so a bundle opened by a search while a
* crawl runs would otherwise be used after the crawl has merged more
* partitions into the dictionary, or after a remerge has rebuilt it,
* and searches would miss or misread what was added. The stamp joins
* what the queue server and ArcTool rewrite when they change the
* bundle: the contents of next_partition.txt, of the Dictionary's
* max_tier.txt, of the partitions' parameters and of a double index's
* status.txt, the names in the Dictionary folder, and the change time
* of each. Contents and names are part of it because two changes within
* one second leave a change time as it was.
* @param string $folder path to the folder of the index bundle
* @return string the stamp; two stamps that differ mean the bundle
* changed on disk between them
*/
public static function indexStamp($folder)
{
$stamp = "";
foreach (["next_partition.txt", "Dictionary/max_tier.txt",
"Documents/pdb_parameters.txt", "status.txt"] as $part) {
$path = "$folder/$part";
clearstatcache(true, $path);
$stamp .= file_exists($path) ? crc32(file_get_contents($path)) .
":" . filemtime($path) . ";" : "-;";
}
$dictionary = "$folder/Dictionary";
clearstatcache(true, $dictionary);
if (is_dir($dictionary)) {
$stamp .= implode(",", scandir($dictionary)) . ":" .
filemtime($dictionary);
}
return $stamp;
}
/**
* clearCache forgets all open bundles and the stamps kept for the
* bundles, so the next getIndex for each name opens it again from disk
*/
public static function clearCache()
{
self::$indexes = [];
self::$index_times = [];
self::$index_stamps = [];
self::$index_stamp_times = [];
}
/**
* getVersion gives the format version of an index, which determines how
* words are looked up in it. Version 0 is the original format, used
* by crawls before VERSION_0_TIMESTAMP, whose word ids are 8 bytes.
* Later indexes record their version, and one that does not is taken
* to be DEFAULT_CRAWL_FORMAT.
*
* @param string $index_name timestamp of the index, or the folder of
* one; a leading minus sign is ignored
* @return int the index's format version
*/
public static function getVersion($index_name)
{
$index_name = (string) $index_name;
$index_name = (empty($index_name) || $index_name[0] != '-') ?
$index_name : substr($index_name, 1);
$index_time = intval($index_name);
if (!is_numeric($index_name)) {
$description_file = $index_name . "/summaries/description.txt";
if (file_exists($description_file)) {
$description = unserialize(
file_get_contents($description_file));
if (!empty($description['DESCRIPTION'])) {
$description = unserialize($description['DESCRIPTION']);
}
if (!empty($description[self::CRAWL_TIME])) {
if (intval($description[self::CRAWL_TIME]) <
C\VERSION_0_TIMESTAMP &&
intval($description[self::CRAWL_TIME]) !=
self::FEED_CRAWL_TIME) {
return 0;
}
}
}
} else if ($index_time != self::FEED_CRAWL_TIME &&
$index_time < C\VERSION_0_TIMESTAMP) {
return 0;
}
$tmp_index = self::getIndex($index_name);
if (isset($tmp_index->version)) {
return $tmp_index->version;
} else if (isset($tmp_index->archive_info['VERSION'])) {
return $tmp_index->archive_info['VERSION'];
}
return C\DEFAULT_CRAWL_FORMAT;
}
/**
* getWordInfo looks a word or phrase up in an index's dictionary and
* gives, for each partition (called a generation in older indexes)
* holding it, where its postings are. Recent answers are remembered
* in $info_cache, which is emptied when the index changes on disk.
*
* @param string $index_name timestamp of the index to look in
* @param string $term_id the word or phrase, as the dictionary keys it
* @param int $threshold stop once the documents found exceed this
* many; -1 for no limit
* @param int $start_generation which partition to look from
* @param int $num_distinct_generations how many partitions to look
* across from $start_generation; -1 for all of them
* @param bool $with_remaining_total whether to also give the total
* number of documents holding the word
* @return array for a current index, the dictionary's rows
* with totals; for an older one, a list of (generation, offset,
* length, ...) rows, or [total, that list]
*/
public static function getWordInfo($index_name, $term_id, $threshold = -1,
$start_generation = -1, $num_distinct_generations = -1,
$with_remaining_total = false)
{
$lookup_hash = crawlHash($index_name . $term_id . $threshold .
$start_generation . $num_distinct_generations .
$with_remaining_total);
/* getIndex comes first: when the bundle changed on disk it also
empties the remembered lookups, which would otherwise answer
from before the change. */
$index = self::getIndex($index_name);
if (isset(self::$info_cache[$lookup_hash])) {
$tmp = self::$info_cache[$lookup_hash];
unset(self::$info_cache[$lookup_hash]);
self::$info_cache[$lookup_hash] = $tmp;
return $tmp;
}
$start_generation = ($start_generation < 0) ? 0 : $start_generation;
/* An index made before the current layout is served by an older
bundle, which does not look words up itself; its dictionary and
its newest shard are asked instead. Newer indexes answer for
themselves, so they are asked directly. */
if (method_exists($index, "getWordInfo")) {
$word_info = $index->getWordInfo($term_id,
$threshold, $start_generation, $num_distinct_generations,
$with_remaining_total);
} else {
$word_info = self::getWordInfoOlderIndex($index, $term_id,
$threshold, $start_generation, $num_distinct_generations,
$with_remaining_total);
}
self::$info_cache[$lookup_hash] = $word_info;
if (count(self::$info_cache) >= self::MAX_REMEMBERED_LOOKUPS) {
array_shift(self::$info_cache);
}
return $word_info;
}
/**
* getWordInfoOlderIndex looks a word up in an index made before
* the current layout. Such an
* index keeps a dictionary of the words its finished generations
* hold, and may have a newest generation whose shard has not been
* folded into that dictionary yet, so both are asked and what they
* say is put together. A newer index answers a lookup itself and
* never reaches here.
*
* @param object $index bundle holding the index being read
* @param string $term_id what to look up, as the index names it
* @param int $threshold stop looking after this many documents; -1
* for no limit
* @param int $start_generation which generation to look from
* @param int $num_distinct_generations how many generations to look
* across; negative for all of them
* @param bool $with_remaining_total whether to answer with the total
* number of documents alongside the list of them
* @return array what the index knows about the word: a list of
* (generation, offset, count, ...) rows, or that list together
* with a total when one was asked for
*/
private static function getWordInfoOlderIndex($index, $term_id,
$threshold, $start_generation, $num_distinct_generations,
$with_remaining_total)
{
$pre_info = [];
if (!empty($index->dictionary)) {
$pre_info = $index->dictionary->getWordInfo($term_id, true,
$threshold, $start_generation, $num_distinct_generations,
true);
}
$last_desired_generation = $start_generation +
$num_distinct_generations;
if (isset($index->generation_info['ACTIVE'])) {
$active_generation = $index->generation_info['ACTIVE'];
if ((empty($index->generation_info['LAST_DICTIONARY_SHARD']) ||
$index->generation_info['LAST_DICTIONARY_SHARD'] <
$active_generation) && ($active_generation <
$last_desired_generation || $last_desired_generation < 0)) {
$active_shard_file = $index->dir_name .
"/posting_doc_shards/index" . $active_generation;
if (file_exists($active_shard_file)) {
if (!empty($index->non_merged_shard) &&
!empty($index->non_merged_generation) &&
$index->non_merged_generation == $active_generation) {
$active_shard = $index->non_merged_shard;
} else {
$active_shard = new IndexShard($active_shard_file, 0,
C\NUM_DOCS_PER_PARTITION, true);
$index->non_merged_shard = $active_shard;
$index->non_merged_generation = $active_generation;
}
$active_info = $active_shard->getWordInfo($term_id, true);
if (is_array($active_info)) {
if (empty($pre_info)) {
$pre_info[0] = 0;
$pre_info[1] = [];
}
$pre_info[1][] = [$active_generation,
$active_info[0], $active_info[1], $active_info[2],
$active_info[3]];
$pre_info[0] += $active_info[2];
}
}
}
}
if (!empty($pre_info[1])) {
list($total, $info) = $pre_info;
} else {
$total = 0;
$info = [];
}
return ($with_remaining_total) ? [$total, $info] : $info;
}
/**
* $term_postings_cache holds, by index, term and partition, the term's
* postings in that partition, as termPostingInPartition read them, so
* a query reads each such list once
* @var array
*/
public static $term_postings_cache = [];
/**
* termPostingInPartition gives a term's posting for the document at
* $doc_map_index in $partition, or [] if that document does not hold
* the term. GroupIterator::computeOutPages calls it for the newest
* copy of a page that reaches the results, to see whether the newest
* copy still holds the word the page was found for. The partition's
* row is found among all the term's rows, since the newest copy may
* be in a partition the query's walk never reached.
*
* @param string $index_name timestamp of the index to look in
* @param string $term_id the term's dictionary key
* @param int $partition partition holding the document
* @param int $doc_map_index position of the document in the
* partition's document map
* @return array the posting, with DOC_MAP_INDEX, FREQUENCY,
* POSITIONS_OFFSET and POSITIONS_LEN, or [] if there is none
*/
public static function termPostingInPartition($index_name, $term_id,
$partition, $doc_map_index)
{
$cache_key = "$index_name $partition " . bin2hex($term_id);
if (!isset(self::$term_postings_cache[$cache_key])) {
$index = self::getIndex($index_name);
$word_info = self::getWordInfo($index_name, $term_id);
$postings = [];
foreach ($word_info['ROWS'] ?? [] as $row) {
if ($row['PARTITION'] != $partition) {
continue;
}
$postings = $row['POSTINGS'] ?? [];
if (empty($postings) && $index) {
$postings = $index->unpackPostings(
$index->getPostingsString($partition,
$row['POSTINGS_OFFSET'], $row['POSTINGS_LEN'],
$term_id))[0] ?? [];
}
break;
}
if (count(self::$term_postings_cache) >=
self::MAX_REMEMBERED_LOOKUPS) {
self::$term_postings_cache = [];
}
self::$term_postings_cache[$cache_key] = $postings;
}
foreach (self::$term_postings_cache[$cache_key] as $posting) {
if ($posting['DOC_MAP_INDEX'] == $doc_map_index) {
return $posting;
}
}
return [];
}
/**
* lookupLatestVersionPage finds the newest copy of a page in an index.
* The dictionary keeps an info: key for each page address, with a row
* for each partition holding a copy, and the last row is the newest.
* A search calls this for each page it scores, so the dictionary is
* asked directly rather than through a model, which would connect to
* the databases on each call.
*
* @param string $url_hash the first 8 bytes of the page's document id,
* which hash its address
* @param string $index_name timestamp of the index to look in
* @return array|null [partition, posting] of the newest copy, or null
* if the index has no info: row for the address
*/
public static function lookupLatestVersionPage($url_hash, $index_name)
{
if (array_key_exists($url_hash, self::$urls_cache)) {
return self::$urls_cache[$url_hash];
}
if (!self::getIndex($index_name)) {
return null;
}
$lookup_start = microtime(true);
$info_term = canonicalTerm("info:" . base64Hash($url_hash));
$index = self::getIndex($index_name);
/* Only the newest row is used, and a dictionary row says only
where its postings are, so a current bundle is asked for its
newest row with the postings read. That walks the dictionary
from its newest tier and stops at the first row, rather than
reading every tier. */
$page_versions = method_exists($index, "newestRow") ?
['ROWS' => [$index->newestRow($info_term)]] :
self::getWordInfo($index_name, $info_term, 0);
AnalyticsManager::addTiming("latest copy lookups", $lookup_start);
if (!empty($page_versions['ROWS']) &&
count($page_versions['ROWS']) > 0) {
$latest_postings_info =
end($page_versions['ROWS'])['POSTINGS'] ?? null;
$latest_partition =
end($page_versions['ROWS'])['PARTITION'] ?? null;
if (is_array($latest_postings_info) &&
count($latest_postings_info) > 0) {
$latest_posting = end($latest_postings_info);
if (count(self::$urls_cache) >=
self::MAX_REMEMBERED_LOOKUPS) {
self::$urls_cache = [];
}
self::$urls_cache[$url_hash] = array($latest_partition,
$latest_posting);
return self::$urls_cache[$url_hash];
}
}
return null;
}
}
- Yioop Repo - Seekquarry