<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\index_bundle_iterators;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\IndexManager;
use seekquarry\yioop\library\storage_formats\IndexShard;
/**
* DocIterator walks all the documents of an IndexDocumentBundle, with
* any words they hold, partition by partition, in the order of each
* partition's document map. NegationIterator walks it beside the walk of
* a word to find the documents without the word, and a query for all the
* documents of an index walks it alone.
*
* A place in the walk is a partition and a row number of that partition's
* document map. This is also the DOC_MAP_INDEX a WordIterator gives for the
* document, so the two walks can be compared and moved to the same place.
* The rows are read through the bundle's doc_map_tools, which keeps a
* document map and the positions of its rows once read, so the walks of
* one index share that work.
*
* @author Chris Pollett
*/
class DocIterator extends IndexBundleIterator
{
/**
* next_offset is the row number of the current partition's document map
* the walk moves to on advance: the row after the last row the current
* block read.
* @var int
*/
public $next_offset;
/**
* last_offset is the number of the last row of the current partition's
* document map, -1 when it has no rows.
* @var int
*/
public $last_offset;
/**
* current_offset is the row number of the current partition's document
* map the walk is at.
* @var int
*/
public $current_offset;
/**
* num_generations is the number of partitions of the bundle: the number
* of the partition being saved to, plus one.
* @var int
*/
public $num_generations;
/**
* current_generation is the number of the partition the walk is in.
* @var int
*/
public $current_generation;
/**
* HOST_KEY_POS is where the hash of a document's host starts in its
* document key, one byte past the byte that gives the document's type.
*/
const HOST_KEY_POS = 17;
/**
* KEY_LEN is the length in bytes of the hash of a document's host.
*/
const KEY_LEN = 8;
/**
* __construct sets up a walk of all the documents of the bundle
* $index_name and moves it to the walk's first document.
*
* @param string $index_name the name of the IndexDocumentBundle to
* walk, which is the timestamp of its crawl
* @param SearchfiltersModel $filter model that says which hosts are
* kept out of search results; their documents are left out of the
* blocks of the walk
* @param int $results_per_block the most documents a block of the walk
* gives at once
* @param int $direction self::ASCENDING to walk from the first
* document added to the last, self::DESCENDING to walk from the
* last to the first
* @param array $ranking_factors weights computeDocRank uses to score
* each document's rank
*/
public function __construct(public $index_name, public $filter = null,
$results_per_block = IndexBundleIterator::RESULTS_PER_BLOCK,
public $direction = self::ASCENDING,
public $ranking_factors = [])
{
$index = IndexManager::getIndex($index_name, $direction);
$this->results_per_block = $results_per_block;
if (empty($index)) {
$this->num_docs = 0;
$this->num_generations = 0;
} else {
$info = $index->getArchiveInfo($index->dir_name);
$this->num_docs = ($info['COUNT'] ?? 0) + ($info['ACTIVE'] ?? 0);
$this->num_generations = $info['SAVE_PARTITION'] + 1;
}
$this->reset();
}
/**
* reset moves the walk back to its first document: the first row of
* the first partition for an ascending walk, the last row of the last
* partition for a descending one.
*/
public function reset()
{
$is_ascending = ($this->direction == self::ASCENDING);
$this->count_block = 0;
$this->seen_docs = 0;
$this->current_block_fresh = false;
$this->moveToGeneration(($is_ascending) ? 0 :
$this->num_generations - 1);
}
/**
* docMapFilename gives the path of the document map of partition
* $generation of the bundle.
*
* @param int $generation number of the partition
* @return string path of the partition's document map file
*/
public function docMapFilename($generation)
{
$index = IndexManager::getIndex($this->index_name);
return $index->getPartitionBaseFolder($generation) . "/" .
IndexDocumentBundle::DOC_MAP_FILENAME;
}
/**
* numRows gives the number of rows of the document map of partition
* $generation: one more than the number of \xFF (byte 255) bytes
* between its rows, or 0 when the map is empty or missing.
*
* @param int $generation number of the partition
* @return int the number of rows of its document map
*/
public function numRows($generation)
{
$index = IndexManager::getIndex($this->index_name);
if (empty($index)) {
return 0;
}
$doc_map_tools = $index->doc_map_tools;
$filename = $this->docMapFilename($generation);
if ($doc_map_tools->load($filename, $doc_map_tools::AS_STRING_MODE,
true) === "") {
return 0;
}
return count($doc_map_tools->getEntryMarkers($filename)) + 1;
}
/**
* moveToGeneration moves the walk to the start of partition
* $generation: its first row ascending, its last row descending. A
* partition without rows is passed over for the next one in the walk's
* direction. When no partition is left, current_generation ends up
* past the last partition ascending, or below 0 descending.
*
* @param int $generation number of the partition to move to
*/
public function moveToGeneration($generation)
{
$is_ascending = ($this->direction == self::ASCENDING);
$step = ($is_ascending) ? 1 : -1;
$this->last_offset = -1;
while ($generation >= 0 && $generation < $this->num_generations) {
$this->last_offset = $this->numRows($generation) - 1;
if ($this->last_offset >= 0) {
break;
}
$generation += $step;
}
$this->current_generation = $generation;
$this->current_offset = ($is_ascending) ? 0 :
max($this->last_offset, 0);
$this->next_offset = $this->current_offset;
}
/**
* isDone says if the walk has passed its last document.
*
* @return bool true when no partition is left to walk
*/
public function isDone()
{
return $this->current_generation < 0 ||
$this->current_generation >= $this->num_generations;
}
/**
* findDocsWithWord reads the next block of the walk: from the current
* row, at most results_per_block documents of the current partition in
* the walk's direction. A row that is not a document, such as a link,
* is passed over, and a document whose host the filter keeps out of
* results is left out of the block. currentDocsWithWord calls it when
* the block it holds is used up.
*
* @return mixed array of document key => document data for the block,
* which is empty when every document of it was left out; -1 when
* the walk has passed its last document
*/
public function findDocsWithWord()
{
if ($this->isDone()) {
return -1;
}
$is_ascending = ($this->direction == self::ASCENDING);
$step = ($is_ascending) ? 1 : -1;
$index = IndexManager::getIndex($this->index_name);
$doc_map_tools = $index->doc_map_tools;
$filename = $this->docMapFilename($this->current_generation);
$docid_len = IndexDocumentBundle::DOCID_LEN;
$termsfilter_len = IndexDocumentBundle::TERMSFILTER_LEN;
$results = [];
$num_docs_so_far = 0;
$row = $this->current_offset;
while ($num_docs_so_far < $this->results_per_block &&
$row >= 0 && $row <= $this->last_offset) {
$entry = $doc_map_tools->findEntryAtIndexTableName($filename,
$row);
$doc_row = $row;
$row += $step;
$doc_key = substr($entry, 0, $docid_len);
if (strlen($entry) <= $docid_len ||
IndexDocumentBundle::isType($doc_key, "link")) {
continue;
}
$num_docs_so_far++;
/* a row of a newer index has a 't' and a bloom filter of the
document's terms between its key and its values */
$values = (strlen($entry) > $docid_len + $termsfilter_len &&
$entry[$docid_len] == 't') ?
substr($entry, $docid_len + $termsfilter_len + 1) :
substr($entry, $docid_len);
$doc_info = $doc_map_tools->unpack($values);
$host_key = substr($doc_key, self::HOST_KEY_POS, self::KEY_LEN);
if (empty($doc_info) || count($doc_info) < 2 ||
(!empty($this->filter) &&
$this->filter->isFiltered($host_key))) {
continue;
}
$item = [self::GENERATION => $this->current_generation];
$num_rows = $this->last_offset + 1;
$item[self::DOC_RANK] = $this->computeDocRank($doc_key,
$doc_row, $this->current_generation,
$this->num_generations, $num_rows, $num_rows, $num_rows,
$this->ranking_factors, $is_ascending);
list($item[self::DOC_LEN], ) =
array_values(array_shift($doc_info));
$item[self::SCORE] = $item[self::DOC_RANK];
list(, $num_description_scores) =
array_values(array_shift($doc_info));
$item[self::DESCRIPTION_SCORES] = array_slice($doc_info, 0,
$num_description_scores);
$item[self::USER_RANKS] = array_slice($doc_info,
$num_description_scores);
$item[self::IS_DOC] = true;
$item[self::KEY] = $doc_key;
list(, $item[self::HASH], $item[self::INLINKS]) =
str_split($doc_key, IndexShard::DOC_KEY_LEN);
$item[self::CRAWL_TIME] = $this->index_name;
$results[$doc_key] = $item;
}
$this->next_offset = $row;
$this->count_block = count($results);
$this->pages = $results;
return $results;
}
/**
* advanceSeenDocs adds the documents of the current block to
* seen_docs, reading the block first if it has not been read, which
* also sets next_offset to the row after the block.
*/
public function advanceSeenDocs()
{
if ($this->current_block_fresh != true) {
$this->currentDocsWithWord();
}
$this->current_block_fresh = false;
$this->seen_docs += $this->count_block;
}
/**
* advance moves the walk past its current block, or to the first
* document at or past a given place in the walk's direction.
*
* @param array $gen_doc_offset a pair of a partition number and a row
* number of its document map. If given, the walk moves to that
* row, or to the next partition in the walk's direction when that
* partition has no such row. A place the walk has already passed
* leaves it where it is.
*/
public function advance($gen_doc_offset = null)
{
if ($this->isDone()) {
return;
}
$is_ascending = ($this->direction == self::ASCENDING);
if ($gen_doc_offset === null) {
$this->advanceSeenDocs();
$this->current_offset = $this->next_offset;
} else {
$current = [$this->current_generation, $this->current_offset];
if ($this->genDocOffsetCmp($current, $gen_doc_offset,
$this->direction) >= 0) {
return;
}
$this->current_block_fresh = false;
if ($this->current_generation != $gen_doc_offset[0]) {
$this->moveToGeneration($gen_doc_offset[0]);
}
if ($this->current_generation == $gen_doc_offset[0]) {
$this->current_offset = ($is_ascending) ?
$gen_doc_offset[1] :
min($gen_doc_offset[1], $this->last_offset);
}
}
$this->next_offset = $this->current_offset;
if ($this->current_offset < 0 ||
$this->current_offset > $this->last_offset) {
$this->moveToGeneration($this->current_generation +
(($is_ascending) ? 1 : -1));
}
}
/**
* currentGenDocOffsetWithWord gives the place of the walk: its
* partition and the row number of its document map.
*
* @return mixed a pair of the partition number and the row number, or
* -1 when the walk has passed its last document
*/
public function currentGenDocOffsetWithWord()
{
if ($this->isDone()) {
return -1;
}
return [$this->current_generation, $this->current_offset];
}
}
- Yioop Repo - Seekquarry