Adjust copyrights years
<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2024 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2024
* @filesource
*/
namespace seekquarry\yioop\library\index_bundle_iterators;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\IndexDocumentBundle;
use seekquarry\yioop\library\PhraseParser;
use seekquarry\yioop\library\PartitionDocumentBundle;
/** For toHexString and Yioop constants*/
require_once __DIR__."/../Utility.php";
/**
* Abstract classed used to model iterating documents indexed in
* an IndexArchiveBundle or set of such bundles.
*
*
* @author Chris Pollett
* @see IndexArchiveBundle
*/
abstract class IndexBundleIterator implements CrawlConstants
{
/**
* Default number of documents returned for each block (at most)
* @var int
*/
const RESULTS_PER_BLOCK = 200;
/**
* Estimate of the number of documents that this iterator can return
* @var int
*/
public $num_docs;
/**
*
* @var int
*/
public $total_num_docs;
/**
* The number of documents already iterated over
* @var int
*/
public $seen_docs;
/**
* The number of documents in the current block
* @var int
*/
public $count_block;
/**
* Cache of what currentDocsWithWord returns
* @var array
*/
public $pages;
/**
* Says whether the value in $this->count_block is up to date
* @var bool
*/
public $current_block_fresh;
/**
* Number of documents returned for each block (at most)
* @var int
*/
public $results_per_block = self::RESULTS_PER_BLOCK;
/**
* Returns the iterators to the first document block that it could iterate
* over
*/
abstract function reset();
/**
* Forwards the iterator one group of docs
* @param array $gen_doc_offset a generation, doc_offset pair. If set,
* the must be of greater than or equal generation, and if equal the
* next block must all have $doc_offsets larger than or equal to
* this value
*/
public abstract function advance($gen_doc_offset = null);
/**
* Gets the doc_offset and generation for the next document that
* would be return by this iterator
*
* @return mixed an array with the desired document offset
* and generation; -1 on fail
*/
public abstract function currentGenDocOffsetWithWord();
/**
* Hook function used by currentDocsWithWord to return the current block
* of docs if it is not cached
*
* @return mixed doc ids and score if there are docs left, -1 otherwise
*/
public abstract function findDocsWithWord();
/**
* This method calculates the max score value for the Doc Quality
* calculation (query independent) for an document returned by this
* iterator. Currently, for all documents we have coded a maximum
* DOC_RANK of 5 (based on code in @see computeDocRank ). This will
* likely need to be revisited in the future.
*
* @return float maximum score for document quality
*/
public function getMaxDocQualityScore()
{
return 5;
}
/**
* Returns a string representation of a plan by which the current iterator
* finds its results
*
* @return string a representation of the current iterator and its
* subiterators, useful for determining how a query will be processed
*/
public function plan()
{
$out = " ".get_class($this) . "\n===============\n";
if (isset($this->word_key)) {
$out .= "Word Key: " . L\toHexString($this->word_key)."\n";
}
if (isset($this->index_name)) {
$out .= "Index Name: " . $this->index_name . "\n";
}
if (isset($this->results_per_block)) {
$out .= "Results Per Block: " . $this->results_per_block . "\n";
}
$out .= "Number of Docs: ".$this->num_docs;
if (isset($this->index_bundle_iterator)) {
$out .= "\n " . str_replace("\n", "\n ",
$this->index_bundle_iterator->plan());
} else if (isset($this->index_bundle_iterators)) {
$out .= "\nNumber of Child Iterators: " .
$this->num_iterators . "\n";
for ($i = 0; $i < $this->num_iterators; $i++) {
$out .= "\n Child $i\n ========\n";
$out .= " ". str_replace("\n", "\n ",
$this->index_bundle_iterators[$i]->plan());
}
}
return $out;
}
/**
* Compares two arrays each containing a (generation, offset) pair.
*
* @param array $gen_doc1 first ordered pair
* @param array $gen_doc2 second ordered pair
* @param int $direction whether the comparison should be done for
* a self::ASCEDNING or a self::DESCENDING search
* @return int -1,0,1 depending on which is bigger
*/
public function genDocOffsetCmp($gen_doc1, $gen_doc2, $direction =
self::ASCENDING)
{
$diff1 = $gen_doc1[0] - $gen_doc2[0];
$diff2 = $gen_doc1[1] - $gen_doc2[1];
if ($diff2 == 0 && $diff1 == $diff2) {
return 0;
}
if ($direction == self::ASCENDING) {
return ($diff1 != 0) ? $diff1 : $diff2;
} else {
return ($diff1 != 0) ? -$diff1 : -$diff2;
}
}
/**
* Returns the direction of a IndexBundleIterator. Depending on the
* iterator could be either forward from the start of an index
* (self::ASCENDING) or backward from the end of the index
* (self::DESCENDING). For this base class, the function always returns
* self::ASCENDING, but subclasses might return different values.
* @return int either CrawlConstants::ASCENDING or
* CrawlConstants::DESCENDING
*/
public function getDirection()
{
return self::ASCENDING;
}
/**
* Gets the current block of doc ids and score associated with the
* this iterators word
*
* @return mixed doc ids and score if there are docs left, -1 otherwise
*/
public function currentDocsWithWord()
{
if ($this->current_block_fresh == true) {
return $this->pages;
}
$this->current_block_fresh = true;
return $this->findDocsWithWord();
}
/**
* Gets the summaries associated with the keys provided the keys
* can be found in the current block of docs returned by this iterator
* @param array $keys keys to try to find in the current block of returned
* results
* @return array doc summaries that match provided keys
*/
public function getCurrentDocsForKeys($keys = null)
{
if ($this->current_block_fresh == false) {
$pages = $this->currentDocsWithWord();
if (!is_array($pages)) {
return $pages;
}
} else {
$pages = & $this->pages;
}
if ($keys == null) {
if (is_array($pages)) {
return $pages;
} else {
return null;
}
}
$out_pages = [];
foreach ($keys as $doc_key) {
if (!isset($pages[$doc_key])) {
continue;
} else {
$doc_info = $pages[$doc_key];
}
if (isset($doc_info[self::SUMMARY_OFFSET]) &&
isset($doc_info[self::GENERATION])) {
$out_pages[$doc_key] = $doc_info;
}
}
return $out_pages;
}
/**
* Get the current block of doc summaries for the word iterator and advances
* the current pointer to the next block of documents. If a doc index is
* the next block must be of docs after this doc_index
*
* @param $doc_offset if set the next block must all have $doc_offsets
* equal to or larger than this value
* @return array doc summaries matching the $this->restrict_phrases
*/
public function nextDocsWithWord($doc_offset = null)
{
$doc_block = $this->getCurrentDocsForKeys();
if ($doc_block == -1 || !is_array($doc_block) ) {
return null;
}
$this->advance($doc_offset);
return $doc_block;
}
/**
* Updates the seen_docs count during an advance() call
*/
public function advanceSeenDocs()
{
if ($this->current_block_fresh != true) {
$doc_block = $this->currentDocsWithWord();
if ($doc_block == -1 || !is_array($doc_block) ) {
return;
}
}
$this->current_block_fresh = false;
$this->seen_docs += $this->count_block;
}
/**
* Sets the value of the result_per_block field. This field controls
* the maximum number of results that can be returned in one go by
* currentDocsWithWord()
*
* @param int $num the maximum number of results that can be returned by
* a block
*/
public function setResultsPerBlock($num)
{
$this->results_per_block = $num;
}
/**
* @param string $doc_key
* @param int $doc_map_index
* @param int $num_seen_partitions
* @param int $number_of_partitions
* @param int $num_doc_keys
* @param float $avg_items_per_partition
* @param int $max_items_per_partition
* @param array $ranking_factors
* @param bool $is_ascending
*/
public function computeDocRank($doc_key, $doc_map_index = 1,
$num_seen_partitions = 0, $number_of_partitions = 1,
$num_doc_keys = PartitionDocumentBundle::MAX_ITEMS_PER_FILE,
$avg_items_per_partition = PartitionDocumentBundle::MAX_ITEMS_PER_FILE,
$max_items_per_partition = PartitionDocumentBundle::MAX_ITEMS_PER_FILE,
$ranking_factors = [], $is_ascending = true)
{
/*
DOC_RANK calculate is a computes a document quality measure
either based on time item was added (freshness) or a
sum of signals (how early or late it was added to index),
whether the url was a CLD or HOST, whether page was a wiki
page, and the number of slashes in the url path
*/
$cld_bonus = $ranking_factors["CLD_URL_BONUS"] ?? 1;
$host_bonus = $ranking_factors["HOST_URL_BONUS"] ?? 1;
$wiki_bonus = $ranking_factors["WIKI_BONUS"] ?? 1;
$num_slashes_bonus = $ranking_factors["NUM_SLASHES_BONUS"] ?? 1;
$max_pre_rank_and_bonuses = $cld_bonus + $host_bonus +
$wiki_bonus + $wiki_bonus + 1;
$last_partition_pos = ($is_ascending) ?
$num_doc_keys - $doc_map_index :
$doc_map_index;
$remaining_partitions = ($is_ascending) ?
$number_of_partitions - $num_seen_partitions :
$num_seen_partitions - 1;
$rank_and_bonuses = ($remaining_partitions *
$avg_items_per_partition)/
(($number_of_partitions + 1) *
($avg_items_per_partition + 1)) +
$last_partition_pos / $max_items_per_partition;
if (IndexDocumentBundle::isAHostDocId($doc_key)) {
$rank_and_bonuses +=
(IndexDocumentBundle::isACldDocId($doc_key)) ?
$cld_bonus : $host_bonus;
}
/**
* For backward compatibility: new bonuses should only be added
* for doc_ids following the new letter_code format. Since all
* old formats use letters (b, t, etc.) to denote the doc
* type, the ASCII values for these letters are all > 96 (i.e.,
* bits 6 and 7 of the doc_id's 9th byte are both true).
* Since all new letter_code formats use bits 4, 5, 6, 7 to
* represent the doc type as int values mapped between 0-8,
* there is no value in a doc_id's 9th byte that can have both
* bits 6 and 7 set to true.
* This difference can be used to check whether $doc_key follows
* the old or new letter_code format.
*/
$doc_id_format = ord($doc_key[
IndexDocumentBundle::DOCID_PART_LEN << 1] ?? 0) & 96;
if ($doc_id_format != 96) {
if (IndexDocumentBundle::isAWikipediaPage($doc_key)) {
$rank_and_bonuses += $wiki_bonus;
}
$rank_and_bonuses += $num_slashes_bonus /
(IndexDocumentBundle::findNumSlashes($doc_key) + 1);
}
return $rank_and_bonuses;
}
}