<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\models;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\AnalyticsManager;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\IndexManager;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\storage_formats\IndexArchiveBundle;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\index_bundle_iterators as I;
use seekquarry\yioop\library\processors\ImageProcessor;
/**
* logging is done during crawl not through web,
* so it will not be used in the phrase model
*/
if (!C\nsdefined("POST_PROCESSING") && !C\nsdefined("LOG_TO_FILES")) {
C\nsdefine("LOG_TO_FILES", false);
}
/**
* PhraseModel finds and formats the results of a search. It parses a
* query into the words and meta words to look up, builds the iterators
* that read the index, fetches the summaries of the results, and scores
* and orders them.
*
* @author Chris Pollett
*/
class PhraseModel extends ParallelModel
{
/**
* $additional_meta_words maps each extra meta word, one that an
* indexing plugin adds, to the length of the snippet a result gets
* when a query uses that meta word. index.php usually sets it.
*
* @var array
*/
public $additional_meta_words;
/**
* $query_info holds the statistics about the current query, its plan
* and the time each step took, shown when query info is on
* @var array
*/
public $query_info;
/**
* $programming_language_map maps a file extension to the programming
* language a file with that extension is written in
* @var string
*/
public $programming_language_map;
/**
* $program_indicator is true when a query asks for source code files
* @var string
*/
public $program_indicator;
/**
* Length of info hash record phrase
*/
const INFO_HASH_LEN = 16;
/**
* JOINED_WORDS_BONUS_SHARE is the part of the host keyword bonus, or
* of the path keyword bonus, that a result gets when the words of a
* query, joined with nothing, a hyphen or an underscore, are part of
* its host or its path. tik tok joined is tiktok, part of the host
* www.tiktok.com. The share is a little under one, so such a match
* counts slightly less than the words typed as they stand.
*/
const JOINED_WORDS_BONUS_SHARE = 0.9;
/**
* EXACT_MATCH_FACTOR is what addExactMatchScore multiplies a query
* word's share of a result's relevance by when the result's title or
* summary holds the word exactly as it was typed, rather than only a
* word with the same stem.
*/
const EXACT_MATCH_FACTOR = 2;
/**
* MMR_PAGES is how many pages of results diversify reorders. The
* reordered window is this many times the number of results on a
* page, so every page inside it is cut from the same reordered list.
*/
const MMR_PAGES = 3;
/**
* UNSPACED_SCRIPTS matches a character of a script written without
* spaces between words, such as Chinese, Japanese or Thai. A typed word
* holding one is looked for anywhere in a result, not as a whole word.
*/
const UNSPACED_SCRIPTS = '/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}' .
'\p{Lao}\p{Khmer}\p{Myanmar}]/u';
/**
* __construct makes a PhraseModel for the named database and sets up
* the map from file extension to programming language.
*
* @param string $db_name the name of the database for the search engine
* @param bool $connect whether to connect to the database by default
* after making the datasource class
* @param object $web_site the WebSite serving pages, when Yioop serves
* its own, so file reads and writes go through its cache
*/
public function __construct($db_name = C\DB_NAME, $connect = true,
$web_site = null)
{
parent::__construct($db_name, $connect, $web_site);
$this->programming_language_map = ['java' => 'java', 'py' => 'py'];
$this->program_indicator = false;
}
/**
* rewriteMixQuery rewrites a query on a crawl mix as a query on the
* crawls the mix is built from.
*
* @param string $query the original before a rewrite
* @param object $mix a mix object saying how the mix is built out of crawls
* @return string a rewritten query in terms of crawls
*/
public function rewriteMixQuery($query, $mix)
{
$disjunct_phrases = explode("|", $query);
$rewrite = "";
if (isset($mix['FRAGMENTS'])) {
foreach ($mix['FRAGMENTS'] as $fragment) {
$pipe = "";
foreach ($disjunct_phrases as $disjunct) {
$rewrite .= $pipe;
$pipe = ' | ';
$disjunct_string = $disjunct;
$base_weight = 1;
$pattern = "/(\s)(index:(\S)+)/";
preg_match_all($pattern, $query, $matches);
if (isset($matches[2][0])) {
$rewrite .= $disjunct;
continue;
}
$pattern = "/(\s)(i:(\S)+)/";
preg_match_all($pattern, $query, $matches);
if (isset($matches[2][0])) {
$rewrite .= $disjunct;
continue;
}
$pattern = "/(\s)(weight:(\S)+)/";
preg_match_all($pattern, $query, $matches);
if (isset($matches[2][0])) {
$base_weight = substr($matches[2][0],
strlen("weight:"));
$disjunct_string =
preg_replace($pattern, "", $disjunct_string);
}
$pattern = "/(\s)(w:(\S)+)/";
preg_match_all($pattern, $query, $matches);
if (isset($matches[2][0])) {
$base_weight = substr($matches[2][0], strlen("w:"));
$disjunct_string =
preg_replace($pattern, "", $disjunct_string);
}
$pipe2 = "";
if (isset($fragment['COMPONENTS'])) {
$start_disjunct_string = $disjunct_string;
foreach ($fragment['COMPONENTS'] as $component) {
$disjunct_string = $start_disjunct_string;
if (isset($component['KEYWORDS'])) {
$disjunct_string .= " ".$component['KEYWORDS'];
}
$sign = ($component['DIRECTION'] ==
self::ASCENDING) ? "" : "-";
$stamp = ($component['CRAWL_TIMESTAMP'] <= 1) ?"" :
" " . $sign . "i:" .
$component['CRAWL_TIMESTAMP'];
$rewrite .= $pipe2.$disjunct_string." w:".
($component['WEIGHT'] * $base_weight).$stamp;
$pipe2 = ' | ';
}
}
}
$num_results = (isset($fragment['RESULT_BOUND']) &&
$fragment['RESULT_BOUND'] > 1) ?
$fragment['RESULT_BOUND'] : 1;
$rewrite .= " #$num_results# ";
}
}
return $rewrite;
}
/**
* getPhrasePageResults returns the formatted summaries of the
* documents that match a query phrase.
*
* @param string $input_phrase the phrase to try to match
* @param int $low return results beginning with the $low document
* @param int $results_per_page how many results to return
* @param bool $format whether to highlight in the returned summaries the
* matched text
* @param SearchfiltersModel $filter Model responsible for keeping
* track of edited and deleted search results
* @param bool $use_cache_if_allowed if true and USE_CACHE is true then
* an attempt will be made to look up the results in
* the file cache. Otherwise, items will be recomputed
* and then potentially restored in cache
* @param int $raw ($raw == 0) normal grouping, ($raw == 1)
* no grouping done on data also no summaries returned (only lookup
* info), $raw > 1 return summaries but no grouping
* @param array $queue_servers a list of urls of yioop machines which might
* be used during lookup
* @param bool $guess_semantics whether to do query rewriting before lookup
* @param int $save_timestamp if this timestamp is nonzero, then save
* iterate position, so can resume on future queries that make
* use of the timestamp
* @param array $ranking_factors field say how url, keywords, and
* title words should influence relevance and doc rank calculations
* @return array an array of summary data
*/
public function getPhrasePageResults(
$input_phrase, $low = 0, $results_per_page = C\NUM_RESULTS_PER_PAGE,
$format = true, $filter = null, $use_cache_if_allowed = true,
$raw = 0, $queue_servers = [], $guess_semantics = true,
$save_timestamp = 0, $ranking_factors = [])
{
if (C\p('QUERY_STATISTICS')) {
$indent= " ";
$in2 = $indent . $indent;
$in3 = $in2 . $indent;
$prs_cnt = 0;
$dis_cnt = 0;
$this->query_info = [];
$this->query_info['QUERY'] =
"<b>PHRASE QUERY</b>: ".$input_phrase."<br>";
$start_time = microtime(true);
}
$results = null;
$answer_score_map = [];
$word_structs = [];
/*
this is a quick and dirty parsing and will usually work,
exceptions would be # or | in quotes or if someone tried
to escape |.
First we split into presentation elements then we split by
disjuncts
*/
$presentation_parts = preg_split('/#(\d)+#/',
$input_phrase, -1, PREG_SPLIT_DELIM_CAPTURE);
$count = 0;
$presentation_parts = array_chunk($presentation_parts, 2);
$num_parts = count($presentation_parts);
$can_use_query_map = ($num_parts == 1);
$query_parts = [];
$last_part = null;
for ($i = 0; $i < $num_parts; $i++) {
if (isset($presentation_parts[$i][0]) &&
($trimmed = trim($presentation_parts[$i][0])) != "") {
$to_return = (isset($presentation_parts[$i][1])) ?
$presentation_parts[$i][1]: 1;
$query_parts[$trimmed][] =
[$count, $to_return];
$last_part = $trimmed;
if (isset($presentation_parts[$i][1])) {
$count += $presentation_parts[$i][1];
} else {
$count++;
}
}
}
$results_high = $low + $results_per_page;
$num_phrases = count($query_parts);
if ($num_phrases > 0) {
$num_last_parts = count($query_parts[$last_part]);
if ($query_parts[$last_part][$num_last_parts - 1][0] +
$query_parts[$last_part][$num_last_parts - 1][1] < $low) {
$query_parts[$last_part][$num_last_parts - 1][1] =
$results_high;
}
}
$qpart = 0;
if (is_string($save_timestamp)) {
$save_parts = explode("-", $save_timestamp);
if (isset($save_parts[1])) {
$qpart = intval($save_parts[1]);
$save_timestamp = intval($save_parts[0]);
}
}
$orig_stimestamp = $save_timestamp;
$network = false;
if ($queue_servers != [] &&
!$this->isSingleLocalhost($queue_servers)) {
$network = true;
}
foreach ($query_parts as $phrase => $pre_result_bounds) {
$phrase_high = $pre_result_bounds[0][1];
$result_bounds = [];
$num_bounds = 0;
foreach ($pre_result_bounds as $bound) {
if ($bound[0] > $results_high) {
break;
}
//rest of presentation after what we'll return so break
$phrase_high = $bound[0] + $bound[1];
if ($phrase_high < $low) {
continue;
}
// this part of presentation is before what we'll return so skip
$result_bounds[] = $bound;
$num_bounds++;
}
if ($num_bounds == 0) {
continue;
}
$is_last_part = ($phrase == $last_part);
if ($is_last_part &&
$result_bounds[$num_bounds - 1][0] +
$result_bounds[$num_bounds - 1][1] < $results_high) {
$result_bounds[$num_bounds - 1][1] = $results_high -
$result_bounds[$num_bounds - 1][0];
}
$phrase_num = max(min($phrase_high, $results_high), $results_high) -
$low;
$word_structs = [];
$format_words = [];
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .= $indent .
"<b>Presentation $prs_cnt:</b><br>";
$this->query_info['QUERY'] .= "$in2<i>Low</i>:".
$result_bounds[0][0]."<br>";
$this->query_info['QUERY'] .= $in2 .
"<i>High</i>: " . $result_bounds[0][1] . "<br>";
$prs_cnt++;
}
$cache_results = false;
if (mb_strlen($phrase) > self::INFO_HASH_LEN ||
!in_array(mb_substr($phrase, 0, 5), ["info:", "link:"])) {
$phrase = mb_strtolower($phrase);
}
if (!empty($_SERVER["USE_CACHE"]) &&
$save_timestamp == "" && $use_cache_if_allowed && !$network) {
$cache_results = self::$cache->get($phrase . $this->index_name);
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .=
"$in2<b>Parse done by Cache Lookup</b><br>";
if (!empty(self::$cache->cache_file) ) {
$this->query_info['QUERY'] .=
"$in2<b>Cache File Used: </b>" .
self::$cache->cache_file . "<br>";
}
}
}
$typed_words = array_values(array_filter(preg_split("/\s+/u",
trim($phrase)), fn($word) => !str_contains($word, ":")));
if ($cache_results) {
list($word_structs, $format_words) = $cache_results;
} else {
$original_has_disjuncts = (str_contains($phrase, "|"));
$original_phrase = $phrase;
/* A site's name typed as words, such as ars technica, is
like a hyphenated entity: its words joined, arstechnica,
are the host word only the site's root page is indexed
under. The joined word is always looked up as well; a
part that finds nothing costs nothing. When it is a word
of the index, the words name a site, so they are kept
together, as no:rewrite keeps them, rather than split into
a part per word, where a word as common as ar, the stem
of ars, outranked the site. */
$plain_words = $typed_words;
$joined_word = mb_strtolower(implode("", $plain_words));
$site_words = !$original_has_disjuncts &&
count($plain_words) >= 2 && count($plain_words) <= 3 &&
preg_match('/^\p{L}+$/u', $joined_word);
$joined_index = preg_match('/\bi:(\S+)/u', $original_phrase,
$index_match) ? $index_match[1] : $this->index_name;
$names_site = $site_words &&
!empty(L\IndexManager::getIndex($joined_index)) &&
!empty(L\IndexManager::getWordInfo($joined_index,
L\canonicalTerm($joined_word))["ROWS"] ?? []);
if (!C\USE_CONJUNCTIVE_QUERY && !$names_site) {
$phrase = $this->rewriteAsDisjunctiveQuery(
$phrase, $guess_semantics);
}
if ($site_words) {
/* The joined word stands for every word typed, so its
part is weighted by how many words it joins. */
$phrase .= " | " . trim($joined_word . " w:" .
count($plain_words) . " " . implode(" ", array_diff(
preg_split("/\s+/u", trim($original_phrase)),
$plain_words)));
}
$disjunct_phrases = explode("|", $phrase);
$can_use_query_map = $can_use_query_map &&
!$original_has_disjuncts && !empty($filter) &&
$low == 0;
$query_map_results = [];
$query_map_urls = [];
if ($can_use_query_map) {
$query_map = $filter->getQueryMap($original_phrase,
L\guessLocaleFromString($original_phrase));
$map_cnt = 0;
foreach ($query_map as $map_url) {
$map_parts = explode("###", $map_url);
$map_result[self::URL] = $map_parts[0];
$map_result[self::PINNED] = true;
$map_result[self::SCORE] = 0;
$map_result[self::DOC_RANK] = floatval(
$map_parts[1] ?? 0);
$map_result[self::RELEVANCE] = 0;
$query_map_results[] = $map_result;
$query_map_urls[$map_parts[0]] = $map_cnt;
$map_cnt++;
}
}
if ($guess_semantics) {
$repeat_check = [];
$phrase = "";
$delim = " ";
foreach ($disjunct_phrases as $disjunct) {
$check = trim($disjunct);
if (isset($repeat_check[$check])) {
continue;
}
$repeat_check[$check] = true;
$phrase .= $delim . $this->guessSemantics(" ".
$disjunct);
$delim = " | ";
}
$disjunct_phrases = explode("|", $phrase);
}
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .=
"$in2<b>Guessed Semantics</b>:<br>$in2$phrase<br>";
}
if (!$network) {
foreach ($disjunct_phrases as $disjunct) {
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY']
.="$in2<b>Disjunct $dis_cnt:" . "</b><br>";
$dis_cnt++;
}
list($word_struct, $found_format_words) =
$this->parseWordStructConjunctiveQuery(
$disjunct, C\USE_CONJUNCTIVE_QUERY);
$format_words = array_merge($format_words,
$found_format_words);
if ($word_struct != null) {
$word_structs[] = $word_struct;
}
/* The field keys make a disjunct of their own, the
first tier of a tiered index. UnionIterator
reads the shortest part first, so the pages that
name the words in their host, title or path are
scored before the full lists fill the room
left. */
if (!empty($word_struct["FIELD_KEYS"])) {
$word_structs[] = array_merge($word_struct,
["KEYS" => $word_struct["FIELD_KEYS"],
"FIELD_PART" => true]);
}
}
if (!empty($_SERVER["USE_CACHE"]) &&
$save_timestamp == "") {
self::$cache->set($phrase .
$this->index_name, [$word_structs, $format_words]);
}
}
}
if ($format) {
if (count($format_words) == 0) {
$format_words = null;
}
} else {
$format_words = null;
}
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .=
"$in2<b>Presentation Parse time</b>: " .
L\changeInMicrotime($start_time)."<br>";
}
if ($orig_stimestamp > 0) {
$save_timestamp_name = "$orig_stimestamp-$qpart";
} else {
$save_timestamp_name = "";
}
$out_results = $this->getSummariesByHash($word_structs,
$low, $phrase_num, $filter, $use_cache_if_allowed, $raw,
$queue_servers, $phrase, $save_timestamp_name,
$format_words, $ranking_factors, $typed_words);
if (isset($out_results['PAGES']) &&
count($out_results['PAGES']) != 0) {
$out_count = 0;
foreach ($result_bounds as $bound) {
for ($i = $bound[0];
$i < min($bound[0] + $bound[1], $results_high);
$i++) {
if (isset($out_results['PAGES'][$out_count])) {
if (empty($original_has_disjuncts) &&
isset($out_results['PAGES'][$out_count]
[self::QUESTION_ANSWERS])) {
$triplets_with_answer =
$out_results['PAGES'][$out_count]
[self::QUESTION_ANSWERS];
$question_phrases =
explode("|",
preg_replace("/\S+\:\S+/u", "", $phrase));
foreach ($question_phrases as $question_phrase){
$question = trim(
PhraseParser::stemCharGramSegment(
$question_phrase,
L\guessLocaleFromString(
$question_phrase), true));
if (isset(
$triplets_with_answer[$question])) {
$out_results['PAGES'][$out_count][
'ANSWER']=
$triplets_with_answer[$question];
$answer =
$triplets_with_answer[$question];
if (array_key_exists(
$answer, $answer_score_map)) {
$answer_score_map[$answer] +=
$out_results['PAGES'][
$out_count][self::SCORE];
} else {
$answer_score_map[$answer] =
$out_results['PAGES'][$out_count]
[self::SCORE];
}
break;
}
}
}
$results['PAGES'][$i] =
$out_results['PAGES'][$out_count];
$out_count++;
}
}
}
if ($is_last_part && isset($out_results['TOTAL_ROWS'])) {
$total_rows = $out_results['TOTAL_ROWS'];
}
}
$qpart++;
}
if (C\p('QUERY_STATISTICS')) {
$format_time = microtime(true);
}
if (isset($out_results['SAVE_POINT'])) {
/*
out_result of last used to back-fill earlier ones that are done
so on crawl mix archive crawls only look at last
*/
$results['SAVE_POINT'] = $out_results['SAVE_POINT'];
}
if (isset($results['PAGES'])) {
ksort($results['PAGES']);
$results["PAGES"] = array_values($results["PAGES"]);
}
if (isset($out_results['HARD_QUERY'])) {
$results['HARD_QUERY'] = $out_results['HARD_QUERY'];
}
if (!is_array($results) || count($results) == 0) {
$results = null;
}
if ($results == null) {
$total_rows = 0;
$results['TOTAL_ROWS'] = 0;
}
$query_map_count = 0;
if (!empty($query_map_results)) {
$results['PAGES'] = $results['PAGES'] ?? [];
$original_num_pages = count($results['PAGES']);
$num_pages = count($results['PAGES']);
for ($p_index = 0; $p_index < $num_pages; $p_index++) {
$page = $results['PAGES'][$p_index];
if (!empty($page[self::URL]) &&
isset($query_map_urls[$page[self::URL]])) {
$qm_index = $query_map_urls[$page[self::URL]];
$qm_result = $query_map_results[$qm_index];
foreach ($page as $field => $value) {
if (in_array($field,
[self::DOC_RANK, self::RELEVANCE])) {
$qm_result[$field] += $page[$field];
} else {
$qm_result[$field] = $page[$field];
}
}
$qm_result[self::SCORE] = $qm_result[self::DOC_RANK] +
$qm_result[self::RELEVANCE];
$query_map_results[$qm_index] = $qm_result;
unset($results['PAGES'][$p_index]);
}
}
$results['PAGES'] = array_merge(
$query_map_results, $results['PAGES']);
$query_map_count = count($results['PAGES']) - $original_num_pages;
}
if (isset($total_rows)) {
$results['TOTAL_ROWS'] = $total_rows + $query_map_count;
} elseif (isset($results['PAGES'])) {
$results['TOTAL_ROWS'] = count($results['PAGES']) +
$query_map_count;
}
if ($raw == 0 && isset($results['TOTAL_ROWS']) &&
$results['TOTAL_ROWS'] > 0) {
if (!empty($filter)) {
$results = $filter->incorporateEditedPageResults($results,
$format_words);
}
if (!empty($answer_score_map)) {
arsort($answer_score_map);
reset($answer_score_map);
$results['BEST_ANSWER'] = key($answer_score_map);
}
}
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .= "<b>Format Time</b>: ".
L\changeInMicrotime($format_time)."<br>";
$this->query_info['ELAPSED_TIME'] =
L\changeInMicrotime($start_time);
$this->db->total_time += $this->query_info['ELAPSED_TIME'];
$this->db->query_log[] = $this->query_info;
}
return $results;
}
/**
* rewriteAsDisjunctiveQuery splits a search string into the phrases
* joined by | that are each looked up in the current index.
*
* @param string $phrase entered by user
* @return string search phrase rewritten as disjunctive with | separators
*/
public function rewriteAsDisjunctiveQuery($phrase)
{
if (($new_phrase =
preg_replace('/no\:rewrite/', ' ', $phrase)) != $phrase) {
return $new_phrase;
}
$phrase = preg_replace('/\s+/', ' ', $phrase);
$locale_tag = L\guessLocaleFromString($phrase);
$search_terms = [];
/*
Extracts all terms specified within quotes into a single
conjunctive query
*/
$phrase = str_replace([""", '"'], """, $phrase);
preg_match_all('/"(.*?)"/', $phrase, $matches);
foreach ($matches[0] as $match) {
$phrase = str_replace($match, '', $phrase);
$search_terms[] = $match;
}
$phrase = trim($phrase);
$split_phrase = explode(" ", $phrase);
$num_phrase_parts = count($split_phrase);
$split_terms = [];
$s = '';
/* Extracts all terms separated by '_and' into a single conjunctive
query
*/
for ($i = 0; $i < $num_phrase_parts; $i++) {
while ($i < $num_phrase_parts - 1 &&
in_array($split_phrase[$i + 1], ['_', '_a', '_and'])) {
$s .= $split_phrase[$i] . " ";
$i += 2;
}
$s .= $split_phrase[$i];
$split_terms[] = $s;
$s = '';
}
$special_words = [];
// Extracts any meta tags specified in the search phrase
$meta_pattern = '/(' . implode('|', array_map('preg_quote',
PhraseParser::$meta_words_list)) . ')/';
$disallowed_pattern = '/^\-/';
foreach ($split_phrase as $word) {
if (preg_match($meta_pattern, $word) ||
preg_match($disallowed_pattern, $word)) {
$special_words[] = $word;
}
}
foreach ($split_terms as $term) {
if (!in_array($term, $special_words)) {
$term = implode(" ",
PhraseParser::segmentSegment($term, $locale_tag));
$search_terms[] = $term;
}
}
$phrase = '';
foreach ($search_terms as $search_term) {
if (!str_contains($search_term, ' ') &&
!str_contains($search_term, '"')) {
$phrase .= $search_term . ' ';
}
}
PhraseParser::hyphenateEntities($phrase, $locale_tag);
$new_terms = explode(" ", $phrase);
foreach ($new_terms as $new_term) {
$new_term = trim($new_term);
if (!in_array($new_term, $search_terms) && strlen($new_term) > 0) {
$search_terms[] = $new_term;
}
}
$num_search_terms = count($search_terms);
for ($i = 0; $i < $num_search_terms; $i++) {
foreach ($special_words as $add_word) {
$search_terms[$i] .= " " . $add_word;
}
}
if (count($search_terms) == 0 && count($special_words) > 0) {
$search_terms = [implode(' ', $special_words)];
}
return implode(" | ", $search_terms);
}
/**
* parseWordStructConjunctiveQuery parses a phrase whose words must all
* be found into a struct. The struct holds the keys of the words
* searched for, the allowed and disallowed phrases, the weight of the
* phrase's results, and which archive to use.
*
* @param string &$phrase string to extract struct from, if the phrase
* semantics is guessed or an if condition is processed the value of
* phrase will be altered. (Helps for feeding to network queries)
* @return array struct representing the conjunctive query
*/
public function parseWordStructConjunctiveQuery(&$phrase)
{
$query = $phrase;
$indent= " ";
$in2 = $indent . $indent;
$in3 = $in2 . $indent;
$in4 = $in2 . $in2;
$phrase = " " . $phrase;
$phrase = $this->parseIfConditions($phrase);
$phrase_string = $phrase;
list($found_metas, $disallow_phrases,
$phrase_string, $query_string, $index_name, $weight,
$locale_tag) =
$this->extractMetaWordInfo($phrase);
/*
we search using the stemmed/char-grammed words, but we format
snippets in the results by bolding either
*/
$query_string = str_replace('"', '', $query_string);
$query_words = explode(" ", $query_string); //not stemmed
if ($this->program_indicator) {
$query_string = $query;
$this->program_indicator = false;
}
$quote_state = false;
$phrase_string = str_replace('"', '"', $phrase_string);
$phrase_parts = explode('"', $phrase_string);
$base_words = [];
$num_words = 0;
/*
array of arrays, one for each quoted set of terms in the query
*/
$quote_positions = [];
foreach ($phrase_parts as $phrase_part) {
if (empty(trim($phrase_part))) {
$quote_state = ($quote_state) ? false : true;
continue;
}
/* still use original phrase string here to handle
acronyms abbreviations and the like that use periods */
if ($quote_state) {
$sub_parts = explode('*', $phrase_part);
$first_part = true;
/*
$term_positions_within_quoted_query is an associative array
of pairs: position_of_quoted_ngram_in_original_query =>
number_of_terms_ngrams_corresponds_to_or_star .
ngrams can be more than one term, but are usually just
1 term, so often this is an array of term positions each
mapped to 1. If having mapping *n => * it indicates
a wild_card occurred before n
*/
$term_positions_within_quoted_query = [];
foreach ($sub_parts as $sub_part) {
if (!$first_part) {
$term_positions_within_quoted_query["*$num_words"] =
"*";
}
$new_words = array_filter(PhraseParser::extractPhrases(
$sub_part, $locale_tag, $index_name, true));
$base_words = array_merge($base_words, $new_words);
foreach ($new_words as $new_word) {
$len = substr_count($new_word, " ") + 1;
$term_positions_within_quoted_query[$num_words] = $len;
$num_words++;
}
$first_part = false;
}
$quote_positions[] = $term_positions_within_quoted_query;
} else {
$new_words =
array_filter(
PhraseParser::extractPhrases($phrase_part, $locale_tag,
$index_name));
$base_words = array_merge($base_words, $new_words);
}
$num_words = count($base_words);
$quote_state = ($quote_state) ? false : true;
}
//stemmed, if have stemmer
$add_metas = $found_metas;
$words = array_merge($base_words, $add_metas);
if (count($words) == 0 && count($disallow_phrases) > 0) {
$words[] = "site:any";
}
if (C\p('QUERY_STATISTICS')) {
if (!isset($this->query_info['QUERY'])) {
$this->query_info['QUERY'] = "";
}
$this->query_info['QUERY'] .= "$in3<i>Index</i>: ".
$index_name."<br>";
$this->query_info['QUERY'] .= "$in3<i>LocaleTag</i>: ".
$locale_tag ."<br>";
$this->query_info['QUERY'] .=
"$in3<i>Stemmed/Char-grammed Words</i>:<br>";
foreach ($base_words as $word) {
$this->query_info['QUERY'] .= "$in4$word<br>";
}
$this->query_info['QUERY'] .= "$in3<i>Meta Words</i>:<br>";
foreach ($found_metas as $word) {
$this->query_info['QUERY'] .= "$in4$word<br>";
}
$this->query_info['QUERY'] .= "$in3<i>Quoted Word Locs</i>:<br>";
foreach ($quote_positions as $quote_position) {
$this->query_info['QUERY'] .= "$in4(";
$comma = "";
foreach ($quote_position as $pos => $len) {
$this->query_info['QUERY'] .= "$comma $pos => $len";
$comma = ",";
}
$this->query_info['QUERY'] .= ")<br>";
}
}
$make_term_id = C\NS_LIB . "canonicalTerm";
if (isset($words) && count($words) == 1 &&
count($disallow_phrases) < 1 && !strpos($words[0], " ")) {
$phrase_string = $words[0];
$phrase_hash = $make_term_id(trim($phrase_string));
$word_struct = ["KEYS" => [$phrase_hash],
"QUOTE_POSITIONS" => null, "DISALLOW_KEYS" => [],
"WEIGHT" => $weight, "INDEX_NAME" => $index_name,
"WORDS" => $words];
} else {
//get a raw list of words and their hashes
$hashes = [];
$word_keys = [];
/* When the index was built with combined terms, a word joined
to one short lang: word and a safe:true or safe:all word, as
guessSemantics adds them, is looked up by its combined key,
and the two meta words are dropped, so no intersection with
them is needed. Meta words follow the ordinary words in
$words, so dropping them moves no ordinary word. */
$combine = [];
$field_keys = [];
$index = IndexManager::getIndex($index_name);
if (is_object($index) && !empty(
$index->archive_info['COMBINED_TERMS'] ?? false)) {
foreach ($found_metas as $meta) {
if (preg_match('/^lang:([a-z]+)$/', $meta, $match)) {
$combine['lang'] = $match[1];
} else if (in_array($meta, ["safe:true", "safe:all"])) {
$combine['safe'] = substr($meta, 5);
}
}
}
if (count($combine) == 2 && count($base_words) > 0) {
foreach ($words as $word) {
$word = trim($word);
if ($word == "lang:" . $combine['lang'] ||
$word == "safe:" . $combine['safe']) {
continue;
}
$is_meta = in_array($word, $found_metas);
$word_keys[] = $is_meta ? $make_term_id($word) :
L\combinedTerm($word, $combine['lang'],
$combine['safe']);
/* An index with field terms also files a word under
its field key for each page that has the word in
its host, title or path. */
if (!empty($index->archive_info['FIELD_TERMS'])) {
$field_keys[] = $is_meta ? $make_term_id($word) :
L\combinedTerm($word, $combine['lang'],
$combine['safe'] .
IndexDocumentBundle::FIELD_TERM_MARK);
}
}
} else {
foreach ($words as $word) {
$word_keys[] = $make_term_id(trim($word));
}
}
if (count($word_keys) == 0) {
$word_keys = null;
$word_struct = null;
}
$disallow_keys = [];
$num_disallow_keys = min(C\MAX_QUERY_TERMS,
count($disallow_phrases));
if ($num_disallow_keys > 0 && C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .= "$in3<i>Disallowed Words</i>:".
"<br>";
}
for ($i = 0; $i < $num_disallow_keys; $i++) {
// check if disallowed is a meta word and stem or not stem
if (mb_strstr($disallow_phrases[$i], ':') === false) {
$disallow_stem = PhraseParser::extractPhrases(
$disallow_phrases[$i], L\getLocaleTag());
//stemmed
} else {
$disallow_stem[0] = $disallow_phrases[$i];
}
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .= "$in4{$disallow_stem[0]}".
"<br>";
}
$disallow_keys[] = $make_term_id($disallow_stem[0]);
}
if ($word_keys !== null) {
$word_struct = ["KEYS" => $word_keys,
"QUOTE_POSITIONS" => $quote_positions,
"DISALLOW_KEYS" => $disallow_keys,
"WEIGHT" => $weight,
"INDEX_NAME" => $index_name,
"WORDS" => $words,
"FIELD_KEYS" => $field_keys
];
}
}
$pre_format_words = [];
foreach ($base_words as $base_word) {
$pre_format_words = array_merge($pre_format_words,
explode(" * ", ($base_word ?? "")));
}
$pre_format_words = array_values(array_unique(
array_merge($query_words, $pre_format_words)));
$format_words = [];
$count = count($pre_format_words);
for ($i = 0; $i < $count; $i++) {
$flag = true;
if ($pre_format_words[$i] == "") {
continue;
}
for ($j = 0; $j < $count; $j++) {
if ($j == $i) {
continue;
}
$hay = mb_strtolower($pre_format_words[$j]);
$needle = mb_strtolower($pre_format_words[$i]);
if ($hay == $needle && $j > $i) {
continue;
}
if (mb_strstr($hay, $needle)) {
$flag = false;
break;
}
}
if ($flag) {
$format_words[] = $pre_format_words[$i];
}
}
return [$word_struct, $format_words];
}
/**
* extractMetaWordInfo reads a query string's meta words. It extracts
* the meta words, which of them are
* "materialized" (i.e., should be encoded as part of word ids),
* disallowed phrases, the query string after meta words removed
* and ampersand substitution applied, the query string with meta words
* but apersand substitution applied, the index and the weights found
* as part of the query string. Finally, it extracts the locale_tag for
* the query
*
* @param string $phrase the query string
* @return array containing items listed above in the description of this
* method
*/
public function extractMetaWordInfo($phrase)
{
$index_name = $this->index_name;
$weight = 1;
$found_metas = [];
$disallow_phrases = [];
$phrase_string = $phrase;
$phrase_string = str_replace("&", "&", $phrase_string);
$meta_words = PhraseParser::$meta_words_list;
if (isset($this->additional_meta_words)) {
$meta_words = array_merge($meta_words, array_keys(
$this->additional_meta_words));
}
$materialized_match_conflict = false;
$locale_tag = L\guessLocaleFromString($phrase_string);
foreach ($meta_words as $meta_word) {
$pattern = "/(\s)($meta_word(\S)+)/";
preg_match_all($pattern, $phrase, $matches);
if (empty($matches[2])) {
continue;
}
if (!in_array($meta_word,
['i:', 'index:', '\-i:', '\-index:', 'w:', 'weight:', '\-'])) {
$matches = $matches[2];
$found_metas = array_merge($found_metas, $matches);
} elseif ($meta_word == 'i:' || $meta_word == 'index:' ||
$meta_word == '\-i:' || $meta_word == '\-index:') {
if (str_starts_with($meta_word, '\-')) {
$index_name = substr($matches[2][0],
strlen($meta_word) - 1);
$index_name = "-$index_name";
} else {
$index_name = substr($matches[2][0], strlen($meta_word));
}
} elseif ($meta_word == '\-') {
if (count($matches[0]) > 0 &&
!str_contains($matches[0][0], ':')) {
foreach ($matches[2] as $disallowed) {
$disallow_phrases[] = substr($disallowed, 1);
}
}
} elseif ($meta_word == 'w:' || $meta_word == 'weight:') {
$weight = substr($matches[2][0], strlen($meta_word));
}
$phrase_string = preg_replace($pattern, "", $phrase_string);
}
if ($materialized_match_conflict) {
$found_metas = [];
$disallow_phrases = [];
$phrase_string = "";
}
$found_metas = array_unique($found_metas);
$disallow_phrases = array_unique($disallow_phrases);
$phrase_string = str_replace("&", "_and_", $phrase_string);
$query_string = preg_replace("/" . C\PUNCT . "/u", " ",
$phrase_string) ?? mb_ereg_replace(C\PUNCT, " ", $phrase_string);
$query_string = preg_replace("/(\s)+/", " ", $query_string);
$query_string = str_replace('_and_', '&', $query_string);
$phrase_string = str_replace('_and_', '&', $phrase_string);
return [$found_metas, $disallow_phrases, $phrase_string, $query_string,
$index_name, $weight, $locale_tag];
}
/**
* guessSemantics guesses from a query what the user is looking for.
* For now it does simple things, such as rewriting a query term that
* is a url as the matching meta word.
*
* @param string $phrase input query to guess semantics of
* @return string a phrase that more closely matches the intentions of the
* query.
*/
public function guessSemantics($phrase)
{
$domain_suffixes = [".com", ".net", ".edu", ".org", ".gov",
".mil", ".ca", ".uk", ".fr", ".ly"];
$len = mb_strlen(trim($phrase));
if ($len > 4) {
foreach ($domain_suffixes as $suffix) {
$phrase = $this->endMatch($phrase, $suffix, "site:", "",
[":", "@"]);
}
$phrase = $this->beginMatch($phrase, "www.", "site:www.");
$phrase = $this->beginMatch($phrase, "http:", "site:http:");
// only rewrite info if longer than hash info record length
if ($len > self::INFO_HASH_LEN) {
$phrase = $this->beginMatch($phrase, "info:", "info:http://",
"/", ["/"]);
$phrase = $this->beginMatch($phrase, "info:", "info:http://",
"", ["http"]);
}
}
$tag = L\guessLocaleFromString($phrase);
if (isset($this->programming_language_map[$tag])) {
$this->program_indicator = true;
}
$tag_parts = explode("-", $tag);
$main_tag = $tag_parts[0];
$tag = str_replace("-", "_", $tag);
$tokenizer_name = C\NS_LOCALE . "$tag\\resources\\Tokenizer";
if (class_exists($tokenizer_name)) {
$tokenizer = new $tokenizer_name();
if (isset($tokenizer->semantic_rewrites)) {
$rewrites = $tokenizer->semantic_rewrites;
$tmp = trim($phrase);
if (isset($rewrites[$tmp])) {
$phrase = $rewrites[$tmp];
}
}
}
$is_question = false;
if (!empty($tokenizer) && method_exists($tokenizer, "isQuestion") &&
method_exists($tokenizer, "questionParser") &&
$tokenizer->isQuestion($phrase)) {
$generated_question = $tokenizer->questionParser(
$phrase, $tag);
if (!empty($generated_question['CONCISE'])) {
$phrase = implode("|", $generated_question['CONCISE']);
} else if (!empty($generated_question['RAW'])) {
$phrase = implode("|", $generated_question['RAW']);
}
$is_question = true;
}
$is_question_delim = "";
if ($len > 0 && !preg_match("/site\:\S{5}|info\:|path\:|ip\:/",
$phrase)) {
if (!preg_match("/safe\:/", $phrase)) {
$safe_term = " " .((!isset($_SESSION['SAFE_SEARCH']) ||
$_SESSION['SAFE_SEARCH'] == "true") ? "safe:true" :
"safe:all") . " ";
$phrase .= $safe_term;
$is_question_delim .= $safe_term;
}
if (!preg_match("/date\:/", $phrase) &&
!empty($_SESSION['TIME_PERIOD_SELECTED']) &&
$_SESSION['TIME_PERIOD_SELECTED'] != 'all') {
$phrase .= " date:" . $_SESSION['TIME_PERIOD_SELECTED'];
}
$time = time();
$phrase = preg_replace("/date\:day/",
"date:" . date('Y-m-d', $time), $phrase);
$phrase = preg_replace("/date\:week/",
"date:" . date('Y-m-W', $time), $phrase);
$phrase = preg_replace("/date\:month/",
"date:" . date('Y-m', $time), $phrase);
$phrase = preg_replace("/date\:year/",
"date:" . date('Y', $time), $phrase);
if (!preg_match("/media\:image|lang\:/", $phrase)) {
$lang_term = " " . "lang:" . $main_tag . " ";
$phrase .= $lang_term;
$is_question_delim .= $lang_term;
}
if ($is_question) {
$phrase = str_replace("|", " $is_question_delim | ", $phrase);
}
if (preg_match("/media\:video/", $phrase)) {
if (!empty($_SESSION['VIDEO_MIN_DURATION'])) {
$phrase .= " duration:" . $_SESSION['VIDEO_MIN_DURATION'] .
"-plus";
}
if (!empty($_SESSION['VIDEO_MAX_DURATION']) &&
$_SESSION['VIDEO_MAX_DURATION'] < 1000000) {
$phrase .= " duration:" . $_SESSION['VIDEO_MAX_DURATION'] .
"-minus";
}
if (!empty($_SESSION['VIDEO_RESOLUTION_SELECTED'])) {
$media_resolution = "media:" .
$_SESSION['VIDEO_RESOLUTION_SELECTED'];
$phrase = preg_replace("/\bmedia\:video\b/",
" $media_resolution ", $phrase);
}
}
if (preg_match("/\bmedia\:image\b/", $phrase)) {
if (!empty($_SESSION['IMAGE_SIZE_SELECTED'])) {
$media_size = "media:" . $_SESSION['IMAGE_SIZE_SELECTED'];
$phrase = preg_replace("/\bmedia\:image\b/",
" $media_size ", $phrase);
}
if (!empty($_SESSION['IMAGE_COLOR_SELECTED']) &&
$_SESSION['IMAGE_COLOR_SELECTED'] != 'all') {
$phrase .= " color:" . $_SESSION['IMAGE_COLOR_SELECTED'];
}
if (!empty($_SESSION['IMAGE_LAYOUT_SELECTED']) &&
$_SESSION['IMAGE_LAYOUT_SELECTED'] != 'all') {
$phrase .= " layout:" . $_SESSION['IMAGE_LAYOUT_SELECTED'];
}
}
}
return $phrase;
}
/**
* beginMatch finds the terms of $phrase in the language $lang_tag that
* begin with $start_with and do not contain $not_contain. A term is a
* run of characters that are not white space. In each such term it
* replaces $start_with with $new_prefix and adds $suffix to the end.
*
* @param string $phrase string to look for terms in
* @param string $start_with what we're looking to see if term begins with
* @param string $new_prefix what to change $start_with to
* @param string $suffix what to tack on to the end of the term if there is
* a match
* @param string $not_contains string match is not allowed to contain
* @param string $lang_tag what language the phrase must be in for the rule
* to apply
*
* @return string $phrase after modifications have been made
*/
public function beginMatch($phrase, $start_with, $new_prefix, $suffix = "",
$not_contains = [], $lang_tag = "en-US")
{
$phrase .= " ";
$quote_start_with = preg_quote($start_with, "/");
$pattern = "/(\s)($quote_start_with(\S)+)/";
$start_pos = strlen($start_with);
preg_match_all($pattern, $phrase, $matches);
$matches = $matches[2];
$result_phrase = preg_replace($pattern, "", $phrase);
foreach ($matches as $match) {
$tag = L\guessLocaleFromString($match, $lang_tag);
$not_check = true;
foreach ($not_contains as $not_contain) {
if (strstr($match, $not_contain)) {
$not_check = false;
break;
}
}
if ($tag == $lang_tag && $not_check) {
$body = substr($match, $start_pos);
$result_phrase .= " " . $new_prefix . $body . $suffix;
} else {
$result_phrase .= " " . $match;
}
}
return $result_phrase;
}
/**
* endMatch finds the terms of $phrase in the language $lang_tag that
* end with $end_with and do not contain $not_contain. A term is a run
* of characters that are not white space. In each such term it
* replaces $end_with with $new_suffix, if that is not empty, and adds
* $prefix to the beginning.
*
* @param string $phrase string to look for terms in
* @param string $end_with what we're looking to see if term ends with
* @param string $prefix what to tack on to the start if there is
* a match
* @param string $new_suffix what to change $end_with to
* @param string $not_contains string match is not allowed to contain
* @param string $lang_tag what language the phrase must be in for the rule
* to apply
*
* @return string $phrase after modifications have been made
*/
public function endMatch($phrase, $end_with, $prefix, $new_suffix = "",
$not_contains = [], $lang_tag = "en-US")
{
$phrase .= " ";
$quote_end_with = preg_quote($end_with, "/");
$pattern = "/(\s)((\S)+$quote_end_with)(\s)/";
$end_len = strlen($end_with);
preg_match_all($pattern, $phrase, $matches);
$matches = $matches[2];
$result_phrase = preg_replace($pattern, " ", $phrase);
foreach ($matches as $match) {
$tag = L\guessLocaleFromString($match, $lang_tag);
$not_check = true;
foreach ($not_contains as $not_contain) {
if (strstr($match, $not_contain)) {
$not_check = false;
break;
}
}
if ($tag == $lang_tag && $not_check) {
if ($new_suffix == "") {
$body = $match;
} else {
$body = substr($match, 0, -$end_len);
}
$result_phrase .= " $prefix".$body.$new_suffix;
} else {
$result_phrase .= " ".$match;
}
}
return $result_phrase;
}
/**
* parseIfConditions works out each if: meta word in a query string,
* which chooses between two pieces of query, and returns the query
* string that results.
*
* @param string $phrase original query string
* @return string query string after if: meta words have been evaluated
*/
public function parseIfConditions($phrase)
{
$cond_token = "if:";
$pattern = "/(\s)($cond_token(\S)+)/";
preg_match_all($pattern, $phrase, $matches);
$matches = $matches[2];
$result_phrase = preg_replace($pattern, "", $phrase);
foreach ($matches as $match) {
$match = substr($match, strlen($cond_token));
$match_parts = explode("!", $match);
if (count($match_parts) < 2) {
continue;
}
if (stristr($result_phrase, $match_parts[0]) !== false) {
$result_phrase .= " ".str_replace("+", " ", $match_parts[1]);
} elseif (isset($match_parts[2])) {
$result_phrase .= " ".str_replace("+", " ", $match_parts[2]);
}
}
return $result_phrase;
}
/**
* getSummariesByHash returns the summaries of the documents that hold
* the given words and pass the other conditions given.
* @param array $word_structs an array of word_structs. Here a word_struct
* is an associative array with at least the following fields
* KEYS -- an array of word keys
* QUOTE_POSITIONS -- an array of positions of words that appeared in
* quotes (so need to be matched exactly)
* DISALLOW_PHRASES -- an array of words the document must not contain
* WEIGHT -- a weight to multiple scores returned from this iterator by
* INDEX_NAME -- an index timestamp to get results from
* @param int $limit number of first document in order to return
* @param int $num number of documents to return summaries of
* @param SearchfiltersModel $filter Model responsible for keeping
* track of edited and deleted search results
* @param bool $use_cache_if_allowed if true and USE_CACHE is true then
* an attempt will be made to look up the results in
* the file cache. Otherwise, items will be recomputed
* and then potentially restored in cache
* @param int $raw ($raw == 0) normal grouping, ($raw > 0)
* no grouping done on data. if ($raw == 1) no lookups of summaries
* done
* @param array $queue_servers a list of urls of yioop machines which might
* be used during lookup
* @param string $original_query if set, the original query that corresponds
* to $word_structs
* @param string $save_timestamp_name if this timestamp is not empty, then
* save iterate position, so can resume on future queries that make
* use of the timestamp. If used then $limit ignored and get next $num
* docs after $save_timestamp 's previous iterate position.
* @param array $format_words words which should be highlighted in
* search snippets returned
* @param array $ranking_factors field say how url, keywords, and
* title words should influence relevance and doc rank calculations
* @param array $typed_words the words of the query as typed, without
* words such as i:, so that addJoinedWordsBonus can look for them
* joined in each result's host and path
* @return array document summaries
*/
public function getSummariesByHash($word_structs, $limit, $num, $filter,
$use_cache_if_allowed = true, $raw = 0, $queue_servers = [],
$original_query = "", $save_timestamp_name = "",
$format_words = null, $ranking_factors = [], $typed_words = [])
{
$indent= " ";
$in2 = $indent . $indent;
$in3 = $in2 . $indent;
$in4 = $in2. $in2;
if (C\p('QUERY_STATISTICS')) {
$lookup_time = microtime(true);
AnalyticsManager::set(AnalyticsManager::TIMINGS, []);
}
$time = time();
$test_query = trim(preg_replace("/\s+/u", " ", $original_query));
if (empty($filter)) {
$filter_time = 0;
} else {
$filter_time = $filter->lastChange();
}
$pages = [];
$generation = 0;
$to_retrieve = ceil(($limit + $num)/C\p('MIN_RESULTS_TO_GROUP')) *
C\p('MIN_RESULTS_TO_GROUP');
$start_slice = floor(($limit)/C\p('MIN_RESULTS_TO_GROUP')) *
C\p('MIN_RESULTS_TO_GROUP');
if ($save_timestamp_name != "") {
$to_retrieve = $num;
$limit = 0;
$start_slice = 0;
}
if (!empty($_SERVER["USE_CACHE"]) && $save_timestamp_name == "") {
$serial_query_info = serialize($raw) . serialize($word_structs) .
$original_query . $this->index_name;
$summary_hash = L\crawlHash($serial_query_info . ":" . $limit .
":" . $num);
if ($use_cache_if_allowed) {
$cache_success = true;
$results = self::$cache->get($summary_hash);
if (!isset($results['TIME']) || $filter_time >
$results['TIME']) {
//if filter has changed since cached, then invalidate cache
$results = false;
}
if (isset($results['TIME'])) {
$cached_time = $time - $results['TIME'];
} else {
$cached_time = $time;
}
if (C\MAX_QUERY_CACHE_TIME > 0 &&
$cached_time > C\MAX_QUERY_CACHE_TIME) {
$results = false;
}
if (isset($results['PAGES'])) {
$close_prefix = C\SCHEDULES_DIR . "/".
self::index_closed_name;
$has_changeable_results = false;
$seen_times = [];
foreach ($results['PAGES'] as $page) {
if (!isset($page[self::CRAWL_TIME]) ||
in_array($page[self::CRAWL_TIME], $seen_times)) {
continue;
}
$seen_times[] = $page[self::CRAWL_TIME];
$current_closed = $close_prefix .
$page[self::CRAWL_TIME] . ".txt";
if (!file_exists($current_closed)) {
//either feed result or from active crawl
$has_changeable_results = true;
break;
}
}
if ($has_changeable_results) {
if ($cached_time > C\MIN_QUERY_CACHE_TIME) {
$results = false;
}
}
}
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .=
"$in2<b>Cache Lookup Time</b>: ".
L\changeInMicrotime($lookup_time)."<br>";
if (!empty(self::$cache->cache_file) ) {
$this->query_info['QUERY'] .=
"$in2<b>Cache File Used:</b> " .
self::$cache->cache_file . "<br>";
}
}
if ($results !== false) {
return $results;
}
}
}
$old_to_retrieve = $to_retrieve;
$get_query_time = microtime(true);
$query_iterator = $this->getQueryIterator($word_structs, $filter, $raw,
$to_retrieve, $queue_servers, $original_query,
$save_timestamp_name, $ranking_factors);
AnalyticsManager::addTiming("query iterators made", $get_query_time);
$get_query_time = L\changeInMicrotime($get_query_time);
$num_retrieved = 0;
$pages = [];
$retrieve_postings_time = microtime(true);
if (is_object($query_iterator)) {
while ($num_retrieved < $to_retrieve) {
$next_docs = $query_iterator->nextDocsWithWord();
if (!is_array($next_docs)) {
break;
}
$pages = array_merge($pages, $next_docs);
$num_retrieved = count($pages);
}
}
/*
at this pount count($pages) >= $to_retrieve or there were
less than $to_retrieve docs. Notice it may the case
count($pages) is larger than $to_retrieve
*/
AnalyticsManager::addTiming("results scored and grouped",
$retrieve_postings_time);
$retrieve_postings_time = L\changeInMicrotime($retrieve_postings_time);
if ($save_timestamp_name != "" && ($queue_servers == [] ||
$this->isSingleLocalhost($queue_servers))) {
// used for archive crawls of crawl mixes
$save_file = C\SCHEDULES_DIR . "/" . self::save_point.
$save_timestamp_name . ".txt";
$iterators = $query_iterator->save_iterators;
$cnt_iterators = (is_array($iterators)) ? count($iterators) : 0;
$save_point = [];
for ($i = 0; $i < $cnt_iterators; $i++) {
$save_point[$i] =
$iterators[$i]->currentGenDocOffsetWithWord();
}
$results["SAVE_POINT"] = $save_point;
$this->filePutContents($save_file, serialize($save_point));
$this->db->setWorldPermissionsRecursive($save_file);
}
$pages = array_values($pages);
$result_count = count($pages);
$sort_time = 0;
// initialize scores
$sort_start = microtime(true);
$max_user_ranks = 0;
$ranking_factors["USER_RANK_BONUS"] ??= C\p('USER_RANK_BONUS');
for ($i = 0; $i < $result_count; $i++) {
if (isset($pages[$i][self::USER_RANKS])) {
$j = count($pages[$i][self::USER_RANKS]);
if ($max_user_ranks < $j) {
$max_user_ranks = $j;
}
}
}
if ($max_user_ranks > 0) {
$rank_weight = $ranking_factors["USER_RANK_BONUS"];
$mean_user_ranks = [];
for ($j = 0; $j < $max_user_ranks; $j++) {
$user_rank_sum = 0;
for ($i = 0; $i < $result_count; $i++) {
$user_rank_sum += floatval($pages[$i][self::USER_RANKS][
$j]['SCORE'] ?? 0);
}
$mean_user_ranks[$j] = $user_rank_sum / $result_count;
}
for ($i = 0; $i < $result_count; $i++) {
for ($j = 0; $j < $max_user_ranks; $j++) {
if (isset($pages[$i][self::USER_RANKS][$j])) {
$pages[$i]["USCORE$j"] =
I\WordIterator::divergenceScore(
$rank_weight * $mean_user_ranks[$j],
$rank_weight * floatval(number_format(
$pages[$i][self::USER_RANKS][$j]['SCORE'] ?? 0,
4, '.', '')));
$pages[$i][self::SCORE] += $pages[$i]["USCORE$j"];
} else {
// might want to print all scores
$pages[$i]["USCORE$j"] = 0;
}
}
}
}
if (!empty($pages)) {
L\orderCallback($pages[0], $pages[0], self::SCORE);
usort($pages, C\NS_LIB . "orderCallback");
}
$sort_time = L\changeInMicrotime($sort_start);
if (empty($results)) {
$results = [];
}
if ($num_retrieved < $to_retrieve) {
$results['TOTAL_ROWS'] = $num_retrieved;
} else {
$results['TOTAL_ROWS'] = $query_iterator->num_docs;
//this is only an approximation
}
if ($raw == 1 && $save_timestamp_name == "") {
$pages = array_slice($pages, $start_slice);
$pages = array_slice($pages, $limit - $start_slice, $num);
$i = 0;
foreach ($pages as $p) {
$pages[$i][self::SCORE] = floatval(number_format(
($p[self::SCORE] ?? 0), 4, '.', ''));
$pages[$i][self::DOC_RANK] = floatval(number_format(
($p[self::DOC_RANK] ?? 0), 4, '.', ''));
$pages[$i][self::RELEVANCE] = floatval(number_format(
($p[self::RELEVANCE] ?? 0), 4, '.', ''));
unset($pages[$i][self::DESCRIPTION_SCORES]);
$i++;
}
$results['PAGES'] = & $pages;
if ($old_to_retrieve != $to_retrieve) {
$results['HARD_QUERY'] = $old_to_retrieve;
}
return $results;
}
if (C\p('QUERY_STATISTICS')) {
$this->query_info['QUERY'] .= "$in2<b>Lookup Offsets Time</b>: ".
L\changeInMicrotime($lookup_time)."<br>";
$machine_times = AnalyticsManager::get("MACHINE_TIMES");
if ($machine_times) {
$this->query_info['QUERY'] .=
"$in3<i>Machine Sub-Times</i> (query/total):<br>".
$machine_times."<br>";
}
$net_times = AnalyticsManager::get("NET_TIMES");
$max_machine_times = AnalyticsManager::get("MAX_MACHINE_TIMES");
if ($net_times && $max_machine_times) {
$this->query_info['QUERY'] .=
"$in3<i>Network Overhead Sub-Time</i>: ".
($net_times - $max_machine_times)."<br>";
}
if ($sort_time) {
$this->query_info['QUERY'] .=
"$in3<i>Get Iterator Sub-Time</i>: " . $get_query_time .
"<br>$in3<i>Retrieve Postings Sub-Time</i>: " .
$retrieve_postings_time . "<br>" .
"$in3<i>Merge-Rank Sub-Time</i>: " .
$sort_time."<br>";
}
foreach (AnalyticsManager::timingsReport() as $timing_line) {
$this->query_info['QUERY'] .= "$in3<i>" .
ucfirst($timing_line) . "</i><br>";
}
if ($query_iterator) {
$this->query_info['PLAN'] = $query_iterator->plan();
}
$summaries_time = microtime(true);
}
/* Use 2* $num because might have some deduplication/robot exclusions
Go from 0 since there may be deduplciation/exclusions before limit
*/
/* every page inside the diversify window is cut from the same
results, so for each of them the summaries of the window and
one page more, for results dropped as duplicates, are fetched */
$mmr_window = min(self::MMR_PAGES * $num, ($num > 0) ?
intdiv(C\p('MIN_RESULTS_TO_GROUP'), $num) * $num : 0);
$get_pages = array_slice($pages, 0, max($limit + 2 * $num,
$mmr_window + $num));
$to_get_count = count($get_pages);
$groups_with_docs = false;
if (preg_match("/\bsite:doc\b/", $original_query)) {
$groups_with_docs = true;
}
$out_pages = [];
$cur_limit = $limit;
$with_qa = (preg_match("/\bqqq\b/i", $original_query)) ? true : false;
// now calculate snippet length
$description_length = self::DEFAULT_DESCRIPTION_LENGTH;
/* additional meta word come from indexing plugins which might need
longer description lengths, say for recipes
*/
if (isset($this->additional_meta_words) &&
is_array($this->additional_meta_words)) {
foreach ($this->additional_meta_words as $meta_word => $length) {
$pattern = "/$meta_word/";
if (preg_match($pattern, $original_query)) {
$description_length = $length;
break; // only match the first found
}
}
}
/* this is a loop because getSummariesFromOffsets might eliminate some
entries forbidden by robot directives or by deduplication
*/
$count_out_pages = 0;
$read_start = microtime(true);
while ($count_out_pages < $to_get_count && $get_pages) {
$out_pages = array_merge($out_pages,
$this->getSummariesFromOffsets($get_pages, $queue_servers,
$raw, $groups_with_docs, $with_qa, $format_words,
$description_length));
if ($save_timestamp_name != "") {
break;
}
$cur_limit += $to_get_count;
$count_out_pages = count($out_pages);
$to_get_count = 2 * ($to_get_count - $count_out_pages);
$get_pages = array_slice($pages, $cur_limit, $to_get_count);
}
AnalyticsManager::addTiming("summaries read", $read_start);
if ($cur_limit > $results['TOTAL_ROWS']) {
$results['TOTAL_ROWS'] = count($out_pages);
} else if ($cur_limit > 0) {
$results['TOTAL_ROWS'] = ceil(
(count($out_pages) * $results['TOTAL_ROWS']) / $cur_limit);
}
$this->addJoinedWordsBonus($out_pages, $typed_words,
$ranking_factors);
$this->addExactMatchScore($out_pages, $typed_words);
if ($save_timestamp_name == "") {
$this->diversify($out_pages, $typed_words, $mmr_window);
}
$out_pages = array_slice($out_pages, $limit, $num);
if (C\p('QUERY_STATISTICS')) {
$summary_times_string = AnalyticsManager::get("SUMMARY_TIMES");
if ($summary_times_string) {
$round_summary_times = unserialize($summary_times_string);
$summary_delta_time = L\changeInMicrotime($summaries_time);
$summary_time_info = "$summary_delta_time<br> $in4";
$sum_max_time = 0;
foreach ($round_summary_times as $summary_times) {
$i = 0;
$max_time = 0;
foreach ($summary_times as $summary_time) {
$summary_time_info .= "ID_$i: ".
number_format($summary_time, 6) . "$indent";
$max_time = ($summary_time > $max_time) ?
$summary_time : $max_time;
$i++;
}
$summary_time_info .= "<br>\n$in4";
$sum_max_time += $max_time;
}
$net_overhead = $summary_delta_time - $sum_max_time;
$summary_time_info .=
"<br>$in3<i>Network Overhead Sub-Time</i>: ".
$net_overhead;
} else {
$summary_time_info = L\changeInMicrotime($summaries_time);
}
$this->query_info['QUERY'] .= "$in2<b>Get Summaries Time</b>: ".
$summary_time_info."<br>";
}
$results['PAGES'] = $out_pages;
$results['TIME'] = time();
if (!empty($_SERVER["USE_CACHE"]) &&
$save_timestamp_name == "") {
self::$cache->set($summary_hash, $results);
}
return $results;
}
/**
* addJoinedWordsBonus adds to the score of a result whose host or path
* holds the words of a query of two or three words joined together, and
* puts the results back in score order. The words are joined three ways:
* with nothing, with a hyphen and with an underscore, so tik tok is
* looked for as tiktok, tik-tok and tik_tok. Such a join found in the
* host, as tiktok is in www.tiktok.com, adds JOINED_WORDS_BONUS_SHARE
* of HOST_KEYWORD_BONUS. Found only in the path, it adds that share of
* PATH_KEYWORD_BONUS. The index holds a host only as its separate words,
* so the join can be looked for only once a result's url is known;
* getSummariesByHash calls this on the results it has fetched the
* summaries of, before it cuts out the page of results asked for.
*
* @param array &$pages results with their urls and scores, changed in
* place
* @param array $typed_words the words of the query as typed
* @param array $ranking_factors the bonuses to use in place of the
* profile's HOST_KEYWORD_BONUS and PATH_KEYWORD_BONUS, where given
*/
public function addJoinedWordsBonus(&$pages, $typed_words,
$ranking_factors)
{
$joined_word = mb_strtolower(implode("", $typed_words));
if (count($typed_words) < 2 || count($typed_words) > 3 ||
!preg_match('/^\p{L}+$/u', $joined_word)) {
return;
}
$words = array_map("mb_strtolower", $typed_words);
$joins = [$joined_word, implode("-", $words), implode("_", $words)];
$host_bonus = self::JOINED_WORDS_BONUS_SHARE *
floatval($ranking_factors["HOST_KEYWORD_BONUS"] ??
C\p('HOST_KEYWORD_BONUS'));
$path_bonus = self::JOINED_WORDS_BONUS_SHARE *
floatval($ranking_factors["PATH_KEYWORD_BONUS"] ??
C\p('PATH_KEYWORD_BONUS'));
$changed = false;
foreach ($pages as $index => $page) {
$url = $page[self::URL] ?? "";
$host = mb_strtolower(parse_url($url, PHP_URL_HOST) ?? "");
$path = mb_strtolower(urldecode(parse_url($url, PHP_URL_PATH) ??
""));
foreach ($joins as $join) {
if (str_contains($host, $join)) {
$pages[$index][self::SCORE] += $host_bonus;
$changed = true;
break;
}
if (str_contains($path, $join)) {
$pages[$index][self::SCORE] += $path_bonus;
$changed = true;
break;
}
}
}
if ($changed) {
usort($pages, fn($first, $second) =>
$second[self::SCORE] <=> $first[self::SCORE]);
}
}
/**
* diversify reorders the top $window results by maximal
* marginal (what a result adds over the results above it) relevance,
* or MMR, after Carbonell and Goldstein (1998), so a result much like
* one above it moves down. The results are picked one at a time from
* the window. Each pick is the result with the highest
*
* lambda * score / top score - (1 - lambda) * similarity
*
* where similarity is the largest cosine similarity between the
* result and a result already picked, and lambda is $lambda. The
* cosine similarity of two results counts the words their titles
* share, over the square root of the product of how many words each
* title has. The query's own words are left out, since every result
* holds them. Titles alone are compared, since a description is
* several times as long to split into words. A lambda of 1 keeps the
* order by score.
*
* The number of title words each pair of results shares is counted
* once, from a list of the results that hold each word, so a pair
* with no word in common costs nothing. Each pick is the largest value
* left. A pick then lowers only the values of the results that share
* a word with it. Scores are not changed, only the order. Results
* after the window keep their order. getSummariesByHash calls it with
* a window of MMR_PAGES pages once the summaries are fetched and the
* other score changes are made, before it cuts out the page of results
* asked for.
*
* @param array &$pages results with their titles and scores, reordered
* in place
* @param array $typed_words the words of the query as typed
* @param int $window how many of the top results to reorder
* @param float $lambda weight of a result's score against its
* similarity to results above it, from 0 to 1; the setting
* MMR_RELEVANCE_WEIGHT in Config.php unless given
*/
public function diversify(&$pages, $typed_words, $window,
$lambda = C\MMR_RELEVANCE_WEIGHT)
{
$window = min($window, count($pages));
$scores = array_column(array_slice($pages, 0, $window),
self::SCORE);
$top_score = max($scores ?: [0]);
if ($lambda >= 1 || $window < 2 || $top_score <= 0 ||
count($scores) < $window) {
return;
}
$query_words = array_flip(array_map("mb_strtolower", $typed_words));
$shares = [];
$sizes = [];
$holders = [];
foreach ($scores as $i => $score) {
$shares[$i] = $lambda * $score / $top_score;
$words = array_diff_key(array_flip(preg_split(
'/[^\p{L}\p{N}]+/u', mb_strtolower(strip_tags(
$pages[$i][self::TITLE] ?? "")), -1, PREG_SPLIT_NO_EMPTY)),
$query_words);
$sizes[$i] = count($words);
foreach (array_keys($words) as $word) {
$holders[$word][] = $i;
}
}
$shared = [];
foreach ($holders as $holder_list) {
$num_holders = count($holder_list);
for ($j = 0; $j < $num_holders; $j++) {
for ($k = $j + 1; $k < $num_holders; $k++) {
$first = $holder_list[$j];
$second = $holder_list[$k];
$shared[$first][$second] =
($shared[$first][$second] ?? 0) + 1;
$shared[$second][$first] = $shared[$first][$second];
}
}
}
$values = $shares;
$picked = [];
while (!empty($values)) {
$best = array_search(max($values), $values);
$picked[] = $best;
unset($values[$best]);
foreach ($shared[$best] ?? [] as $i => $count) {
if (isset($values[$i])) {
$values[$i] = min($values[$i], $shares[$i] -
(1 - $lambda) * $count /
sqrt($sizes[$i] * $sizes[$best]));
}
}
}
$reordered = [];
foreach ($picked as $i) {
$reordered[] = $pages[$i];
}
$pages = array_merge($reordered, array_slice($pages, $window));
}
/**
* addExactMatchScore multiplies the relevance a result got for a
* query word by EXACT_MATCH_FACTOR where the result's title or summary
* holds the word exactly as it was typed, and puts the results back
* in score order. The index holds the stems of words, so a query for
* anime also finds animal, which has the same stem, and the
* divergence-from-randomness score the index gives the stem cannot
* tell the two apart. Once the title and summary of a result are in
* hand, getSummariesByHash calls this so that an exact match counts
* for more than a match of the stem alone. A result's relevance is the
* sum of the divergence scores of the query's terms, and the iterators
* do not keep the terms' scores apart, so each typed word is taken to
* have earned an equal share of it. With one word the whole relevance
* is multiplied. A word is matched whole and in any case, except that
* a word in a script written without spaces, such as Chinese, is
* matched wherever it appears. A word to leave out, written with a
* leading minus, is not looked for.
*
* @param array &$pages results with their titles, summaries, relevance
* and scores, changed in place
* @param array $typed_words the words of the query as typed
*/
public function addExactMatchScore(&$pages, $typed_words)
{
$patterns = [];
foreach ($typed_words as $typed_word) {
if (str_starts_with($typed_word, "-")) {
continue;
}
$word = PhraseParser::foldDiacritics(mb_strtolower(
preg_replace('/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/u', "",
$typed_word)));
if ($word == "") {
continue;
}
$quoted = preg_quote($word, "/");
$patterns[$word] = preg_match(self::UNSPACED_SCRIPTS, $word) ?
"/$quoted/iu" :
"/(?<![\p{L}\p{N}])$quoted(?![\p{L}\p{N}])/iu";
}
if (empty($patterns)) {
return;
}
$changed = false;
foreach ($pages as $index => $page) {
/* The index files a word both with and without diacritics, so
the word and the text are compared without them: jose
matches José. */
$text = PhraseParser::foldDiacritics(
($page[self::TITLE] ?? "") . " " .
strip_tags($page[self::DESCRIPTION] ?? ""));
$found = 0;
foreach ($patterns as $pattern) {
$found += preg_match($pattern, $text);
}
$relevance = $page[self::RELEVANCE] ?? 0;
if ($found == 0 || $relevance <= 0) {
continue;
}
$added = (self::EXACT_MATCH_FACTOR - 1) * $relevance * $found /
count($patterns);
$pages[$index][self::RELEVANCE] = $relevance + $added;
$pages[$index][self::SCORE] = ($page[self::SCORE] ?? 0) + $added;
$changed = true;
}
if ($changed) {
usort($pages, fn($first, $second) =>
($second[self::SCORE] ?? 0) <=> ($first[self::SCORE] ?? 0));
}
}
/**
* getSummariesFromOffsets looks up the summary of each page given,
* using the page's self::SUMMARY_OFFSET field. Where a summary is an
* HTTP Location redirect, it looks up the page redirected to in turn.
* This method handles robot meta tags which might forbid indexing.
*
* @param array &$pages of page data without text summaries
* @param array &$queue_servers array of queue server to find data on
* @param int $raw only lookup locations if 0
* @param bool $groups_with_docs whether to return only groups that
* contain at least one doc as opposed to a groups with only links
* @param bool $with_question_answer_info whether question answer info
* in summaries needs to be returned
* @param array $format_words words which should be highlighted in
* search snippets returned
* @param int $description_length length of snippets to be returned
* for each search result
* @return array pages with summaries added
*/
public function getSummariesFromOffsets(&$pages, &$queue_servers,
$raw, $groups_with_docs, $with_question_answer_info,
$format_words = null, $description_length =
self::DEFAULT_DESCRIPTION_LENGTH)
{
if ($raw != 0) {
$format_words = null;
} else {
if ($format_words == null) {
$format_words = [];
}
}
$lookups = [];
$summary_exclude_fields = [self::HEADER, self::PAGE, self::LINKS,
self::DESCRIPTION_SCORES];
if (!$with_question_answer_info) {
$summary_exclude_fields[] = self::QUESTION_ANSWERS;
}
$page_indexes = [];
$index = 0;
// make array containing just what to look up
foreach ($pages as $page) {
$key = $page[self::KEY];
if (!empty($page[self::SUMMARY_OFFSET]) &&
is_array($page[self::SUMMARY_OFFSET])) {
$lookups[$key] = $page[self::SUMMARY_OFFSET];
} else {
$machine_id = (isset($page[self::MACHINE_ID])) ?
$page[self::MACHINE_ID] :$this->current_machine;
$page[self::SUMMARY_OFFSET] ??= 'PDB';
$lookups[$key][] =
[$machine_id, $key,
$page[self::CRAWL_TIME],
$page[self::GENERATION],
$page[self::SUMMARY_OFFSET]];
}
$page_indexes[$key] = $index;
$index++;
}
$lookup_queue_servers = $queue_servers;
if ($queue_servers && !in_array(C\p('NAME_SERVER'), $queue_servers) &&
C\nsdefined('ADD_NAME_SERVER_LOOKUP_SERVERS')) {
$lookup_queue_servers[] = C\p('NAME_SERVER');
//name server might still have news
}
if (count($lookup_queue_servers) == 1 &&
$lookup_queue_servers[0] == C\p('NAME_SERVER') &&
C\baseUrl() == C\p('NAME_SERVER')) {
// for now only do mirror non-lookup offsets
$lookup_queue_servers = [];
}
/* the pages that have only a link summary are looked up to get
their document summaries */
$summaries = $this->getCrawlItems($lookups, $lookup_queue_servers,
$summary_exclude_fields, $format_words, $description_length);
$lookups = [];
// link summaries we want to remember in case don't have doc summary
$link_summaries = [];
// check for redirects and doc summaries
foreach ($summaries as $hash_url => $summary) {
$lookup_url = false;
if (isset($summaries[$hash_url][self::LOCATION]) &&
$summaries[$hash_url][self::LOCATION] != []) {
$tmp_url = explode(" => ",
$summaries[$hash_url][self::DESCRIPTION]);
if (isset($tmp_url[1])) {
$lookup_url = trim($tmp_url[1]);
}
} elseif (isset($summaries[$hash_url][self::HASH])) {
$hash_parts = explode('|', $summaries[$hash_url][self::HASH]);
if (isset($hash_parts[3])) {
$lookup_url = $hash_parts[1];
}
}
if ($lookup_url) {
$crawl_time = $pages[$page_indexes[$hash_url]][
self::CRAWL_TIME];
$lookups[$hash_url] = [$lookup_url, $crawl_time];
$link_summaries[$hash_url] = $summaries[$hash_url];
unset($summaries[$hash_url]);
}
}
// lookup redirects
$loc_summaries = $this->getCrawlItems($lookups, $lookup_queue_servers,
$summary_exclude_fields, $format_words, $description_length);
// delete summaries we found from $link_summaries
if (is_array($loc_summaries)) {
$loc_hashes = array_keys($loc_summaries);
foreach ($loc_hashes as $hash_url) {
unset($link_summaries[$hash_url]);
}
} else {
$loc_summaries = [];
}
$summaries = array_merge($summaries, $loc_summaries,
$link_summaries);
$out_pages = [];
$seen_keys = [];
$seen_descriptions = [];
/* insert lookup values back into pages, dedeup,
handle robot meta tags present on page
*/
$i = 0;
$thumb_sites = [];
foreach ($pages as $page) {
$key = $page[self::KEY];
$add_page = false;
if (isset($summaries[$key]) &&
!isset($seen_keys[$key])) {
$seen_keys[$key] = true;
$hash_description =
L\crawlHash($summaries[$key][self::DESCRIPTION] ?? "");
if (isset($seen_descriptions[$hash_description])) {
continue;
}
$seen_descriptions[$hash_description] = true;
$metas = $summaries[$key][self::ROBOT_METAS] ?? [];
if (!empty($metas)) {
if (!in_array("NOINDEX", $metas) &&
!in_array("NONE", $metas)) {
$add_page = true;
}
} else {
$add_page = true;
}
if (!empty($add_page)) {
$out_pages[$i] = array_merge($page, $summaries[$key]);
if (!empty($out_pages[$i][self::IMAGE_LINK])) {
$image_link = $out_pages[$i][self::IMAGE_LINK];
$link_scheme = substr($image_link, 0, 7);
if ($link_scheme == "feed://") {
$image_link_parts = explode("/", $image_link);
if (!empty($image_link_parts[3])) {
list(, , $sf, $image_hash) = $image_link_parts;
}
} else { //old style image link
if(preg_match("/sf\=([^\&]+)/", $image_link,
$sf_match) && preg_match("/n\=([^\&]+)/",
$image_link, $n_match)) {
$sf = $sf_match[1];
$image_hash = $n_match[1];
}
}
if (!empty($sf) && !empty($image_hash)) {
$feeds_path = L\ResourceFolders::feedsPath();
$image_extensions = ["webp" => "image/webp",
"jpg" => "image/jpeg"];
$image_link_found = false;
foreach ($image_extensions as $image_extension =>
$image_mime_type) {
$image_path = "$feeds_path/" . basename($sf) .
"/" . basename($image_hash) .
".$image_extension";
if (file_exists($image_path)) {
$image_string = $this->fileGetContents(
$image_path);
$out_pages[$i][self::IMAGE_LINK] =
"data:$image_mime_type;base64," .
base64_encode($image_string);
$image_link_found = true;
break;
}
}
if (!$image_link_found) {
unset($out_pages[$i][self::IMAGE_LINK]);
}
}
}
$i++;
}
}
}
return $out_pages;
}
/**
* getQueryIterator builds the iterator that reads the results of a
* query from the supplied $word_structs.
*
* @param array $word_structs an array of word_structs. Here a word_struct
* is an associative array with at least the following fields
* KEYS -- an array of word keys
* QUOTE_POSITIONS -- an array of positions of words that appreared in
* quotes (so need to be matched exactly)
* DISALLOW_PHRASES -- an array of words the document must not contain
* WEIGHT -- a weight to multiple scores returned from this iterator by
* INDEX_NAME -- an index timestamp to get results from
* @param SearchfiltersModel $filter Model responsible for keeping track
* of edited and deleted search results
* @param int $raw ($raw == 0) normal grouping, ($raw == 1)
* no grouping done on data also no summaries returned (only lookup
* info), $raw > 1 return summaries but no grouping
* @param int $to_retrieve number of items to retrieve from location in
* iterator
* @param array $queue_servers a list of urls of yioop machines which might
* be used during lookup
* @param string $original_query if set, the original query that corresponds
* to $word_structs
* @param string $save_timestamp_name if this timestamp is non empty, then
* when making iterator get sub-iterators to advance to gen doc_offset
* stored with respect to save_timestamp if exists.
* @param array $ranking_factors field say how url, keywords, and
* title words should influence relevance and doc rank calculations
* @return object an iterator for iterating through results to the
* query
*/
public function getQueryIterator($word_structs, $filter, $raw,
&$to_retrieve, $queue_servers = [], $original_query = "",
$save_timestamp_name = "", $ranking_factors = [])
{
$iterators = [];
$part_words = [];
$field_parts = [];
$total_iterators = 0;
$network_flag = false;
$min_group_flag = false;
$min_group_override = false;
if ($queue_servers != [] &&
!$this->isSingleLocalhost($queue_servers)) {
$network_flag = true;
$total_iterators = 1;
if (!in_array(C\p('NAME_SERVER'), $queue_servers) &&
C\nsdefined('ADD_NAME_SERVER_LOOKUP_SERVERS')) {
$queue_servers[] = C\p('NAME_SERVER');
//name server might still have news
}
$num_servers = count($queue_servers);
if ((!isset($this->index_name) || !$this->index_name) &&
isset($word_structs[0]["INDEX_NAME"])) {
$index_name = $word_structs[0]["INDEX_NAME"];
} else {
$index_name = $this->index_name;
}
$iterators[0] = new I\NetworkIterator($original_query,
$queue_servers, $index_name, $filter, $save_timestamp_name,
$ranking_factors);
}
if (!$network_flag) {
$index_name = $this->index_name ?? "";
if ((!isset($this->index_name) || !$this->index_name) &&
isset($word_structs[0]["INDEX_NAME"])) {
$index_name = $word_structs[0]["INDEX_NAME"];
}
//we assume all indexes in use of the same version
$make_term_id = C\NS_LIB . "canonicalTerm";
$doc_iterate_hashes = [substr($make_term_id("site:any"), 0, 9),
substr(L\crawlHash("site:any"), 0, 9),
substr($make_term_id("site:doc"), 0, 9),
substr(L\crawlHash("site:doc"), 0, 9)];
if ($save_timestamp_name != "") {
// used for archive crawls of crawl mixes
$save_file = C\SCHEDULES_DIR . '/' . self::save_point .
$save_timestamp_name . ".txt";
if (file_exists($save_file)) {
$save_point =
unserialize(file_get_contents($save_file));
} else {
$save_point = [];
}
$save_count = 0;
}
foreach ($word_structs as $word_struct) {
if (!is_array($word_struct)) {
continue;
}
$word_keys = $word_struct["KEYS"];
$distinct_word_keys = [];
$seen_keys = [];
foreach ($word_keys as $wkey) {
if (is_string($wkey) || is_string($wkey[0])) {
$tmp_key = is_string($wkey) ? $wkey : $wkey[0];
if (!isset($seen_keys[$tmp_key])) {
$seen_keys[$tmp_key] = true;
$distinct_word_keys[] = $wkey;
}
} else {
$distinct_word_keys[] = $wkey;
}
}
$quote_positions = $word_struct["QUOTE_POSITIONS"];
$disallow_keys = $word_struct["DISALLOW_KEYS"];
$index_name = (string) $word_struct["INDEX_NAME"];
$weight = $word_struct["WEIGHT"];
$num_word_keys = count($word_keys);
$total_iterators = count($distinct_word_keys);
$word_iterators = [];
$word_iterator_map = [];
if ($num_word_keys < 1) {
continue;
}
$sum = 0;
$lookup_cutoff = max(C\p('MIN_RESULTS_TO_GROUP'), $to_retrieve);
for ($i = 0; $i < $total_iterators; $i++) {
$current_key = (is_string($distinct_word_keys[$i])) ?
$distinct_word_keys[$i] : (is_string(
$distinct_word_keys[$i][0]) ? $distinct_word_keys[$i][0]
: $distinct_word_keys[$i][0][0]);
if (!is_string($current_key)) {
$current_key = $current_key[0];
}
if (in_array(substr($current_key, 0, 9),
$doc_iterate_hashes)) {
$actual_index_name = $index_name;
$direction = self::ASCENDING;
if (($index_name[0] == "-")) {
$actual_index_name = substr($index_name, 1);
$direction = self::DESCENDING;
}
$word_iterators[$i] = new I\DocIterator(
$actual_index_name, $filter, $to_retrieve,
$direction, $ranking_factors);
$min_group_override = true;
} else {
$distinct_key = $distinct_word_keys[$i];
$distinct_key_id = $distinct_key;
$direction = self::ASCENDING;
$actual_index_name = $index_name;
if (($index_name[0] == "-")) {
$actual_index_name = substr($index_name, 1);
$direction = self::DESCENDING;
}
$word_iterators[$i] =
new I\WordIterator($distinct_key_id,
$actual_index_name, true, $filter, $to_retrieve,
$direction, $ranking_factors);
$min_group_override = true;
}
foreach ($word_keys as $index => $key) {
if (isset($distinct_word_keys[$i]) &&
$key == $distinct_word_keys[$i]) {
$word_iterator_map[$index] = $i;
}
}
}
$num_disallow_keys = count($disallow_keys);
if ($num_disallow_keys > 0) {
for ($i = 0; $i < $num_disallow_keys; $i++) {
$actual_index_name = $index_name;
$direction = self::ASCENDING;
if (($index_name[0] == "-")) {
$actual_index_name = substr($index_name, 1);
$direction = self::DESCENDING;
}
$disallow_iterator =
new I\WordIterator($disallow_keys[$i],
$actual_index_name, false, $filter,
I\IndexBundleIterator::RESULTS_PER_BLOCK,
$direction, $ranking_factors);
$word_iterators[$num_word_keys + $i] =
new I\NegationIterator($disallow_iterator);
}
}
$num_word_keys += $num_disallow_keys;
if ($num_word_keys == 1 && $weight == 1) {
$base_iterator = $word_iterators[0];
} else {
$base_iterator = new I\IntersectIterator(
$word_iterators, $word_iterator_map, $quote_positions,
$weight);
$min_group_flag = true;
if ($save_timestamp_name == "") {
$base_iterator->sync_timer_on = true;
} else {
$base_iterator->sync_timer_on = false;
}
}
if ($save_timestamp_name != "") {
if (isset($save_point[$save_count]) &&
$save_point[$save_count] != -1) {
$base_iterator->advance($save_point[$save_count]);
} else {
$save_point[$save_count] = -1;
}
$save_count++;
}
/* A field part comes right after the part of the same
word's other key. When both are single word walks over
different keys, the union is told they are a pair, so
it can score the field key from the other key's
postings. */
$plain_iterator = end($iterators);
if (!empty($word_struct["FIELD_PART"]) &&
$base_iterator instanceof I\WordIterator &&
$plain_iterator instanceof I\WordIterator &&
!isset($field_parts[count($iterators) - 1]) &&
$plain_iterator->word_key != $base_iterator->word_key) {
$field_parts[count($iterators)] = count($iterators) - 1;
}
$iterators[] = $base_iterator;
$part_words[] = $word_struct["WORDS"] ?? [];
}
}
$num_iterators = count($iterators); //if network_flag should be 1
if ($num_iterators < 1) {
return null;
} elseif ($num_iterators == 1) {
$union_iterator = $iterators[0];
} else {
$actual_index_name = $index_name;
if (($index_name[0] == "-")) {
$actual_index_name = substr($index_name, 1);
}
$index = IndexManager::getIndex($actual_index_name);
$index_info = $index->getArchiveInfo($index->dir_name);
$union_iterator = new I\UnionIterator($iterators,
$actual_index_name, $part_words, $field_parts);
}
$raw = intval($raw);
if ($raw > 0) {
$group_iterator = $union_iterator;
} else {
$group_iterator =
new I\GroupIterator($union_iterator, $total_iterators,
$this->current_machine);
}
if ($network_flag) {
$union_iterator->results_per_block =
$group_iterator->results_per_block;
} elseif ($save_timestamp_name != "") {
$group_iterator->save_iterators = $iterators;
} elseif ($min_group_flag && !$min_group_override) {
$group_iterator->results_per_block = max(
C\p('MIN_RESULTS_TO_GROUP')/20, 1);
$union_iterator->results_per_block =
$group_iterator->results_per_block;
$to_retrieve = -1;
}
return $group_iterator;
}
}
- Yioop Repo - Seekquarry