<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\executables;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\controllers\SearchController;
if (php_sapi_name() != 'cli' ||
defined("seekquarry\\yioop\\configs\\IS_OWN_WEB_SERVER")) {
echo "BAD REQUEST"; exit();
}
/** so can output plans */
define("seekquarry\\yioop\\configs\\QUERY_STATISTICS", true);
/** Loads common constants for web crawling*/
require_once __DIR__."/../library/LocaleFunctions.php";
if (!C\PROFILE) {
echo "Please configure the search engine instance by visiting" .
"its web interface on localhost.\n";
exit();
}
/*
* We'll set up multi-byte string handling to use UTF-8
*/
mb_internal_encoding("UTF-8");
mb_regex_encoding("UTF-8");
/**
* QueryTool runs a Yioop search from the command line against an index
* and prints the results, the query plan or a cached page, with the
* query statistics. Run with no arguments, it prints how to use it.
*
* @author Chris Pollett
*/
class QueryTool implements CrawlConstants
{
/**
* __construct makes a QueryTool; it needs no set-up.
*/
public function __construct()
{
}
/**
* start runs the search the command line arguments give and prints
* what was asked for: results, the plan or a cached page.
*/
public function start()
{
global $argv;
if (!isset($argv[1])) {
$this->usageMessageAndExit();
}
$_SERVER['NO_LOGGING'] = true;
$query = $argv[1];
$results_per_page = (isset($argv[2])) ? (is_numeric($argv[2]) ?
$argv[2] : 10 ) : 10;
$limit = (isset($argv[3]) && is_numeric($argv[3])) ? $argv[3] : 0;
L\setLocaleObject((isset($argv[4])) ? $argv[4] : C\p('DEFAULT_LOCALE'));
$start_time = microtime(true);
/* If a query hangs, the user can press Ctrl-C. The tool then
prints where the query was and the times it has so far.
Windows does not have this signal. There, Ctrl-C just ends the
tool. */
if (function_exists("pcntl_signal")) {
pcntl_async_signals(true);
pcntl_signal(SIGINT, function () use ($start_time) {
$this->showWhereStopped($start_time);
exit(1);
});
}
$controller = new SearchController();
if (!empty($argv[2]) && $argv[2] == "cache") {
$this->printCacheLookup($controller, $query);
$data = $controller->cacheRequest($query);
var_dump($data);
exit();
}
ini_set("memory_limit", C\INDEX_FILE_MEMORY_LIMIT);
$data = $controller->queryRequest($query, $results_per_page, $limit);
if (isset($argv[2]) && ($argv[2] == "plan" || $argv[2] == "explain")) {
echo "\n" . $controller->model("phrase")->db->query_log[0]["PLAN"]
."\n";
exit();
}
if (!isset($data['PAGES'])) {
$data['PAGES'] = [];
}
foreach ($data['PAGES'] as $page) {
echo "============\n";
echo "TITLE: ". trim($page[self::TITLE]). "\n";
echo "URL: ". trim($page[self::URL]). "\n";
echo "IPs: ";
if (isset($page[self::IP_ADDRESSES])) {
foreach ($page[self::IP_ADDRESSES] as $address) {
echo $address." ";
}
}
echo "\n";
echo "DESCRIPTION: " . wordwrap(trim($page[self::DESCRIPTION])) .
"\n";
echo "Language: " . ($page[self::LANG] ?? "none stored") .
"\n";
echo "Encoding: " . ($page[self::ENCODING] ?? "none stored") .
"\n";
echo "Rank: " . $page[self::DOC_RANK] . "\n";
echo "Relevance: " . $page[self::RELEVANCE] . "\n";
echo "Score: " . $page[self::SCORE] . "\n";
if (!empty($page[self::PINNED])) {
echo "This was a pinned result\n";
}
echo "============\n\n";
}
$data['ELAPSED_TIME'] = L\changeInMicrotime($start_time);
echo "QUERY STATISTICS\n";
echo "============\n";
echo "ELAPSED TIME: ".$data['ELAPSED_TIME']."\n";
foreach (L\AnalyticsManager::timingsReport() as $timing_line) {
echo $timing_line . "\n";
}
if (isset($data['LIMIT'])) {
echo "LOW: ".$data['LIMIT']."\n";
}
if (isset($data['HIGH'])) {
echo "HIGH: ".min($data['TOTAL_ROWS'],
$data['LIMIT'] + $data['RESULTS_PER_PAGE'])."\n";
}
if (isset($data['TOTAL_ROWS'])) {
echo "TOTAL ROWS: ".$data['TOTAL_ROWS']."\n";
}
if (isset($data['ERROR'])) {
echo $data['ERROR']."\n";
}
$this->printHomePageScores($controller, $query);
}
/**
* printHomePageScores prints, for each word of a query that is not a
* meta word, where the home page of the site named by that word stands
* in the query's results: https://www.word.com/ and https://word.com/.
* A search for a company's name is most often a search for its site,
* so when that page is in the index but not near the top, this shows
* its place, doc rank, relevance and score. The query is run again for
* every result it scores, MIN_RESULTS_TO_GROUP of them, so the scores
* are the ones the shown results were ranked by. start calls it after
* the statistics.
*
* @param SearchController $controller used to run the search again
* @param string $query the query whose results were printed
*/
public function printHomePageScores($controller, $query)
{
$home_pages = [];
foreach (preg_split('/\s+/', trim($query)) as $word) {
if (preg_match('/^[\p{L}\p{N}\-]+$/u', $word)) {
$word = mb_strtolower($word);
$home_pages[] = "https://www.$word.com/";
$home_pages[] = "https://$word.com/";
}
}
if (empty($home_pages)) {
return;
}
$all_scored = intval(C\p('MIN_RESULTS_TO_GROUP'));
$found = $controller->queryRequest($query, $all_scored, 0);
$pages = $found['PAGES'] ?? [];
echo "SITE HOME PAGES OF THE QUERY'S WORDS\n";
echo "============\n";
foreach ($home_pages as $url) {
$place = 0;
foreach ($pages as $index => $page) {
if (trim($page[self::URL] ?? "") == $url) {
$place = $index + 1;
break;
}
}
if ($place == 0) {
echo "$url: not among the " . count($pages) .
" results returned\n";
continue;
}
$page = $pages[$place - 1];
echo "$url: place $place of " . count($pages) .
" results returned, Rank " .
$page[self::DOC_RANK] . ", Relevance " .
$page[self::RELEVANCE] . ", Score " . $page[self::SCORE] .
"\n";
}
}
/**
* printCacheLookup prints, for every crawl on this machine, what the
* cache of $url leads to, before the page the cache command chooses is
* printed, so that a cache that returns the wrong page can be traced.
* The cache command takes the current index and then moves to the
* nearest crawl holding the url, so every crawl is a candidate. For
* each crawl it prints the document key and partition the info: meta
* word of $url leads to, and the url and title of the summary stored
* under that key. A stored url that differs from $url means the meta
* word leads to the wrong document in that crawl.
* @param SearchController $controller used to reach the crawl model
* @param string $url whose cached page is being looked up
*/
public function printCacheLookup($controller, $url)
{
$crawl_model = $controller->model("crawl");
echo "Current index: " .
$crawl_model->getCurrentIndexDatabaseName() . "\n";
$queue_servers = $controller->model("machine")->getQueueServerUrls();
$crawls = $crawl_model->getCrawlList(false, true, $queue_servers);
foreach ($crawls as $crawl) {
$index_name = $crawl['CRAWL_TIME'] ?? "";
if (empty($index_name)) {
continue;
}
$lookup = $crawl_model->lookupSummaryOffsetGeneration(
mb_strtolower($url), $index_name);
if (!is_array($lookup)) {
echo "Crawl $index_name: no document for the url\n";
continue;
}
list($doc_key, $partition) = $lookup;
$summary = L\IndexManager::getIndex($index_name)->getSummary(
$doc_key, $partition);
echo "Crawl $index_name: partition $partition, document key " .
L\toHexString($doc_key) . "\n stored url " .
($summary[CrawlConstants::URL] ?? "none") .
"\n stored title " .
($summary[CrawlConstants::TITLE] ?? "none") . "\n";
}
}
/**
* showWhereStopped prints, when the query is stopped with the
* keyboard's control and C keys together, how
* long it ran, the calls it was in, innermost first, with file and
* line, and the query statistics timed so far. It shows where a query
* that never finishes spends its time.
*
* @param float $start_time when the query started, as microtime gives it
*/
public function showWhereStopped($start_time)
{
echo "\nSTOPPED after " . L\changeInMicrotime($start_time) .
" seconds, in:\n";
foreach (debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS) as $frame) {
$place = basename($frame['file'] ?? "") . ":" .
($frame['line'] ?? "");
echo " " . ($frame['class'] ?? "") . ($frame['type'] ?? "") .
($frame['function'] ?? "") . " at $place\n";
}
echo "QUERY STATISTICS SO FAR\n";
foreach (L\AnalyticsManager::timingsReport() as $timing_line) {
echo $timing_line . "\n";
}
}
/**
* usageMessageAndExit prints how to use QueryTool, then ends the
* program.
*/
public function usageMessageAndExit()
{
echo "\nQueryTool.php is used to run a Yioop";
echo " query from the command line.\n For example,\n";
echo " php QueryTool.php 'chris pollett' \n returns results ".
"from the default index of a search on 'chris pollett'.\n";
echo "The general command format is:\n";
echo " php QueryTool.php query num_results start_num lang_tag\n\n";
echo "QueryTool.php can also be used to explain the plan by which\n";
echo "Yioop will compute query results. For this usage one types:\n";
echo " php QueryTool.php query plan\n";
echo "or\n";
echo " php QueryTool.php query explain\n";
echo "For example,";
echo " php QueryTool.php 'chris pollett' explain\n";
echo "Finally QueryTool.php can also be used to retrieve a cache of ".
"a page via the syntax";
echo " php QueryTool.php url_of_cached_page cache\n";
exit();
}
}
$query_tool = new QueryTool();
$query_tool->start();
- Yioop Repo - Seekquarry