<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\executables;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\data_structures\BloomFilterFile;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\storage_formats\IndexArchiveBundle;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\IndexManager;
use seekquarry\yioop\library\storage_formats\IndexShard;
use seekquarry\yioop\library\data_structures\PackedTableTools;
use seekquarry\yioop\library\storage_formats\PartitionDocumentBundle;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\processors as P;
use seekquarry\yioop\library\indexing_plugins\WordfilterPlugin;
use seekquarry\yioop\library\storage_formats\WebArchiveBundle;
use seekquarry\yioop\library\media_jobs\FeedsUpdateJob;
use seekquarry\yioop\controllers\AdminController;
use seekquarry\yioop\models as M;
use seekquarry\yioop\models\ImpressionModel;
if (php_sapi_name() != 'cli' ||
defined("seekquarry\\yioop\\configs\\IS_OWN_WEB_SERVER")) {
echo "BAD REQUEST"; exit();
}
/** This tool does not need logging*/
$_SERVER["LOG_TO_FILES"] = false;
/** USE_CACHE false rules out file cache as well*/
$_SERVER["USE_CACHE"] = false;
/** For crawlHash, crawlHashWord function */
require_once __DIR__."/../library/Utility.php";
require_once C\BASE_DIR . '/configs/Config.php';
if (!C\PROFILE) {
echo "Please configure the search engine instance by visiting" .
"its web interface on localhost.\n";
exit();
}
ini_set("memory_limit", C\ARC_TOOL_MEMORY_LIMIT); /*reading in a whole
shard might take a fair bit of memory
*/
/*
* We'll set up multi-byte string handling to use UTF-8
*/
mb_internal_encoding("UTF-8");
mb_regex_encoding("UTF-8");
/**
* Command line program that allows one to examine the content of
* the WebArchiveBundles and IndexArchiveBundles of Yioop crawls.
* To see all of the available command run it from the command line
* with a syntax like:
*
* php ArcTool.php
*
* @author Chris Pollett (non-yioop archive code derived from earlier
* stuff by Shawn Tice)
*/
class ArcTool extends DictionaryUpdater implements CrawlConstants
{
/**
* The maximum number of documents the ArcTool list function
* will read into memory in one go.
*/
const MAX_BUFFER_DOCS = 200;
/**
* The maximum number of documents the ArcTool will rebuild/migrate in
* one go
*/
const MAX_REBUILD_DOCS = 8000;
/**
* KEPT_LOG_NAME ends the name of the file in the log folder where a
* rebuild or remerge keeps the whole log of each DictionaryUpdater
* process slot, the slot's number coming first. The terminal shows only
* what the rebuild is doing and where it may safely continue from;
* the processes' own lines go to these files.
*/
const KEPT_LOG_NAME = "-DictionaryUpdaterRebuild.log";
/**
* $rebuild_progress holds the rebuild pool's latest progress line, how
* far building and merging have got, so that the line reportRebuildEnd
* prints when ArcTool stops says where the rebuild was.
* @var string
*/
public $rebuild_progress = "";
/**
* $rebuild_stop_reason holds why the rebuild pool stopped itself, set
* just before it exits, or why a signal stopped it, so that the line
* reportRebuildEnd prints names the reason. It stays empty when the
* rebuild ends by finishing.
* @var string
*/
public $rebuild_stop_reason = "";
/**
* Initializes the ArcTool, for now does nothing
*/
public function __construct()
{
}
/**
* Runs the ArcTool on the supplied command line arguments
*/
public function start()
{
global $argv;
if (!isset($argv[1]) || (!isset($argv[2]) &&
!in_array($argv[1], ['list', 'user-impressions'], true)) ||
(in_array($argv[1], ['dict', 'inject', 'make-filter'], true)
&& !isset($argv[3]) ) || ($argv[1] === 'doc-lookup'
&& !isset($argv[4]) ) ) {
$this->usageMessageAndExit();
}
if (!in_array($argv[1], [ "check-filter", "make-filter", "list",
"test-page"])) {
if (!in_array($argv[1], ["inject"])) {
/* An index may be named by its folder's full path as well
as by its name in the cache or archives folder. */
$path = is_dir($argv[2]) ? $argv[2] :
UrlParser::getDocumentFilename($argv[2]);
if ($path == $argv[2] && !file_exists($path)) {
$path = C\CACHE_DIR . "/" . $path;
if (!file_exists($path)) {
$path = C\ARCHIVES_DIR . "/" . $argv[2];
}
}
$kind = $this->getArchiveKind($path);
if ($kind == "DoubleIndexBundle" && !in_array(
$argv[1], ["info", "migrate"]) ) {
$bundle_index = $argv[3];
unset($argv[3]);
$argv = array_values($argv);
$path .= "-$bundle_index";
}
} else if (is_numeric($argv[2])) {
$path = $argv[2];
} else {
$this->usageMessageAndExit();
}
}
switch ($argv[1]) {
case "check-filter":
$this->checkFilter($argv[2], $argv[3]);
break;
case "count":
$argv[3] ??= false;
$this->outputCountBundle($path, $argv[3]);
break;
case "doc-lookup":
$this->outputDocLookup($argv[2], intval($argv[3]),
intval($argv[4]));
break;
case "test-page":
$this->outputTestPage($argv[2], $argv[3] ?? "");
break;
case "dict":
if (isset($argv[4]) && $argv[4] == 'details') {
$argv[4] = -1;
$argv[5] = -1;
$argv[6] = "details";
}
$argv[4] = (isset($argv[4])) ? intval($argv[4]) : -1;
$argv[5] = (isset($argv[5])) ? intval($argv[5]) : -1;
$argv[6] = (!empty($argv[6]) && $argv[6] == "details") ?
true : false;
$this->outputDictInfo($path, $argv[3], $argv[4], $argv[5],
$argv[6]);
break;
case "info":
$this->outputInfo($path);
break;
case "inject":
$this->inject($path, $argv[3]);
break;
case "list":
$this->outputArchiveList();
break;
case "make-filter":
$argv[4] ??= -1;
$this->makeFilter($argv[2], $argv[3], $argv[4]);
break;
case "migrate":
$argv[3] ??= 1;
$this->migrateIndexArchive($path, $argv[3]);
break;
case "partition":
$argv[3] ??= 0;
$this->outputPartitionInfo($path, $argv[3]);
break;
case "fix-partition":
$argv[3] ??= 0;
$argv[4] ??= -1;
$this->fixPartitionIndexes($path, $argv[3], $argv[4]);
break;
case "rebuild":
case "remerge":
$this->startRebuildInBackground($argv);
break;
case "rebuild-status":
echo $this->rebuildStatus($path) . "\n";
break;
case "rebuild-in-background":
case "remerge-in-background":
$argv[3] ??= 0;
$argv[4] ??= 1;
$argv[5] ??= -1;
$this->rebuildIndexBundle($path, $argv[3],
$argv[1] == "rebuild-in-background", $argv[4], $argv[5]);
break;
case "show":
if (!isset($argv[3])) {
$this->usageMessageAndExit();
}
$argv[4] ??= 1;
$this->outputShowPages($path, $argv[3], $argv[4]);
break;
case "user-impressions":
$key = isset($argv[2]) ? (string)$argv[2] : null;
$partition = isset($argv[3]) ? (int)$argv[3] : null;
$limit = isset($argv[4]) ? (int)$argv[4] : null;
$this->displayLogRecords($key, $partition, $limit);
break;
case "item-impressions":
$itemId = isset($argv[2]) ? (int)$argv[2] : null;
$partition = isset($argv[3]) ? (int)$argv[3] : null;
$limit = isset($argv[4]) ? (int)$argv[4] : null;
if ($itemId === null) {
$this->usageMessageAndExit();
}
$this->displayLogRecordsByItemId($itemId, $partition, $limit);
break;
case "user-items":
if (!isset($argv[2]) || !is_numeric($argv[2])) {
$this->usageMessageAndExit();
}
$this->getItemIdByUser((int)$argv[2]);
break;
case "group-items":
if (!isset($argv[2]) || !is_numeric($argv[2])) {
$this->usageMessageAndExit();
}
$this->getItemIdByGroup((int)$argv[2]);
break;
case "type-items":
if (!isset($argv[2])) {
$this->usageMessageAndExit();
}
$this->getItemIdByType($argv[2]);
break;
default:
$this->usageMessageAndExit();
}
}
/**
* Lists the Web or IndexArchives in the crawl directory
*/
public function outputArchiveList()
{
$yioop_pattern = C\CACHE_DIR . "/{" .
self::double_index_base_name . "," . self::index_data_base_name .
"}*";
$archives = glob($yioop_pattern, GLOB_BRACE);
$archives_found = false;
if (is_array($archives) && count($archives) > 0) {
$archives_found = true;
echo "\nFound Yioop Archives:\n";
echo "=====================\n";
foreach ($archives as $archive_path) {
$name = $this->getArchiveName($archive_path);
echo $name . " ";
$archive_type = $this->getArchiveKind($archive_path);
if (in_array($archive_type, ["FeedArchiveBundle",
"DoubleIndexBundle", "IndexArchiveBundle",])) {
$bundle_class = C\NS_STORAGE_FORMATS . $archive_type;
$info = $bundle_class::getArchiveInfo($archive_path);
if (in_array(substr($info["DESCRIPTION"], 0, 2),
['s:', 'a:'])) {
$info = unserialize($info["DESCRIPTION"]);
}
echo $info["DESCRIPTION"];
}
echo "\n";
}
}
$nonyioop_pattern = C\ARCHIVES_DIR . "/*/arc_description.ini";
$archives = glob($nonyioop_pattern);
if (is_array($archives) && count($archives) > 0 ) {
$archives_found = true;
echo "\nFound Non-Yioop Archives:\n";
echo "=========================\n";
foreach ($archives as $archive_path) {
$len = strlen("/arc_description.ini");
$path = substr($archive_path, 0, -$len);
echo $this->getArchiveName($path)."\n";
}
}
if (!$archives_found) {
echo "No archives currently in crawl directory \n";
}
echo "\n";
}
/**
* Determines whether the supplied path is a WebArchiveBundle,
* an IndexArchiveBundle, DoubleIndexBundle, or non-Yioop Archive.
* Then outputs to stdout header information about the
* bundle by calling the appropriate sub-function.
*
* @param string $archive_path The path of a directory that holds
* WebArchiveBundle,IndexArchiveBundle, or non-Yioop archive data
*/
public function outputInfo($archive_path)
{
$bundle_name = $this->getArchiveName($archive_path);
echo "Bundle Name: " . $bundle_name."\n";
$archive_type = $this->getArchiveKind($archive_path);
echo "Bundle Type: " . $archive_type."\n";
if ($archive_type === false) {
$this->badFormatMessageAndExit($archive_path);
}
if (in_array($archive_type, ["DoubleIndexBundle", "FeedDocumentBundle",
"IndexDocumentBundle",])) {
$call = "outputInfo" . $archive_type;
$archive_name = C\NS_STORAGE_FORMATS . $archive_type;
$info = $archive_name::getArchiveInfo($archive_path);
$this->$call($info, $archive_path);
}
}
/**
* Outputs how a crawl would process the page at $uri, as the Test Page
* screen under Page Options does for a uri: the page is fetched with
* the crawler's own FetchUrl, or read from disk for a file:/// uri, then
* handed to the page processor for its mime type, with the built-in
* web scrapers for HTML. The summary the processor makes is printed
* field by field, then whether the word filter plugin's saved rules,
* or its Factory Settings when none are saved, keep the page, then the
* words and meta words the summary would be indexed under.
*
* @param string $uri http, https or file:/// address of the page
* @param string $mime_type mime type to process the page as; when empty,
* the type the server sent, or for a file the type its name suggests
*/
public function outputTestPage($uri, $mime_type = "")
{
P\PageProcessor::initializeIndexedFileTypes();
if (str_starts_with($uri, "file://")) {
$file_name = substr($uri, strlen("file://"));
if (!is_file($file_name)) {
echo "No file at $file_name\n";
return;
}
$site = [self::URL => $uri, self::HTTP_CODE => 200,
self::PAGE => file_get_contents($file_name, false, null, 0,
C\PAGE_RANGE_REQUEST), self::TYPE =>
UrlParser::guessMimeTypeFromFileName($file_name)];
} else {
$site = FetchUrl::getPages([[self::URL => $uri]], true,
C\PAGE_RANGE_REQUEST, C\TEMP_DIR)[0];
}
$site[self::TYPE] = ($mime_type == "") ? strtolower(trim(explode(";",
$site[self::TYPE] ?? "text/html")[0])) : $mime_type;
if ($site[self::TYPE] == "text/html" &&
empty($site[self::ENCODING])) {
$site[self::ENCODING] = L\guessEncodingHtmlXml(
$site[self::PAGE] ?? "");
}
echo "URI: $uri\nHTTP code: " . ($site[self::HTTP_CODE] ?? "none") .
"\nMime type: {$site[self::TYPE]}\n";
L\convertUtf8IfNeeded($site, self::PAGE, self::ENCODING);
$processor_name = P\PageProcessor::$mime_processor[
$site[self::TYPE]] ?? "";
if ($processor_name == "") {
echo "No page processor handles {$site[self::TYPE]}\n";
return;
}
$processor_name = C\NS_PROCESSORS . $processor_name;
$page_processor = new $processor_name([], C\MAX_DESCRIPTION_LEN);
if (L\generalIsA($processor_name, C\NS_PROCESSORS .
"HtmlProcessor")) {
$page_processor->scrapers = (new M\ScraperModel()
)->getAllScrapers();
}
$summary = $page_processor->handle(substr($site[self::PAGE] ?? "",
0, C\PAGE_RANGE_REQUEST), $uri);
echo "Processor: " . substr(strrchr($processor_name, "\\"), 1) .
"\n\nSUMMARY\n";
if (empty($summary)) {
echo "The processor made no summary.\n";
return;
}
$names = array_flip((new \ReflectionClass(CrawlConstants::class)
)->getConstants());
foreach ($summary as $field => $value) {
$value = is_array($value) ? count($value) . " item(s): " .
json_encode(array_slice($value, 0, 5, true)) : $value;
echo ($names[$field] ?? $field) . ": " .
mb_substr(preg_replace("/\s+/u", " ", (string)$value), 0,
300) . "\n";
}
$word_filter = new WordfilterPlugin();
$word_filter->rules_string = $word_filter->default_rules_string;
$word_filter->setConfiguration($word_filter->loadConfiguration());
$filtered = $summary;
$word_filter->pageSummaryProcessing($filtered, $uri);
echo "\nWord filter: " . ($filtered === false ?
"drops the page (NOPROCESS)" : "keeps the page") . "\n";
$lang = $summary[self::LANG] ?? "en-US";
$word_lists = PhraseParser::extractPhrasesInLists(
($summary[self::TITLE] ?? "") . " " .
($summary[self::DESCRIPTION] ?? ""), $lang);
echo "\nWORDS (" . count($word_lists['WORD_LIST'] ?? []) . "): " .
implode(" ", array_slice(array_keys($word_lists['WORD_LIST'] ??
[]), 0, 80)) . "\n";
}
/**
* Outputs the $doc_map_index'th document from the $partition partition
* of the IndexDocumentBundle in folder $index_name
*
* @param string $index_name folder containing an IndexDocumentBundle
* DoubleIndexBundle or FeedDocumentBundle
* @param int $partition which partition to do the lookup in
* @param int $doc_map_index index of which document to lookup
* @return array associative array of field => values associated with
* document
*/
public function outputDocLookup($index_name, $partition, $doc_map_index)
{
if (substr($index_name, 0, strlen(self::index_data_base_name)) ==
self::index_data_base_name) {
$index_name = substr($index_name,
strlen(self::index_data_base_name));
}
if (substr($index_name, 0, strlen(self::double_index_base_name)) ==
self::double_index_base_name) {
$index_name = substr($index_name,
strlen(self::double_index_base_name));
}
$index = IndexManager::getIndex($index_name);
$base_folder = $index->getPartitionBaseFolder($partition);
$doc_map_filename = $base_folder . "/" .
IndexDocumentBundle::DOC_MAP_FILENAME;
$doc_map_tools = $index->doc_map_tools;
$entry = $doc_map_tools->findEntryAtIndexTableName($doc_map_filename,
$doc_map_index);
$docid_len = IndexDocumentBundle::DOCID_LEN;
$termsfilter_len = IndexDocumentBundle::TERMSFILTER_LEN;
$doc_key = substr($entry, 0, $docid_len);
$entry = (strlen($entry) >= ($docid_len + $termsfilter_len + 1) &&
$entry[$docid_len] == 't') ?
substr($entry, $docid_len + $termsfilter_len + 1) :
substr($entry, $docid_len);
$doc_map_tools = $index->doc_map_tools;
echo "Doc Key: " . L\toHexString($doc_key) . "\n";
echo "Partition: $partition\n";
echo "Doc Map Entries\n------------\n";
$doc_info = $doc_map_tools->unpack($entry);
$first_row = array_shift($doc_info);
$second_row = array_shift($doc_info);
if (!empty($first_row)) {
list($num_words, $score) = array_values($first_row);
echo "Number of Words in Document: $num_words\n";
echo "Document Score: $score\n";
}
$num_scores = count($doc_info);
if (!empty($second_row)) {
list($title_length, $num_description_scores) = array_values(
$second_row);
echo "Title Length $title_length\n";
}
$score_index = 0;
if ($num_description_scores > 0) {
echo "Description Scores\n-------------------\n";
while ($score_index < $num_description_scores) {
$doc_row = $doc_info[$score_index];
$score_index++;
echo "Pos: " . $doc_row['POS'] .
" Score: ". $doc_row['SCORE']. "\n";
}
}
if ($score_index < $num_scores) {
echo "User Rank Scores\n-------------------\n";
while ($score_index < $num_scores) {
$doc_row = $doc_info[$score_index];
echo "Score Index: " . $score_index .
" Score: ". $doc_row['SCORE'] . "\n";
$score_index++;
}
}
echo "Doc:\n----\n";
$document = $index->documents->get($doc_key, $partition);
var_dump($document);
}
/**
* Prints the dictionary records for a word in an IndexDocumentBundle
*
* @param string $archive_path the path of a directory that holds
* an IndexArchiveBundle
* @param string $word to look up dictionary record for
* @param int $start_record first record to list out
* @param int $num_records max records to list our
* @param bool $details whether to show posting list details or not
*/
public function outputDictInfo($archive_path, $word, $start_record,
$num_records, $details)
{
$bundle_index = -1;
if (preg_match("/\-\d$/", $archive_path)) {
$bundle_index = substr($archive_path, -1);
$archive_path = substr($archive_path, 0, -2);
}
$bundle_name = $this->getArchiveName($archive_path);
echo "\nBundle Name: $bundle_name\n";
$archive_type = $this->getArchiveKind($archive_path);
echo "Bundle Type: $archive_type\n";
if (!in_array($archive_type, ["FeedDocumentBundle",
"DoubleIndexBundle", "IndexDocumentBundle",])) {
$this->badFormatMessageAndExit($archive_path, "index");
}
preg_match("/\d+$/", $archive_path, $matches);
$index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
if (isset($bundle_index) && $bundle_index >= 0) {
$index_timestamp .= "-$bundle_index";
} else if ($bundle_name == "IndexDataFeed") {
$index_timestamp = "feed";
}
$hash_key = L\canonicalTerm($word);
$start_time = microtime(true);
echo "Looking up in dictionary:\n";
echo " Key: " . $hash_key . "\n";
echo " Key (in Hex): ". L\toHexString($hash_key) . "\n";
$info = IndexManager::getWordInfo($index_timestamp, $hash_key,
-1);
$index = IndexManager::getIndex($index_timestamp);
echo "Dictionary Lookup Time:" . L\changeInMicrotime($start_time)
. "\n";
if (!$info) {
echo " Key not found\n";
exit();
}
$found = true;
echo "\nBundle Dictionary Entries for '$word':\n";
echo "====================================\n";
$i = 0;
if ($index->archive_info['VERSION'] < "4.0") {
echo "Cannot compute results for non-migrated indexes.\n";
echo "Please use ArcTool's migrate command to upgrade index\n";
}
foreach ($info['ROWS'] as $record) {
if ($start_record < 0 || $record['PARTITION'] >= $start_record) {
echo "RECORD: $i\n";
echo "PARTITION: {$record['PARTITION']}\n";
echo "NUMBER OF DOCS: {$record['NUM_DOCS']}\n";
$postings_offset = (empty($record['POSTINGS_OFFSET'])) ?
-1: $record['POSTINGS_OFFSET'];
echo "POSTINGS_OFFSET: $postings_offset\n";
$postings_len = (empty($record['POSTINGS_LEN']))?
-1 : $record['POSTINGS_LEN'];
echo "POSTINGS_LEN: $postings_len\n\n";
$is_postings_array = isset($record['POSTINGS']) &&
is_array($record['POSTINGS']);
if ($postings_offset == -1 && !$is_postings_array) {
echo "No POSTINGS_LEN or LAST_BLOB_LEN, record postings".
" cannot be looked up\n";
} else if ($is_postings_array) {
echo "Active Record\n-------------\n";
echo "By default list details of all postings\n";
var_dump($record['POSTINGS']);
} else if ($details) {
$postings_entry = $index->getPostingsString(
$record['PARTITION'], $postings_offset,
$postings_len);
$postings = $index->postings_tools->unpack($postings_entry);
$index->deDeltaPostingsSumFrequencies($postings);
var_dump($postings);
}
}
$i++;
if ($num_records > 0 && $i >= $start_record + $num_records) {
break;
}
}
}
/**
* Prints information about the number of words and frequencies of words
* within the $index'th partition in the bundle
*
* @param string $archive_path the path of a directory that holds
* an IndexDocumentBundle
* @param int $num of partition to show info for
*/
public function outputPartitionInfo($archive_path, $num)
{
if (preg_match("/\-\d$/", $archive_path)) {
$bundle_index = substr($archive_path, -1);
$archive_path = substr($archive_path, 0, -2);
}
$bundle_name = $this->getArchiveName($archive_path);
echo "\nBundle Name: $bundle_name\n";
$archive_type = $this->getArchiveKind($archive_path);
echo "Bundle Type: $archive_type\n";
if (!in_array($archive_type, ["FeedArchiveBundle",
"DoubleIndexBundle", "IndexDocumentBundle",])) {
$this->badFormatMessageAndExit($archive_path, "index");
}
preg_match("/\d+$/", $archive_path, $matches);
$index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
if (isset($bundle_index) && $bundle_index >= 0) {
$index_timestamp .= "-$bundle_index";
} else if ($bundle_name == "IndexDataFeed") {
$index_timestamp = "feed";
}
$index = IndexManager::getIndex($index_timestamp);
$partition_bundle = $index->documents;
$num_partitions = $partition_bundle->parameters["SAVE_PARTITION"] ?? -1;
$num_partitions++;
echo "Number of Partitions: $num_partitions\n";
echo "\nInformation for Partition $num\n";
echo "====================================\n";
$_SERVER["NO_LOGGING"] = true;
$partition = $partition_bundle->loadPartitionIndex($num);
if (!$partition) {
echo "Partition $num does not exists\n";
return;
}
$statistics =
$index->buildInvertedIndexPartition($num, null, true);
echo "Number of Distinct Terms Indexed: " .
count($statistics['TERM_STATISTICS'])."\n";
echo "Number of Docs in Shard: " . $statistics['NUM_DOCS'] ."\n";
echo "Number of Link Items in Shard: ".$statistics['NUM_LINKS']."\n";
echo "Total Links and Docs: ".($statistics['NUM_DOCS'] +
$statistics['NUM_LINKS'])."\n\n";
echo "Term histogram for shard\n";
echo "------------------------\n";
$term_frequencies = $statistics['TERM_STATISTICS'];
arsort($term_frequencies);
$i = 1;
echo "Freq Rank\t# Term\t# Docs Term Appears In\n";
foreach ($term_frequencies as $term => $num_docs) {
$term = rtrim($term, '_');
$term = str_replace("3A", ":", $term);
echo "$i\t\t\t$term\t\t\t$num_docs\n";
$i ++;
}
}
/**
* Recomputes the hash index (.ix) files for a range of partitions
* from start_partition to end_partition in the documents subfolder of
* an IndexDocumentBundle. An ix file contains a sequence of compressed
* 4-tuple (doc_id, summary_offset, summary_length, cache_length)
* corresponding to a partition file (these end in .txt.gz and are
* a sequence of compressed document summaries followed by orginal
* documents).
* @param string $archive_path the path of a directory that holds
* an IndexDocumentBundle
* @param int $start_partition first partition to recompute
* @param int $end_partition last partition to recompute (inclusive)
*/
public function fixPartitionIndexes($archive_path, $start_partition,
$end_partition = -1)
{
if (preg_match("/\-\d$/", $archive_path)) {
$bundle_index = substr($archive_path, -1);
$archive_path = substr($archive_path, 0, -2);
}
$bundle_name = $this->getArchiveName($archive_path);
echo "\nBundle Name: $bundle_name\n";
$archive_type = $this->getArchiveKind($archive_path);
echo "Bundle Type: $archive_type\n";
if (!in_array($archive_type, ["FeedArchiveBundle",
"DoubleIndexBundle", "IndexDocumentBundle",])) {
$this->badFormatMessageAndExit($archive_path, "index");
}
preg_match("/\d+$/", $archive_path, $matches);
$index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
if (isset($bundle_index) && $bundle_index >= 0) {
$index_timestamp .= "-$bundle_index";
} else if ($bundle_name == "IndexDataFeed") {
$index_timestamp = "feed";
}
$index = IndexManager::getIndex($index_timestamp);
$partition_bundle = $index->documents;
if ($end_partition == -1) {
$end_partition = $partition_bundle->parameters["SAVE_PARTITION"];
}
$partition_tools = new PackedTableTools(
["PRIMARY KEY" => ["DOC_ID", IndexDocumentBundle::DOCID_LEN],
"SUMMARY_OFFSET" => "INT", "SUMMARY_LENGTH" => "INT",
"CACHE_PAGE_LENGTH" => "INT"]);
$buffer_size = 1000000;
$gz_start = '/\x1F\x8B\x08\x00\x00\x00\x00.../';
for ($i = $start_partition; $i <= $end_partition; $i++) {
$partition_filename = $partition_bundle->getPartition($i);
echo "Reconstructing Index File (.ix) for $partition_filename\n";
if (!file_exists($partition_filename)) {
echo "\nPartition File: $partition_filename does not exists!";
echo "\nStopping!";
exit();
}
$first_time = true;
$fh = fopen($partition_filename, "r");
$remainder = "";
$offset = 0;
$is_summary = true;
$cnt = 0;
$partition_index = "";
do {
$gztext = fread($fh, $buffer_size);
$objects = preg_split($gz_start, $remainder . $gztext);
$num_objects = count($objects);
$start = ($first_time) ? 1 : 0;
$first_time = false;
for ($j = $start ; $j < $num_objects; $j++) {
$compress = $objects[$j];
if ($j == $num_objects - 1 && !feof($fh)) {
$remainder = $compress;
break;
}
$len = 10 + strlen($compress);
$site_string = @gzinflate($compress);
if (empty($site_string)) {
echo "Couldn't uncompress item $cnt\n";
$offset += $len;
$last_summary = false;
continue;
}
$is_summary = (str_starts_with($site_string, 'a:')) ? true :
false;
$is_cache = (str_starts_with($site_string, 's:')) ? true :
false;
if ($is_summary) {
$site = @unserialize($site_string);
$last_summary = true;
if (empty($site)) {
echo "Couldn't unserialize item $cnt\n";
$offset += $len;
continue;
} else if (is_string($site)) {
echo "Item $cnt is a cache page not a summary.\n";
$offset += $len;
continue;
}
$doc_id = IndexDocumentBundle::computeDocId($site);
$summary_offset = $offset;
$summary_len = $len;
} else if ($is_cache && $last_summary) {
$last_summary = false;
$cache_len = $len;
if ($cnt % 1000 == 0) {
echo "...extracted and indexed $cnt items from:\n" .
"$partition_filename\n";
echo "Last Summary Offset: $summary_offset," .
" Summary Length: $summary_len, Cache Length: ".
"$cache_len\n";
}
$out_value = $partition_tools->pack([
"SUMMARY_OFFSET" => $summary_offset,
"SUMMARY_LENGTH" => $summary_len,
"CACHE_PAGE_LENGTH" => $cache_len
]);
$partition_tools->add($partition_index, $doc_id,
$out_value, $partition_tools::ADD_MEM_TABLE_STRING);
$cnt++;
} else {
$last_summary = false;
}
$offset += $len;
}
} while(!feof($fh));
$partition_index_name =
$partition_bundle->getPartitionIndex($i);
echo "Saving $partition_index_name.\n";
$partition_tools->save($partition_index_name, $partition_index);
}
}
/**
* Counts and outputs the number of docs and links in each shard
* in the archive supplied in $archive_path as well as an overall count
*
* @param string $archive_path patch of archive to count
* @param bool $set_count flag that controls whether after computing
* the count to write it back into the archive
*/
public function outputCountBundle($archive_path, $set_count = false)
{
$bundle_index = -1;
if (preg_match("/\-\d$/", $archive_path)) {
$bundle_index = substr($archive_path, -1);
$archive_path = substr($archive_path, 0, -2);
}
$bundle_name = $this->getArchiveName($archive_path);
echo "\nBundle Name: $bundle_name\n";
$archive_type = $this->getArchiveKind($archive_path);
echo "Bundle Type: $archive_type\n";
if (!in_array($archive_type, ["FeedDocumentBundle",
"DoubleIndexBundle", "IndexDocumentBundle",])) {
$this->badFormatMessageAndExit($archive_path, "index");
}
preg_match("/\d+$/", $archive_path, $matches);
$index_timestamp = (isset($matches[0])) ? $matches[0] : 0;
if (isset($bundle_index) && $bundle_index >= 0) {
$index_timestamp .= "-$bundle_index";
} else if ($bundle_name == "IndexDataFeed") {
$index_timestamp = "feed";
}
$index = IndexManager::getIndex($index_timestamp);
$partition_bundle = $index->documents;
$num_partitions = $partition_bundle->parameters["SAVE_PARTITION"] ?? -1;
$num_partitions++;
$partition_bundle->index_cache_size = 1;
if ($num_partitions <= 0) {
echo "Archive does not appear to have data yet";
exit();
} else {
echo "Number of partitions: $num_partitions\n";
}
$count = 0;
$visited_urls_count = 0;
echo "Partition Counts\n===========\n";
for ($i = 0; $i < $num_partitions; $i++ ) {
$partition = $partition_bundle->loadPartitionIndex($i);
$partition_keys = array_keys($partition);
$docs_partition = 0;
foreach($partition_keys as $partition_key) {
if (IndexDocumentBundle::isType($partition_key, 'doc')) {
$docs_partition++;
}
}
$num_links_and_docs = count($partition_keys);
echo "\nPartition:$i\n=======\n";
echo "Number of Docs in Partition: " . $docs_partition . "\n";
echo "Number of Link Items in Partition: " . $num_links_and_docs .
"\n";
$visited_urls_count += $docs_partition;
$count += $num_links_and_docs;
}
echo "\n=======\n";
echo "Total Number of Docs Seen:".$visited_urls_count."\n";
echo "Total Number of Link or Doc Items:".$count."\n";
if ($set_count == "save") {
echo "\nSaving count to bundle...\n";
$info = IndexDocumentBundle::getArchiveInfo($archive_path);
$info['COUNT'] = $count - $docs_partition;
$info['ACTIVE_COUNT'] = $docs_partition;
$info['VISITED_URLS_COUNT'] = $visited_urls_count;
IndexDocumentBundle::setArchiveInfo($archive_path, $info);
echo "..done\n";
}
}
/**
* Given a complete path to an archive returns its filename
*
* @param string $archive_path a path to a yioop or non-yioop archive
* @return string its filename
*/
public function getArchiveName($archive_path)
{
$start = C\ARCHIVES_DIR . "/";
if (strstr($archive_path, $start)) {
$start_len = strlen($start);
$name = substr($archive_path, $start_len);
} else {
$name = UrlParser::getDocumentFilename($archive_path);
}
return $name;
}
/**
* Outputs to the terminal if the bloom filter $filter_path contains
* the string $item
* @param string $filter_path name of bloom filter file to check if
* contains item
* @param string $item item to chheck in in bloom filter
*/
public function checkFilter($filter_path, $item)
{
$item = trim($item);
if (!file_exists($filter_path)) {
echo "Filter File: $filter_path does not exist.";
exit();
}
$filter = BloomFilterFile::load($filter_path);
if ($filter->contains($item)) {
echo "$item is contained in the filter\n";
} else {
echo "$item is not contained in the filter\n";
}
}
/**
* Makes a BloomFilterFile object from a dictionary file $dict_file which
* has items listed one per line, or items listed as some column of a CSV
* file. The result is output to $filter_path
* @param string $dict_file to make BloomFilterFile from
* @param string $filter_path of file to serialize BloomFilterFile to
* @param int $column_num if negative assumes $dict_file has one entry
* per line, if >=0 then is the index of the column in a csv to use
* for items
*/
public function makeFilter($dict_file, $filter_path, $column_num = -1)
{
$lines = file($dict_file);
$filter = new BloomFilterFile($filter_path, count($lines));
$i = 0;
foreach ($lines as $line) {
$item = $line;
if ($column_num != -1) {
$line_parts = explode(",", $line);
$item = $line_parts[$column_num] ?? "";
}
$item = trim($item, " \t\n\r\0\x0B\"\'");
if (!empty($item)) {
$item = mb_strtolower($item);
$i++;
if ($i % 10000 == 0) {
echo "Added $i items so far. Most recent: $item \n";
}
$filter->add($item);
}
}
$filter->save();
}
/**
* Outputs to stdout header information for a FeedDocumentBundle
* bundle.
*
* @param array $info header info that has already been read from
* the description.txt file
* @param string $archive_path file path of the folder containing the bundle
* @param string $alternate_description used as the text for description
* rather than what's given in $info
* @param bool $only_storage_info output only info about storage statistics
* don't output info about crawl parameters
* @param bool $only_crawl_params output only info about crawl parameters
* not storage statistics
*/
public function outputInfoFeedDocumentBundle($info, $archive_path,
$alternate_description = "", $only_storage_info = false,
$only_crawl_params = false)
{
$this->outputInfoIndexDocumentBundle($info, $archive_path,
$alternate_description, $only_storage_info,
$only_crawl_params);
}
/**
* Outputs to stdout header information for a IndexDocumentBundle
* bundle.
*
* @param array $info header info that has already been read from
* the description.txt file
* @param string $archive_path file path of the folder containing the bundle
* @param string $alternate_description used as the text for description
* rather than what's given in $info
* @param bool $only_storage_info output only info about storage statistics
* don't output info about crawl parameters
* @param bool $only_crawl_params output only info about crawl parameters
* not storage statistics
*/
public function outputInfoIndexDocumentBundle($info, $archive_path,
$alternate_description = "", $only_storage_info = false,
$only_crawl_params = false)
{
$more_info = unserialize($info['DESCRIPTION']);
$more_info = is_array($more_info) ? $more_info : [];
unset($info['DESCRIPTION']);
$info = array_merge($info, $more_info);
$description = ($alternate_description) ? $alternate_description :
"Description: " . $info['DESCRIPTION'];
echo "$description\n";
if (!$only_crawl_params) {
$num_partitions = $info['SAVE_PARTITION'] + 1;
echo "Number of partitions: $num_partitions \n";
echo "Maximum number of documents or links per partition:" .
$info['MAX_ITEMS_PER_FILE'] . "\n";
echo "Number of stored links and documents: " .
($info['ACTIVE_COUNT'] + $info['COUNT']) .
"\n";
echo "Number of stored documents: " . $info['VISITED_URLS_COUNT'] .
"\n";
}
if ($only_storage_info) {
return;
}
if (isset($info['active_archive'])) {
echo "Active Archive Bundle: " . $info['active_archive'] . "\n";
}
if (!empty($info['repeat_time'])) {
echo "Last Swap Time: " . date("r", $info['repeat_time']) . "\n";
}
if (!empty($info['repeat_frequency'])) {
echo "Repeat Frequency: " . $info['repeat_frequency'] .
" seconds\n";
}
$crawl_order = (isset($info[self::CRAWL_ORDER]) &&
$info[self::CRAWL_ORDER] == self::BREADTH_FIRST) ?
"Breadth First" : "Page Importance";
echo "Crawl order was: $crawl_order\n";
$channel = (isset($info[self::CHANNEL])) ? $info[self::CHANNEL] : 0;
echo "Crawl Channel was: $channel.\n";
if ($info['DESCRIPTION'] == 'feed') {
echo "Feed Bundle, look at SearchSsources in web interface to see";
echo "\n feed sources.\n";
} else {
echo "Seed sites:\n";
foreach ($info[self::TO_CRAWL] as $seed) {
echo " $seed\n";
}
if ($info[self::RESTRICT_SITES_BY_URL]) {
echo "Sites allowed to crawl:\n";
foreach ($info[self::ALLOWED_SITES] as $site) {
echo " $site\n";
}
}
echo "Sites not allowed to be crawled:\n";
if (is_array($info[self::DISALLOWED_SITES])) {
foreach ($info[self::DISALLOWED_SITES] as $site) {
echo " $site\n";
}
}
echo "Page Rules:\n";
if (isset($info[self::PAGE_RULES])) {
foreach ($info[self::PAGE_RULES] as $rule) {
echo " $rule\n";
}
}
echo "\n";
}
}
/**
* Outputs to stdout header information for a DoubleIndexBundle
* bundle.
*
* @param array $info header info that has already been read from
* the description.txt file
* @param string $archive_path file path of the folder containing the bundle
*/
public function outputInfoDoubleIndexBundle($info, $archive_path)
{
$this->outputInfoIndexArchiveBundle($info, $archive_path, "",
false, true);
$this->outputInfoIndexArchiveBundle($info, $archive_path . "/bundle0",
"Bundle 0\n=======", true);
echo "\n";
$this->outputInfoIndexArchiveBundle($info, $archive_path . "/bundle1",
"Bundle 1\n=======", true);
}
/**
* Adds a list of urls as a upcoming schedule for a given queue bundle.
* Can be used to make a closed schedule startable
*
* @param string $timestamp for a queue bundle to add urls to
* @param string $url_file_name name of file consist of urls to inject into
* the given crawl
*/
public function inject($timestamp, $url_file_name)
{
$admin = new AdminController();
$machine_urls = $admin->model("machine")->getQueueServerUrls();
$num_machines = count($machine_urls);
$new_urls = file_get_contents($url_file_name);
$inject_urls = $admin->convertStringCleanArray($new_urls);
if (!$inject_urls || count($inject_urls) == 0) {
echo "\nNo urls in $url_file_name to inject.\n\n";
exit();
}
$crawl_model = $admin->model("crawl");
$seed_info = $crawl_model->getCrawlSeedInfo($timestamp, $machine_urls);
if (!$seed_info) {
echo "\nNo queue bundle with timestamp: $timestamp.\n\n";
exit();
}
$seed_info['seed_sites']['url'][] = "#\n#". date('r')."\n#";
$seed_info['seed_sites']['url'] = array_merge(
$seed_info['seed_sites']['url'], $inject_urls);
$crawl_model->setCrawlSeedInfo($timestamp, $seed_info, $machine_urls);
$crawl_model->injectUrlsCurrentCrawl($timestamp, $inject_urls,
$machine_urls);
echo "Urls injected!";
}
/**
* Used to list out the pages/summaries stored in a bundle at
* $archive_path. It lists to stdout $num many documents starting at $start.
*
* @param string $archive_path path to bundle to list documents for
* @param int $start first document to list
* @param int $num number of documents to list
*/
public function outputShowPages($archive_path, $start, $num)
{
if (preg_match("/\-\d$/", $archive_path)) {
$bundle_index = substr($archive_path, -1);
$archive_path = substr($archive_path, 0, -2);
}
$fields_to_print = [
self::URL => "URL",
self::IP_ADDRESSES => "IP ADDRESSES",
self::TIMESTAMP => "DATE",
self::HTTP_CODE => "HTTP RESPONSE CODE",
self::TYPE => "MIMETYPE",
self::URL_PARENT => "URL OF PARENT",
self::SCRAPER_LABEL => "SCRAPER LABEL",
self::AVERAGE_COLOR => "AVERAGE_COLOR",
self::DURATION => "DURATION",
self::WIDTH => "WIDTH",
self::HEIGHT => "HEIGHT",
self::ENCODING => "CHARACTER ENCODING",
self::DESCRIPTION => "DESCRIPTION",
self::PAGE => "PAGE DATA"];
$archive_type = $this->getArchiveKind($archive_path);
if ($archive_type === false) {
$this->badFormatMessageAndExit($archive_path);
}
$nonyioop = false;
//for yioop archives we set up a dummy iterator
$iterator = (object) [];
$iterator->end_of_iterator = false;
$archive_name = C\NS_STORAGE_FORMATS . $archive_type;
if ($archive_type == "IndexDocumentBundle" ||
$archive_type == "FeedDocumentBundle") {
$info = $archive_name::getArchiveInfo($archive_path);
$num = min($num, $info["ACTIVE_COUNT"] + $info["COUNT"] - $start);
$num_generations = $info['SAVE_PARTITION'] + 1;
$index_archive = new IndexDocumentBundle($archive_path);
$archive = $index_archive->documents;
} else if ($archive_type == "DoubleIndexBundle") {
$bundle_path = "$archive_path/bundle$bundle_index";
$info = IndexDocumentBundle::getArchiveInfo($bundle_path);
$num = min($num, $info["ACTIVE_COUNT"] + $info["COUNT"] - $start);
$num_generations = $info['SAVE_PARTITION'] + 1;
$index_archive = new IndexDocumentBundle($archive_path);
$archive = $index_archive->documents;
} else {
$nonyioop = true;
$num_generations = 1;
//for non-yioop archives we set up a real iterator
$iterator = $this->instantiateIterator($archive_path,
$archive_type);
if ($iterator === false) {
$this->badFormatMessageAndExit($archive_path);
}
}
if (!$nonyioop && isset($this->tmp_results)) {
unset($this->tmp_results);
}
$num = max($num, 0);
$total = $start + $num;
$seen = 0;
$generation = 0;
while(!$iterator->end_of_iterator &&
$seen < $total && $generation < $num_generations) {
if ($nonyioop) {
$partition = (object) [];
$partition_count = 1;
$iterator->seekPage($start);
if ($iterator->end_of_iterator) {
break;
}
$seen += $start;
} else {
$partition = $archive->loadPartitionIndex($generation);
$partition_count = count($partition);
if ($seen + $partition_count < $start) {
$generation++;
$seen += $partition_count;
continue;
}
$keys = array_keys($partition);
unset($partition);
$seen_generation = 0;
}
$seen_generation = 0;
while($seen < $total && $seen_generation < $partition_count) {
if ($nonyioop) {
$num_to_get = min(self::MAX_BUFFER_DOCS, $total - $seen);
$objects = $iterator->nextPages($num_to_get);
$seen += count($objects);
} else {
$num_to_get = min($total - $seen,
$partition_count - $seen_generation,
self::MAX_BUFFER_DOCS);
$objects = [];
for ($i = $seen_generation; $i < $num_to_get; $i++) {
$object = $archive->get($keys[$i], $generation);
$summary = $object[self::SUMMARY];
unset($object[self::SUMMARY]);
$object = array_merge($object, $summary);
$objects[] = $object;
}
$seen += $num_to_get;
$seen_generation += $num_to_get;
}
$num_to_get = count($objects);
if ($seen >= $start) {
$num_to_show = min($seen - $start, $num_to_get);
$cnt = 0;
$first = $num_to_get - $num_to_show;
foreach ($objects as $object) {
if ($cnt >= $first) {
$out = "";
if (isset($object[self::TIMESTAMP])) {
$object[self::TIMESTAMP] =
date("r", $object[self::TIMESTAMP]);
}
foreach ($fields_to_print as $key => $name) {
if (isset($object[$key])) {
$out .= "[$name]\n";
if ($key != self::IP_ADDRESSES) {
$out .= $object[$key]."\n";
} else {
foreach ($object[$key] as $address) {
$out .= $address."\n";
}
}
}
}
$out .= "==========\n\n";
echo "BEGIN ITEM, LENGTH:".strlen($out)."\n";
echo $out;
}
$cnt++;
}
}
if ($objects == null) break;
}
$generation++;
}
if (isset($this->tmp_results)) {
//garbage collect savepoint folder for non-yioop archives
$dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
$db = new $dbms_manager();
$db->unlinkRecursive($this->tmp_results);
}
}
/**
* Copies an IndexArchiveBundle (or derived class) on disk into
* an IndexDocumentBundle (on disk). The new bundle will be
* at the old bundle's location while the old bundle is renamed
* after the process to "Old" . name_of_old_bundle
*
* @param string $archive_path file path to a IndexArchiveBundle
* @param int $number_of_processes number of processes that should be
* used to perform the migration (more is faster, but this is only used
* for migration of indexes between 3.0 and 4.0)
*/
public function migrateIndexArchive($archive_path, $number_of_processes = 1)
{
$archive_type = $this->getArchiveKind($archive_path);
if (in_array($archive_type, ["FeedDocumentBundle",
"IndexDocumentBundle"])) {
$version = IndexManager::getVersion($archive_path);
if ($version < "4.0") {
echo "Checking if bundle upgradable...\n";
echo "--------------------------------\n";
echo "...Feed or IndexDocumentBundle Version $version ".
"detected!\n";
echo "...Upgrading bundle to version ".
IndexDocumentBundle::DEFAULT_VERSION . "...\n";
$this->migrateIndexDocument($archive_path,
$number_of_processes);
return;
}
}
$archive_type_use = ($archive_type == "DoubleIndexBundleOld") ?
"DoubleIndexBundle" : $archive_type;
$archive_name = C\NS_STORAGE_FORMATS . $archive_type_use;
$archive_map = [
"DoubleIndexBundleOld" => "DoubleIndexBundle",
"FeedArchiveBundle" => "FeedDocumentBundle",
"IndexArchiveBundle" => "IndexDocumentBundle",
];
if (!isset($archive_map[$archive_type])) {
$this->badFormatMessageAndExit($archive_path, "migratable index");
}
$file_name_pos = strrpos($archive_path, "/");
$parent_path = substr($archive_path, 0, $file_name_pos);
$file_name = substr($archive_path, $file_name_pos + 1);
$out_archive_path = "$parent_path/Tmp$file_name";
$new_archive_type = $archive_map[$archive_type];
$new_archive_name = C\NS_LIB . $new_archive_type;
$info = $archive_name::getArchiveInfo($archive_path);
if ($new_archive_type == "FeedDocumentBundle") {
$dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
$db = new $dbms_manager();
$db->connect();
$out_index_archive = new $new_archive_name($out_archive_path, $db,
false, $info["DESCRIPTION"], C\NUM_DOCS_PER_PARTITION);
$sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='rss'
OR TYPE='html' OR TYPE='json' OR TYPE='regex')";
$result = $db->execute($sql);
$feeds = [];
while ($feed = $db->fetchArray($result)) {
FeedsUpdateJob::parseFeedAuxInfo($feed);
$feeds[] = $feed;
}
$out_index_archive->feeds = $feeds;
} else if ($new_archive_type == "DoubleIndexBundle") {
$out_index_archive = new $new_archive_name($out_archive_path,
false, $info["DESCRIPTION"], C\NUM_DOCS_PER_PARTITION,
$info["repeat_frequency"]);
} else {
$out_index_archive = new $new_archive_name($out_archive_path,
false, $info["DESCRIPTION"], C\NUM_DOCS_PER_PARTITION);
}
$num_bundles = ($new_archive_type == "DoubleIndexBundle") ? 2 : 1;
for ($i = 0; $i < $num_bundles; $i++) {
$bundle_path = ($new_archive_type == "DoubleIndexBundle") ?
"$archive_path/bundle$i" : $archive_path;
$archive = new WebArchiveBundle($bundle_path . "/summaries");
L\crawlLog("Processing bundle $i.");
$generation_info = unserialize(
file_get_contents("$bundle_path/generation.txt"));
$num_generations = $generation_info['ACTIVE'] + 1;
$seen = 0;
$generation = 0;
$keypad = "\x00\x00\x00\x00";
while($generation < $num_generations) {
$partition = $archive->getPartition($generation, false);
L\crawlLog("Processing partition $generation");
L\crawlLog("Number of objects in partition:" .
$partition->count);
L\crawlLog("Partition Version:" . $partition->version);
$seen_partition = 0;
$more_objects = ($partition->count > 0);
while($more_objects) {
$num_to_get = self::MAX_REBUILD_DOCS;
$offset = $partition->iterator_pos;
$objects = $partition->nextObjects($num_to_get);
if (empty($objects)) {
$more_objects = false;
continue;
}
$seen_partition += count($objects);
$cnt = 0;
$store_sites = [];
foreach ($objects as $object) {
$site = $object[1];
if (isset($site['DUMMY_OFFSET']) || empty($site)) {
// first item in a partition is a dummy record
continue;
}
if (!isset($site[self::TYPE]) ||
$site[self::TYPE] != "link") {
$cnt++;
}
$doc_id = $out_index_archive->computeDocId($site);
if ($doc_id) {
$page = $site[self::PAGE] ?? "";
unset($site[self::PAGE]);
$store_sites[] = [self::DOC_ID => $doc_id,
self::SUMMARY => $site, self::PAGE => $page];
}
}
$seen_partition += $num_to_get;
unset($objects);
if (!empty($store_sites)) {
if ($new_archive_type == "FeedDocumentBundle") {
$result = $out_index_archive->addPagesAndSeenKeys(
$store_sites, $cnt);
} else {
$result = $out_index_archive->addPages($store_sites,
$cnt);
}
if ($result) {
L\crawlLog("..Store $cnt pages succeeded.");
} else {
L\crawlLog("..STORE ERROR!!!! for the following ".
"documents:");
L\crawlLog(print_r($seen_sites, true));
L\crawlLog("..Store $cnt pages failed.");
}
unset($store_sites);
}
}
$out_index_archive->updateDictionary();
$generation++;
}
if ($new_archive_type == "DoubleIndexBundle" &&
$i < 1) {
$out_index_archive->swapActiveBundle();
} else {
$out_index_archive->forceSave();
}
}
if ($new_archive_type == "DoubleIndexBundle") {
$status = ["repeat_frequency" => $info['repeat_frequency'],
"repeat_time" => $info['repeat_time'],
"swap_count" => $info['swap_count'],
"DESCRIPTION" => $out_index_archive->description
];
file_put_contents($out_index_archive->dir_name . "/status.txt",
serialize($status));
copy($archive_path . "/StartCrawlSchedule.txt",
$out_index_archive->dir_name . "/StartCrawlSchedule.txt");
}
$old_archive_path = "$parent_path/Old$file_name";
rename($archive_path, $old_archive_path);
rename($out_archive_path, $archive_path);
echo <<< EOD
Migration complete!
Test out the migrated index to see if it is working correctly.
The original index has been stored at:
$old_archive_path
If the new migrated index is working correctly you can delete the original
index. Otherwise, you can move the original index back to
$archive_path
to use it again.
EOD;
}
/**
* Migrates a Yioop >=3 index to the most recent index format of Yioop.
*
* @param string $archive_path path to a Yioop IndexDocumentBundle to
* migrate
* @param int $number_of_processes number of computer processes to use
* to performt the migration (more should be faster)
*/
public function migrateIndexDocument($archive_path, $number_of_processes=1)
{
$new_documents_folder = $archive_path . "/" .
IndexDocumentBundle::DOCUMENTS_FOLDER;
$old_documents_folder = "$archive_path/documents";
if (!file_exists($old_documents_folder)) {
echo "...Could not find any documents in bundle. Stopping!\n";
return;
}
rename($old_documents_folder, $old_documents_folder . "-old");
$old_documents_folder .= "-old";
if (!file_exists($new_documents_folder)) {
echo "...Creating new Documents folder\n";
mkdir($new_documents_folder);
chmod($new_documents_folder, 0777);
}
$old_param_file = "$old_documents_folder/pdb_parameters.txt";
if (file_exists($old_param_file)) {
$pdb_parameters = unserialize(file_get_contents($old_param_file));
$pdb_parameters["OVERFLOW_THRESHOLD"] = C\OVERFLOW_THRESHOLD;
$pdb_parameters["PARTITION_SIZE_THRESHOLD"] =
PartitionDocumentBundle::PARTITION_SIZE_THRESHOLD;
file_put_contents("$new_documents_folder/pdb_parameters.txt",
serialize($pdb_parameters));
}
$old_partitions = glob("$old_documents_folder/partition_*.txt.gz");
$i = 0;
$folder_path = $new_documents_folder;
foreach ($old_partitions as $old_partition) {
$block = sprintf("%'.05d",
floor($i / PartitionDocumentBundle::BLOCK_FACTOR));;
if (C\OVERFLOW_THRESHOLD > 0 &&
$old_partition > C\OVERFLOW_THRESHOLD) {
$folder_path .=
"/" . PartitionDocumentBundle::OVERFLOW_DIR_FOLDER;
if (!file_exists($folder_path)) {
mkdir($folder_path);
chmod($folder_path, 0777);
}
}
$block_path = "$new_documents_folder/".
PartitionDocumentBundle::BLOCK_PREFIX . $block;
if (!file_exists($block_path)) {
mkdir($block_path);
chmod($block_path, 0777);
}
$partition_path = "$block_path/".
PartitionDocumentBundle::PARTITION_PREFIX .
sprintf("%'.010d", $i);
if (!file_exists($partition_path)) {
mkdir($partition_path);
chmod($partition_path, 0777);
}
$dir_name = pathinfo($old_partition, PATHINFO_DIRNAME);
$base_name = pathinfo($old_partition, PATHINFO_BASENAME);
$file_path = substr($old_partition, 0, -strlen(".txt.gz"));
rename($old_partition, "$partition_path/" .
PartitionDocumentBundle::ARCHIVE_FILENAME . ".txt.gz");
if (file_exists("$file_path.ix")) {
rename("$file_path.ix", "$partition_path/" .
PartitionDocumentBundle::INDEX_FILENAME);
}
$i++;
}
$dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
$db = new $dbms_manager();
$db->unlinkRecursive($archive_path . "/" .
IndexDocumentBundle::DICTIONARY_FOLDER, true);
$db->unlinkRecursive($archive_path . "/positions_doc_maps", true);
$db->unlinkRecursive($old_documents_folder);
echo "...Building Dictionary in new format:\n";
$this->rebuildIndexBundle($archive_path, 0, true, $number_of_processes);
echo "...Dictionary Rebuilt!\n";
$archive_info = unserialize(file_get_contents($archive_path . "/" .
IndexDocumentBundle::ARCHIVE_INFO_FILE));
$archive_info["VERSION"] = IndexDocumentBundle::DEFAULT_VERSION;
file_put_contents($archive_path . "/" .
IndexDocumentBundle::ARCHIVE_INFO_FILE, serialize($archive_info));
echo "...Updating bundles version number!\n";
echo "Bundle Upgrade Complete!\n";
}
/**
* Used to recompute both the index shards and the dictionary
* of an index archive. The first step involves re-extracting the
* word into an inverted index from the summaries' web_archives.
* Then a reindex is done.
*
* @param string $archive_path file path to a IndexArchiveBundle
* @param mixed $start_generation which web archive generation to start
* rebuild from. If 'continue' then keeps going from where last attempt at
* a rebuild was.
* @param bool $force_recompute whether to force the recomputation of
* partition inverted indexes or to try to use existing ones if present
* @param int $number_of_processes number of CPU processes to use
* when trying to recompute partition inverted indexes
* @param mixed $last_generation which web archive generation to end
* rebuild at. Default value of -1 goes till last partition
*/
public function rebuildIndexBundle($archive_path, $start_generation = 0,
$force_recompute = true, $number_of_processes = 1,
$last_generation = -1)
{
$rebuilding = ($force_recompute) ? "Rebuilding" : "Remerging";
$rebuild = ($force_recompute) ? "rebuild" : "remerge";
$sleep_time = ($force_recompute) ? 15 : 3;
if (!$force_recompute) {
$number_of_processes = 1;
}
L\crawlLog("$rebuilding index using $number_of_processes processes!!");
$bundle_index = -1;
$bundle_path = $archive_path;
if (preg_match("/\-\d$/", $archive_path)) {
$bundle_index = substr($archive_path, -1);
$archive_path = substr($archive_path, 0, -2);
/* we can rebuild DoubleIndexBundle's by rebuilding each archive
separate
*/
$bundle_path = "$archive_path/bundle$bundle_index";
}
$archive_type = $this->getArchiveKind($bundle_path);
$archive_name = C\NS_STORAGE_FORMATS . $archive_type;
if (!in_array($archive_type, ["FeedDocumentBundle",
"IndexDocumentBundle"])) {
$this->badFormatMessageAndExit($archive_path, "rebuildable index");
}
$next_partition_path = $bundle_path . "/".
IndexDocumentBundle::NEXT_PARTITION_FILE;
if (trim($start_generation) === "continue") {
if (file_exists($next_partition_path)) {
$start_generation =
intval(file_get_contents($next_partition_path));
echo "Restarting $rebuild index from $start_generation\n";
} else {
$start_generation = 0;
}
}
$dictionary_path = $bundle_path .
IndexDocumentBundle::DICTIONARY_FOLDER;
$dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
$db = new $dbms_manager();
if ($start_generation == 0) {
$db->unlinkRecursive($bundle_path . "/" .
IndexDocumentBundle::DICTIONARY_FOLDER, false);
/* Every partition is built again by this code, which stores
combined terms and field terms, so queries may ask for them
afterwards. */
if ($archive_type == "IndexDocumentBundle") {
IndexDocumentBundle::setArchiveInfo($bundle_path,
["COMBINED_TERMS" => true, "FIELD_TERMS" => true]);
}
}
file_put_contents($next_partition_path, $start_generation);
if ($archive_type == "FeedDocumentBundle") {
$dbms_manager = C\NS_DATASOURCES . ucfirst(C\p('DBMS')) . "Manager";
$db = new $dbms_manager();
$db->connect();
$index_archive = new $archive_name($bundle_path, $db,
false, null, C\NUM_DOCS_PER_PARTITION);
$sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='rss'
OR TYPE='html' OR TYPE='json' OR TYPE='regex')";
$result = $db->execute($sql);
$feeds = [];
while ($feed = $db->fetchArray($result)) {
FeedsUpdateJob::parseFeedAuxInfo($feed);
$feeds[] = $feed;
}
$index_archive->feeds = $feeds;
} else {
$index_archive = new $archive_name($bundle_path,
false, "", C\NUM_DOCS_PER_PARTITION);
}
$save_partition = $index_archive->documents->parameters[
"SAVE_PARTITION"] ?? 0;
L\crawlLog("Save partition is $save_partition");
$updater_option = ($force_recompute) ? "rebuild_%d" : "remerge";
$options = "run %d '$bundle_path' $updater_option";
$old_next_partition = -1;
$next_partition = $start_generation;
$continue = false;
$recent_log_times = [];
$dictionary_logs = [];
/* A rebuild also runs one merging process beside its builders, and
that process writes the log of the slot after the last builder. */
$num_logs = ($force_recompute) ? $number_of_processes + 1 :
$number_of_processes;
for ($i = 0; $i < $num_logs; $i++) {
$dictionary_logs[$i] = C\LOG_DIR . "/$i-DictionaryUpdater.log";
$fp = fopen($dictionary_logs[$i], "w");
fclose($fp);
file_put_contents(C\LOG_DIR . "/$i" . self::KEPT_LOG_NAME, "");
$recent_log_times[$i] = time();
}
L\crawlLog("Each DictionaryUpdater process's own log is kept in " .
C\LOG_DIR . "/<process number>" . self::KEPT_LOG_NAME);
$end_partition = ($last_generation > 0) ? min($save_partition,
$last_generation) : $save_partition;
if ($force_recompute) {
$this->reportRebuildEnd();
$this->buildAndMergePartitions($bundle_path, $next_partition,
$end_partition, $number_of_processes, $dictionary_logs,
$recent_log_times);
echo "\nIndex $rebuild complete!\n";
return;
}
$rebuild_dones = [];
while ($next_partition < $end_partition) {
if ($old_next_partition != $next_partition) {
$old_next_partition = $next_partition;
$num_forks = min($end_partition - $next_partition,
$number_of_processes);
echo "Num forks: $num_forks, ".
"num processes $number_of_processes\n";
for ($i = 0; $i < $num_forks; $i++) {
$process_partition = $next_partition + $i;
$process_options = sprintf($options, $i,
$process_partition);
L\crawlLog(
"Begin Processing Partition: $process_partition");
L\crawlLog("Exec'ing DictionaryUpdater with parameters: " .
$process_options);
L\CrawlDaemon::execScriptInOwnProcess(C\BASE_DIR .
"/executables/DictionaryUpdater.php", $process_options);
}
} else {
for ($i = 0; $i < $num_forks; $i++) {
if (file_exists($dictionary_logs[$i])) {
$recent_log_data =
file_get_contents($dictionary_logs[$i]);
if (!empty($recent_log_data)) {
$recent_log_times[$i] = time();
file_put_contents(C\LOG_DIR . "/$i" .
self::KEPT_LOG_NAME, $recent_log_data,
FILE_APPEND);
}
$fp = fopen($dictionary_logs[$i], "w");
fclose($fp);
}
}
clearstatcache();
for ($i = 0; $i < $num_forks; $i++) {
if (empty($rebuild_dones[$i]) &&
time() > $recent_log_times[$i] + 5 * C\LOG_TIMEOUT) {
L\crawlLog("DictionaryUpdater seems to have crashed, ".
"exiting ArcTool");
exit();
}
}
sleep(5);
}
$finished = true;
for ($i = 0; $i < $num_forks; $i++) {
$rebuild_status_file = "$bundle_path/rebuild_$i";
if (file_exists($rebuild_status_file)) {
$rebuild_dones[$i] = true;
} else {
$finished = false;
}
}
if ($finished) {
$next_partition += $num_forks;
file_put_contents($next_partition_path, $next_partition);
$this->rebuild_progress = $this->rebuildStatus($bundle_path);
L\crawlLog($this->rebuild_progress);
$rebuild_dones = [];
for ($i = 0; $i < $num_forks; $i++) {
$rebuild_status_file = "$bundle_path/rebuild_$i";
unlink($rebuild_status_file);
$recent_log_times[$i] = time();
}
}
}
echo "\nIndex $rebuild complete!\n";
}
/**
* startRebuildInBackground starts an ArcTool rebuild or remerge in a
* process of its own and returns at once, so the rebuild keeps going
* when the terminal that asked for it closes. The new process runs
* this script with the same arguments, the command changed to
* rebuild-in-background or remerge-in-background, and its output goes
* to ArcToolRebuild.log in the log folder, which this prints how to
* follow. It uses CrawlDaemon::execScriptInOwnProcess, which works on
* Linux, macOS and Windows. The switch in start calls it for the
* rebuild and remerge commands.
*
* @param array $argv the command line ArcTool was given
*/
public function startRebuildInBackground($argv)
{
$arguments = array_slice($argv, 1);
$arguments[0] .= "-in-background";
$log_file = C\LOG_DIR . "/ArcToolRebuild.log";
L\CrawlDaemon::execScriptInOwnProcess(C\BASE_DIR .
"/executables/ArcTool.php",
implode(" ", array_map("escapeshellarg", $arguments)), $log_file);
echo "The {$argv[1]} of {$argv[2]} is running in a process of its " .
"own, so closing this terminal does not stop it.\n" .
"Its progress goes to $log_file\n" .
"Follow it with: tail -f $log_file\n";
}
/**
* reportRebuildEnd makes a rebuild say on the terminal how it ended,
* so that a rebuild that stops leaves a line saying why. It registers a
* shutdown function that prints one line: whether PHP met a fatal
* error, the reason the pool gave if it stopped itself, and the last
* progress line before ArcTool exits. The rebuild runs in a process of
* its own that startRebuildInBackground started, so closing the
* terminal that asked for it does not stop it; a rebuild that ends with
* no such line was stopped by a signal, such as SIGKILL.
*/
public function reportRebuildEnd()
{
$note = function ($line) {
L\crawlLog($line);
};
register_shutdown_function(function () use ($note) {
$error = error_get_last();
$fatal_types = [E_ERROR, E_PARSE, E_CORE_ERROR,
E_COMPILE_ERROR];
$how = (!empty($error) && in_array($error["type"], $fatal_types,
true)) ? "with a fatal error: " . $error["message"] .
" in " . $error["file"] . " line " . $error["line"] :
"with no fatal error";
$note("ArcTool rebuild ended $how. " . $this->rebuild_stop_reason .
" " . $this->rebuild_progress);
});
}
/**
* rebuildStatus says, in a few lines, how far a rebuild or remerge of
* an index has got and where it may safely continue from if it stopped
* now. The safe place is the count of partitions the dictionary holds,
* read from the index's next_partition.txt, which a merge writes only
* after it has finished adding a partition. A merge cut off part way
* is undone when the index is next opened for writing, so continuing
* from that count loses nothing and adds nothing twice. The lines also
* name the partitions being built and merged, and the run of
* partitions above the safe place whose files are already built. The
* rebuild-status command prints it for an index that is not being
* rebuilt, and buildAndMergePartitions prints it whenever it changes.
*
* @param string $bundle_path folder of the index
* @param array $building the partitions being built now, if any
* @param int $merging the partition being merged now, or -1 if none
* @return string the lines to print
*/
public function rebuildStatus($bundle_path, $building = [],
$merging = -1)
{
$merged = intval(@file_get_contents($bundle_path . "/" .
IndexDocumentBundle::NEXT_PARTITION_FILE));
$bundle = new IndexDocumentBundle($bundle_path, true);
$save_partition = $bundle->documents->parameters[
"SAVE_PARTITION"] ?? 0;
$built_end = $merged;
while ($built_end < $save_partition &&
!in_array($built_end, $building) && $built_end != $merging &&
$bundle->partitionIsBuilt($built_end)) {
$built_end++;
}
$name = basename($bundle_path);
$lines = ["The dictionary holds partitions 0 to " .
($merged - 1) . " of the $save_partition closed partitions."];
if ($merging >= 0) {
$lines[] = "Merging partition $merging.";
}
if (!empty($building)) {
sort($building);
$lines[] = "Building partitions " . implode(", ", $building) .
".";
}
if ($built_end > $merged) {
$lines[] = "Partitions $merged to " . ($built_end - 1) .
" are built and not yet merged.";
}
$lines[] = "If stopped now, continue safely from partition " .
"$merged: php ArcTool.php rebuild $name continue";
if ($built_end > $merged) {
$lines[] = "or, to merge the built ones without building them " .
"again: php ArcTool.php remerge $name continue";
}
return implode("\n", $lines);
}
/**
* buildAndMergePartitions rebuilds the inverted index of each partition
* from $next_partition up to $end_partition and adds each to the
* dictionary. Up to $number_of_processes DictionaryUpdater processes
* build partitions at once, and a new build starts as soon as any one
* of them finishes, so a slow partition holds up only itself. One more
* DictionaryUpdater process merges built partitions into the dictionary
* in order, beginning as soon as the lowest unmerged partition is
* built, while later partitions are still being built. The merge has
* to go in order, since the dictionary's count of partitions it holds
* says every partition below it is in. rebuildIndexBundle calls this
* for a rebuild; a remerge alone has nothing to build.
*
* @param string $bundle_path folder of the index being rebuilt
* @param int $next_partition first partition to rebuild, which is the
* number the dictionary's partition count holds
* @param int $end_partition partition after the last one to rebuild
* @param int $number_of_processes most builders to run at once; the
* merging process is one more
* @param array $dictionary_logs log file of each process slot, the
* last slot being the merging process's
* @param array $recent_log_times time each slot's log last grew, used
* to tell a crashed process from a slow one
*/
public function buildAndMergePartitions($bundle_path, $next_partition,
$end_partition, $number_of_processes, $dictionary_logs,
$recent_log_times)
{
$merge_slot = $number_of_processes;
/* Read only: used to find each partition's folder, so a build that
reported done can be checked for the files it should have made. */
$bundle = new IndexDocumentBundle($bundle_path, true);
/* A DictionaryUpdater leaves a rebuild_ file saying done when it
ends, including the one a crawl runs, so an index can hold such
files from before this rebuild. One left in a slot's name would
read as that slot's build finishing at once. */
foreach (glob("$bundle_path/rebuild_*") as $stale_done_file) {
unlink($stale_done_file);
}
$failed_builds = [];
$to_rebuild = [];
$building = [];
$built = [];
$to_build = $next_partition;
$to_merge = $next_partition;
$merging = false;
while ($to_merge < $end_partition) {
clearstatcache();
foreach ($building as $slot => $partition) {
$done_file = "$bundle_path/rebuild_$slot";
if (file_exists($done_file)) {
unlink($done_file);
unset($building[$slot]);
$folder = $bundle->getPartitionBaseFolder($partition);
$missing = [];
foreach (IndexDocumentBundle::PARTITION_FILENAMES as
$filename) {
if (!file_exists("$folder/$filename")) {
$missing[] = $filename;
}
}
if (empty($missing)) {
$built[$partition] = true;
L\crawlLog("Built Partition: $partition");
continue;
}
/* A builder that stops early still reports done, so the
files are the only proof the partition was built. A
partition merged without them would be missing from
the dictionary with no error anywhere. */
$failed_builds[$partition] =
($failed_builds[$partition] ?? 0) + 1;
L\crawlLog("Partition $partition finished without " .
implode(", ", $missing) . ", attempt " .
$failed_builds[$partition]);
if ($failed_builds[$partition] > 1) {
$this->rebuild_stop_reason =
"Partition $partition could not be " .
"built twice; stopping so the dictionary is not " .
"left without it. Its builder's log above says " .
"why.";
L\crawlLog($this->rebuild_stop_reason);
exit(1);
}
$to_rebuild[] = $partition;
}
}
$merge_done_file = "$bundle_path/rebuild_$merge_slot";
if ($merging && file_exists($merge_done_file)) {
unlink($merge_done_file);
L\crawlLog("Merged Partition: $to_merge");
unset($built[$to_merge]);
$to_merge++;
$merging = false;
}
for ($slot = 0; $slot < $number_of_processes; $slot++) {
if (isset($building[$slot]) || (empty($to_rebuild) &&
$to_build >= $end_partition)) {
continue;
}
$partition = empty($to_rebuild) ? $to_build++ :
array_shift($to_rebuild);
L\crawlLog("Begin Processing Partition: $partition");
L\CrawlDaemon::execScriptInOwnProcess(C\BASE_DIR .
"/executables/DictionaryUpdater.php",
"run $slot '$bundle_path' rebuild_$partition");
$building[$slot] = $partition;
$recent_log_times[$slot] = time();
}
if (!$merging && !empty($built[$to_merge])) {
L\crawlLog("Begin Merging Partition: $to_merge");
L\CrawlDaemon::execScriptInOwnProcess(C\BASE_DIR .
"/executables/DictionaryUpdater.php",
"run $merge_slot '$bundle_path' merge_$to_merge");
$merging = true;
$recent_log_times[$merge_slot] = time();
}
$running = array_keys($building);
if ($merging) {
$running[] = $merge_slot;
}
foreach ($running as $slot) {
$recent_log_data = file_exists($dictionary_logs[$slot]) ?
file_get_contents($dictionary_logs[$slot]) : "";
if (!empty($recent_log_data)) {
$recent_log_times[$slot] = time();
file_put_contents(C\LOG_DIR . "/$slot" .
self::KEPT_LOG_NAME, $recent_log_data, FILE_APPEND);
file_put_contents($dictionary_logs[$slot], "");
} else if (time() > $recent_log_times[$slot] +
5 * C\LOG_TIMEOUT) {
/* Every long loop in a build or a merge reports at
least once a LOG_TIMEOUT, so a slot silent this long
is stuck if its process is alive, and gone if not.
The process's lock, which holds its id, says which. A
stuck merger is named with its id and partition and
the rebuild stops. A stuck builder is named, stopped,
and then handled as a builder that is gone: its
partition is handed out again. Where the system cannot
tell whether a process is alive, as on Windows without
posix, the lock reads as held and the slot is reported
as stuck. */
$lock_file = L\CrawlDaemon::getLockFileName(
"DictionaryUpdater", $slot);
$quiet = time() - $recent_log_times[$slot];
$recent_log_times[$slot] = time();
if (L\CrawlDaemon::lockHeldByLiveProcess($lock_file)) {
$work = ($slot == $merge_slot) ?
"merging partition $to_merge" :
"building partition " . $building[$slot];
$current_file = ($slot == $merge_slot) ? "" :
IndexDocumentBundle::currentDocumentFile(
$bundle_path, $building[$slot]);
if ($current_file && file_exists($current_file)) {
$current = explode("\t",
file_get_contents($current_file), 2);
$work .= ", on document " .
($current[1] ?? $current[0]);
}
$process_id = L\CrawlDaemon::lockProcessId(
$lock_file);
$stuck = "DictionaryUpdater $slot, process " .
"$process_id, is stuck: it has logged nothing " .
"for $quiet seconds while $work.";
if ($slot == $merge_slot) {
$this->rebuild_stop_reason = $stuck .
" Stopping the rebuild.";
L\crawlLog($this->rebuild_stop_reason);
exit(1);
}
/* The builder is stopped and its partition handed
out again. The next builder of the partition
reads the document the stopped one was on from
currentDocumentFile and skips it. */
L\crawlLog($stuck . " Stopping that process and " .
"building the partition again without the " .
"document.");
L\CrawlDaemon::stopProcess($process_id);
}
if ($slot == $merge_slot) {
/* A merge that stopped part way may have added some
of its rows, and running it again could add them
twice, so the rebuild stops here instead. */
$this->rebuild_stop_reason =
"The merge of partition $to_merge " .
"stopped without finishing; stopping the " .
"rebuild. Running it again from partition " .
"$to_merge continues from there.";
L\crawlLog($this->rebuild_stop_reason);
exit(1);
}
$partition = $building[$slot];
unset($building[$slot]);
$failed_builds[$partition] =
($failed_builds[$partition] ?? 0) + 1;
L\crawlLog("DictionaryUpdater $slot stopped while " .
"building partition $partition, attempt " .
$failed_builds[$partition]);
if ($failed_builds[$partition] > 1) {
$this->rebuild_stop_reason =
"Partition $partition could not be " .
"built twice; stopping so the dictionary is not " .
"left without it.";
L\crawlLog($this->rebuild_stop_reason);
exit(1);
}
$to_rebuild[] = $partition;
}
}
$status = $this->rebuildStatus($bundle_path,
array_values($building), $merging ? $to_merge : -1);
if ($status != $this->rebuild_progress) {
$this->rebuild_progress = $status;
L\crawlLog($status);
}
sleep(2);
}
}
/**
* Used to create an archive_bundle_iterator for a non-yioop archive
* As these iterators sometimes make use of a folder to store savepoints
* We create a temporary folder for this purpose in the current directory
* This should be garbage collected elsewhere.
*
* @param string $archive_path path to non-yioop archive
* @param string $iterator_type name of archive_bundle_iterator used to
* iterate over archive.
* @param return an ArchiveBundleIterator of the correct type using
* a temporary folder to store savepoints
*/
public function instantiateIterator($archive_path, $iterator_type)
{
$iterate_timestamp = filectime($archive_path);
$result_timestamp = strval(time());
$this->tmp_results = C\TEMP_DIR . '/TmpArchiveExtract'.
$iterate_timestamp;
$dbms_manager = C\NS_DATASOURCES. ucfirst(C\p('DBMS')) . "Manager";
$db = new $dbms_manager();
if (file_exists($this->tmp_results)) {
$db->unlinkRecursive($this->tmp_results);
}
@mkdir($this->tmp_results);
$iterator_class = C\NS_ARCHIVE . "{$iterator_type}Iterator";
$iterator = new $iterator_class($iterate_timestamp, $archive_path,
$result_timestamp, $this->tmp_results);
$db->setWorldPermissionsRecursive($this->tmp_results);
return $iterator;
}
/**
* Retrieves the log data from the ImpressionRecords folder.
*
* @param string $key Optional primary key (user_id) to filter records by.
* @param int $partition Optional partition number; defaults to the
* current SAVE_PARTITION.
* @param int $limit Optional maximum number of records to display;
* defaults to all records.
* @return void
*/
public function displayLogRecords($key = null, $partition = null,
$limit = null)
{
$folder = C\LOG_DIR . "/" . ImpressionModel::IMPRESSION_FOLDER;
$format = ImpressionModel::IMPRESSION_FORMAT;
$pdb = new PartitionDocumentBundle($folder, $format);
$max_partition = $pdb->parameters['SAVE_PARTITION'];
$partitions = ($partition === null) ? range(0, $max_partition) :
[ (int)$partition ];
foreach ($partitions as $part) {
$index = $pdb->loadPartitionIndex($part, false,
PackedTableTools::APPEND_MODE);
$keys_to_process = ($key === null) ? array_keys($index) :
[ (string)$key ];
foreach ($keys_to_process as $user_id) {
if (! isset($index[$user_id])) {
continue;
}
$packed_row = $index[$user_id];
$rows = ($limit === null) ?
$pdb->table_tools->unpack($packed_row)
: $pdb->table_tools->unpack($packed_row, 0, $limit);
foreach ($rows as $record) {
echo "Key: {$user_id}\n";
print_r($record);
echo "-------------------------\n";
}
}
}
}
/**
* Like displayLogRecords(), but filters by item_id instead of user_id.
*
* @param int $item_id The item ID to filter on (required)
* @param int $partition Optional partition index
* @param int $limit Optional max number of records
*/
public function displayLogRecordsByItemId($item_id, $partition = null,
$limit = null)
{
if (!is_numeric($item_id)) {
echo "Invalid item_id\n";
return;
}
$folder = C\LOG_DIR . "/" . ImpressionModel::IMPRESSION_FOLDER;
$format = ImpressionModel::IMPRESSION_FORMAT;
$pdb = new PartitionDocumentBundle($folder, $format);
$max_partition = $pdb->parameters['SAVE_PARTITION'];
$partitions = ($partition !== null) ? [(int)$partition] :
range(0, $max_partition);
$matched = [];
foreach ($partitions as $part) {
$index = $pdb->loadPartitionIndex($part, false,
PackedTableTools::APPEND_MODE);
if (empty($index)) {
continue;
}
foreach ($index as $key => $packed_row) {
$rows = $pdb->table_tools->unpack($packed_row);
foreach ($rows as $record) {
if ($record['item_id'] == $item_id) {
$matched[] = ['key' => $key, 'record' => $record];
if ($limit !== null && count($matched) >= $limit) {
break 3;
}
}
}
}
}
if (empty($matched)) {
echo "No records found for item_id: $item_id\n";
return;
}
foreach ($matched as $entry) {
echo "Key: {$entry['key']}\n";
print_r($entry['record']);
echo "-------------------------\n";
}
}
/**
* Retrieve and display all item IDs associated with a given user.
*
* This method queries the ITEM_IMPRESSION_SUMMARY table for entries
* matching the specified USER_ID. Each matching record’s USER_ID column is
* printed on its own line to stdout.
*
* @param int $user_id The user identifier to filter GROUP_ITEM records by.
*/
public function getItemIdByUser(int $user_id)
{
$model = new ImpressionModel();
$db = $model->db;
$dbinfo = ["DBMS" => C\p('DBMS'), "DB_HOST" => C\p('DB_HOST'),
"DB_USER" => C\p('DB_USER'), "DB_PASSWORD" => C\p('DB_PASSWORD'),
"DB_NAME" => C\p('DB_NAME') ];
$sql = "SELECT ITEM_ID FROM ITEM_IMPRESSION_SUMMARY WHERE USER_ID = ?";
$res = $db->execute($sql, [$user_id]);
while ($row = $db->fetchArray($res)) {
echo $row['ITEM_ID'], "\n";
}
}
/**
* Retrieve and display all item IDs belonging to a specific group.
*
* This method queries the GROUP_ITEM table for entries matching the
* specified GROUP_ID. Each resulting record’s ID column is printed on its
* own line.
*
* @param int $group_id The group identifier to filter GROUP_ITEM records
* by.
*/
public function getItemIdByGroup(int $group_id)
{
$model = new ImpressionModel();
$db = $model->db;
$dbinfo = ["DBMS" => C\p('DBMS'), "DB_HOST" => C\p('DB_HOST'),
"DB_USER" => C\p('DB_USER'), "DB_PASSWORD" => C\p('DB_PASSWORD'),
"DB_NAME" => C\p('DB_NAME') ];
$sql = "SELECT ID FROM GROUP_ITEM WHERE GROUP_ID = ?";
$res = $db->execute($sql, [$group_id]);
while ($row = $db->fetchArray($res)) {
echo $row['ID'], "\n";
}
}
/**
* Retrieve and display all item IDs of a specific type.
*
* Accepts either a human-readable type name (thread, wiki, group,
* query, cache, resource) or the corresponding integer code. Looks
* up the integer code via an internal map, then queries
* ITEM_IMPRESSION_SUMMARY for rows matching that type and prints each
* ITEM_ID on its own line. If no records match, outputs
* "No Items of that type".
*
* @param string|int $item_type The type name or numeric code to use.
* Valid names: thread, wiki, group, query, cache, resource.
* Numeric codes: 1 (thread), 2 (wiki), 3 (group), 4 (query), 5 (cache),
* 6 (resource).
*/
public function getItemIdByType($item_type)
{
$type_map = ['thread' => C\THREAD_IMPRESSION,
'wiki' => C\WIKI_IMPRESSION, 'group' => C\GROUP_IMPRESSION,
'query' => C\QUERY_IMPRESSION, 'cache' => C\CACHE_IMPRESSION,
'resource' => C\RESOURCE_IMPRESSION];
if (is_string($item_type)) {
$key = strtolower($item_type);
if (!isset($type_map[$key])) {
$valid = implode(', ', array_keys($type_map));
echo "Unknown item type '{$item_type}'. Valid types are: " .
"{$valid}\n";
return;
}
$type_id = $type_map[$key];
} else {
$type_id = (int)$item_type;
}
$model = new ImpressionModel();
$db = $model->db;
$dbinfo = ["DBMS" => C\p('DBMS'), "DB_HOST" => C\p('DB_HOST'),
"DB_USER" => C\p('DB_USER'), "DB_PASSWORD" => C\p('DB_PASSWORD'),
"DB_NAME" => C\p('DB_NAME')];
$sql ="SELECT ITEM_ID FROM ITEM_IMPRESSION_SUMMARY WHERE ITEM_TYPE = ?";
$res = $db->execute($sql, [$type_id]);
$row = $db->fetchArray($res);
if (!$row) {
echo "No Items of that type\n";
} else {
do {
echo $row['ITEM_ID'], "\n";
} while ($row = $db->fetchArray($res));
}
}
/**
* Outputs the "hey, this isn't a known bundle message" and then exit()'s.
*
* @param string $archive_name name or path to what was supposed to be
* an archive
* @param string $allowed_archives a string list of archives types
* that $archive_name could belong to
*/
public function badFormatMessageAndExit($archive_name,
$allowed_archives = "web or index")
{
echo <<< EOD
$archive_name does not appear to be a $allowed_archives archive bundle
EOD;
exit();
}
/**
* Outputs the "how to use this tool message" and then exit()'s.
*/
public function usageMessageAndExit()
{
echo <<< EOD
ArcTool is used to look at the contents of FeedDocumentBundle's,
IndexDocumentBundle's, DoubleIndexBundle's, and BloomFilterFile's. It will look
for these using the path provided or will check in the Yioop! crawl directory
as a fall back.
The available commands for ArcTool are:
php ArcTool.php check-filter filter_file item
/* outputs whether item is in the BloomFilterFile given by filter_file
*/
php ArcTool.php count bundle_name
php ArcTool.php count double_index_name which_bundle
or
php ArcTool.php count bundle_name save
php ArcTool.php count double_index_name which_bundle save
/* returns the counts of docs and links for each partition in a bundle
as well as an overall total. The second command saves the just
computed count into the index description (can be used to
fix the index count if it gets screwed up).
*/
php ArcTool.php test-page uri
php ArcTool.php test-page uri mime_type
/*
outputs how a crawl would process the page at uri, an http, https or
file:/// address, as Page Options > Test Page does: the summary the
page processor makes, whether the word filter keeps the page, and
the words it would be indexed under. mime_type, such as text/html,
overrides the type the server sent or the file name suggests
*/
php ArcTool.php doc-lookup bundle_name partition doc_map_index
/*
returns the document stored in partition at doc_map_index
(here doc_map_index is the value that would be stored in a posting)
*/
php ArcTool.php dict bundle_info word [details]
php ArcTool.php dict bundle_info word start_record num_records [details]
/* returns index dictionary records for word stored in index archive bundle
or double index bundle. bundle_info is either the name of the bundle,
or for double index bundle, the name of the bundle whitespace which
sub-bundle. In the later case you should provide which bundle
you want dictionary info for. This command also supports start
and number of record parameters. If the word details is added to the end
of the command then additional information about each
doc record (as opposed to just their total number) is printed
*/
php ArcTool.php fix-partition bundle_name
php ArcTool.php fix-partition bundle_name start_partition
php ArcTool.php fix-partition bundle_name start_partition end_partition
/* recomputes the hash index (index) file in each partition folder
(BSOME_NUMBER/PSOME_NUMBER folder)
for a range of partitions from start_partition to end_partition in the
Documents subfolder of an IndexDocumentBundle. An index file contains
a sequence of compressed 4-tuple (doc_id, summary_offset,
summary_length, cache_length)
corresponding to a archive.txt.gz file (these are
a sequence of compressed document summaries followed by original
documents).
*/
php ArcTool.php info bundle_name
// return info about documents stored in archive.
php ArcTool.php inject timestamp file
/* injects the urls in file as a schedule into crawl of given timestamp
This can be used to make a closed index unclosed and to allow for
continued crawling. */
php ArcTool.php list
/* returns a list of all the archives in the Yioop! crawl directory,
including non-Yioop! archives in the /archives sub-folder.*/
php ArcTool.php make-filter dict_file filter_file
php ArcTool.php make-filter dict_file filter_file column_num
/* outputs to filter_file a BloomFilterFile made by inserting the items
in dict_file. If column_num is negative then dict_file is assumed to
list one item to insert per line. If column_num >=0 then dict_file
is assumed to be a csv file and column_num is the column that items
will be inserted from.
*/
php ArcTool.php migrate bundle_name
php ArcTool.php migrate bundle_name number_of_processes
/* migrates old Yioop index formats such as FeedArchiveBundle,
IndexArchiveBundle, and old style DoubleIndexBundle to their
modern respective equivalents FeedDocumentBundle, IndexDocumentBundle,
and modern DoubleIndexBundle. It can also migrate a version < 4
FeedDocumentBundle, IndexDocumentBundle to a current version bundle
The default number of processes that are used for index rebuilding is 1,
however, rebuilding is faster if you specify a number_of_processes
proportional to the number of CPU cores of your machine.
*/
php ArcTool.php partition bundle_name partition_number
php ArcTool.php partition double_index_name which_bundle partition_number
/* Prints information about the number of words and frequencies of words
within the partition_number'th partition in the index document bundle
or double index bundle (in which case need to say either 0 or 1 bundle)
*/
php ArcTool.php rebuild bundle_info
php ArcTool.php rebuild bundle_info partition_num
php ArcTool.php rebuild bundle_info partition_num number_of_processes
php ArcTool.php rebuild bundle_info partition_num number_of_processes \
end_partition
/* re-extracts words from summaries files in bundle_name a partition at a
time, builds an inverted index for that partition and adds to the global
dictionary. bundle_info is either the name of the bundle, or for
double index bundle, the name of the bundle whitespace which sub-bundle.
If this process crashes the keyword continue can be used
to continue from where it left off. If a partition_num is supplied
process continue from that partition number. The keyword "continue" can
be used to continue from the last processed partitition numebr.
The default number of processes that are used for index rebuilding is 1,
however, rebuilding is faster if you specify a number_of_processes
proportional to the number of CPU cores of your machine. That many
processes build partitions at once, a new one starting as soon as
any finishes, and one more process adds built partitions to the
dictionary in order while later ones are still being built.
*/
php ArcTool.php remerge bundle_info
php ArcTool.php remerge bundle_info partition_num
php ArcTool.php remerge bundle_info partition_num number_of_processes
php ArcTool.php remerge bundle_info partition_num number_of_processes \
end_partition
/* this operates like the previously described rebuild command except
if the inverted index files for a partition already exist in that
partition they are not recomputed (if they don't exist, they
are recomputed), only the merging of them into the
global dictionary is computed.
*/
php ArcTool.php rebuild-status bundle_name
/* says how many partitions the dictionary of bundle_name holds, which
partitions after them are built but not merged, and the rebuild or
remerge command that safely continues from there. bundle_name may
be a name in the cache folder or the index folder's full path. A
running rebuild prints the same lines whenever they change, and
keeps each DictionaryUpdater process's own log in the log folder as
<process number>-DictionaryUpdaterRebuild.log.
*/
php ArcTool.php show bundle_name start num
php ArcTool.php show double_index_name which_bundle start num
/* outputs items start through num from bundle_name or name of Yioop or
non-Yioop archive crawl folder */
php ArcTool.php user-impressions [key] [partition] [limit]
/* displays impression log records from the ImpressionRecords log folder
for a given user_id (key). If key is omitted, records for all users
are returned. Optionally, you can specify which partition to read from
and how many records to display. If no partition or limit is given,
it will scan all partitions and return all matching records.
*/
php ArcTool.php item-impressions item_id [partition] [limit]
/* displays impression log records from the ImpressionRecords log folder
for a given item_id. Optionally, you can specify
which partition to read from and how many records to display.
If no partition or limit is given, it will use the latest partition and
display all available records for that item_id.
*/
php ArcTool.php user-items user_id
/* List all item IDs for the specified user. */
php ArcTool.php group-items group_id
/* List all item IDs for the specified group. */
php ArcTool.php type-items item_type
/*
Lists all ITEM_IDs matching the specified type. The type may be
given as a name (thread, wiki, group, query, cache, resource)
or as its numeric code (1–6). Outputs one ID per line, or
"No Items of that type" if none are found.
*/
EOD;
exit();
}
}
/* ArcTool runs its command only when it is the script PHP was started
with, so a test can load the class and call a method itself. */
if (realpath($argv[0] ?? "") === __FILE__) {
$arc_tool = new ArcTool();
$arc_tool->start();
}
- Yioop Repo - Seekquarry