<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\CrawlConstants as CC;
use seekquarry\yioop\library\data_structures\LSMTree;
use seekquarry\yioop\library\data_structures\UndoLog;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\models\Model;
/**
* UndoLogTest checks that UndoLog puts back the files of a step that did
* not finish, and leaves alone those of a step that did. A step left
* part way is made to look like one of a process that has ended by
* giving it the id of a process that is not running.
*/
class UndoLogTest extends UnitTest
{
/**
* setUp makes an empty folder for the log and the files it guards.
*/
public function setUp()
{
$folder = C\TEST_DIR . "/test_files/undo_log_test";
$this->tearDown();
mkdir($folder);
$this->test_objects['FOLDER'] = $folder;
}
/**
* tearDown removes the folder and everything in it.
*/
public function tearDown()
{
$folder = C\TEST_DIR . "/test_files/undo_log_test";
foreach (glob("$folder/*/*") ?: [] as $file) {
unlink($file);
}
foreach (glob("$folder/*") ?: [] as $file) {
is_dir($file) ? rmdir($file) : unlink($file);
}
if (file_exists($folder)) {
rmdir($folder);
}
}
/**
* unfinishedStepIsUndoneTestCase changes three files the way a killed
* step would, appending to one, rewriting one and making one, then
* checks undo puts each back and removes the log.
*/
public function unfinishedStepIsUndoneTestCase()
{
$folder = $this->test_objects['FOLDER'];
file_put_contents("$folder/appended", "old part");
file_put_contents("$folder/rewritten", "old whole");
$log = new UndoLog($folder);
$log->begin("partition 3 build", ["$folder/appended"],
["$folder/rewritten", "$folder/new"]);
file_put_contents("$folder/appended", " and a torn tail", FILE_APPEND);
file_put_contents("$folder/rewritten", "new half");
file_put_contents("$folder/new", "new file");
/* The step is made to look like a killed process's, since a step
of the process that is still running is not undone. */
$holder = self::finishedProcessId();
$log_path = "$folder/" . UndoLog::LOG_NAME;
file_put_contents($log_path, preg_replace('/^(\w+\t)\d+-/m',
'${1}' . $holder . '-', file_get_contents($log_path)));
$undone = (new UndoLog($folder))->undo();
$this->assertEqual($undone, ["partition 3 build"],
"undo names the step it undid");
$this->assertEqual(file_get_contents("$folder/appended"), "old part",
"an appended file is cut back to its old length");
$this->assertEqual(file_get_contents("$folder/rewritten"),
"old whole", "a rewritten file is copied back");
$this->assertFalse(file_exists("$folder/new"),
"a file the step made is removed");
$this->assertFalse(file_exists("$folder/" . UndoLog::LOG_NAME),
"the log is removed after undo");
}
/**
* finishedStepIsKeptTestCase checks that a step that finished is not
* undone, and that the log and its copies go once every step finished.
*/
public function finishedStepIsKeptTestCase()
{
$folder = $this->test_objects['FOLDER'];
file_put_contents("$folder/rewritten", "old whole");
$log = new UndoLog($folder);
$step_id = $log->begin("merge", [], ["$folder/rewritten"]);
file_put_contents("$folder/rewritten", "new whole");
$log->finish($step_id);
$this->assertEqual((new UndoLog($folder))->undo(), [],
"a finished step is not undone");
$this->assertEqual(file_get_contents("$folder/rewritten"),
"new whole", "a finished step's file keeps its new contents");
$this->assertFalse(file_exists("$folder/" . UndoLog::COPIES_NAME),
"the copies are removed once every step finished");
}
/**
* syncPathsLeavesFilesWholeTestCase checks that syncPaths, run on a
* folder of files, leaves each file holding what was written to it.
*/
public function syncPathsLeavesFilesWholeTestCase()
{
$folder = $this->test_objects['FOLDER'] . "/tier";
mkdir($folder);
for ($i = 0; $i < 3; $i++) {
file_put_contents("$folder/D$i", "rows of file $i");
}
UndoLog::syncPaths([$folder]);
for ($i = 0; $i < 3; $i++) {
$this->assertEqual(file_get_contents("$folder/D$i"),
"rows of file $i", "sync keeps file $i whole");
}
}
/**
* tornRecordIsIgnoredTestCase checks that a log whose last line was
* cut short by a crash is read without it. The step belongs to a
* process that has ended, as a killed step's would.
*/
public function tornRecordIsIgnoredTestCase()
{
$folder = $this->test_objects['FOLDER'];
file_put_contents("$folder/appended", "old part");
$holder = self::finishedProcessId();
file_put_contents("$folder/" . UndoLog::LOG_NAME,
"BEGIN\t$holder-1\tbuild\nLENGTH\t$holder-1\t" .
"$folder/appended\t8\nBEGIN\t2-");
file_put_contents("$folder/appended", " more", FILE_APPEND);
$this->assertEqual((new UndoLog($folder))->undo(), ["build"],
"only the whole step is undone");
$this->assertEqual(file_get_contents("$folder/appended"), "old part",
"its file is cut back");
}
/**
* runningStepIsKeptTestCase checks that a step whose process still
* runs is not undone, since it is in the middle of its work, and that
* its record stays in the log.
*/
public function runningStepIsKeptTestCase()
{
$folder = $this->test_objects['FOLDER'];
file_put_contents("$folder/rewritten", "new half");
$holder = getmypid();
file_put_contents("$folder/" . UndoLog::LOG_NAME,
"BEGIN\t$holder-1\tbuild\nNEW\t$holder-1\t$folder/rewritten\n");
$this->assertEqual((new UndoLog($folder))->undo(), [],
"a running step is not undone");
$this->assertTrue(file_exists("$folder/rewritten"),
"its file is left as it is");
$this->assertTrue(str_contains(file_get_contents("$folder/" .
UndoLog::LOG_NAME), "BEGIN\t$holder-1"),
"its record stays in the log");
}
/**
* checkpointCutsBackOrNamesTestCase checks the checkpoint finish
* writes: a file found longer than its checkpointed length, as a
* stopped later write leaves it, is cut back to it on undo, and one
* found shorter, as a lost write leaves it, is named.
*/
public function checkpointCutsBackOrNamesTestCase()
{
$folder = $this->test_objects['FOLDER'];
file_put_contents("$folder/postings", "0123456789");
file_put_contents("$folder/doc_map", "abcdef");
$log = new UndoLog($folder);
$step_id = $log->begin("build", [],
["$folder/postings", "$folder/doc_map"]);
$log->finish($step_id, ["$folder/postings", "$folder/doc_map"]);
file_put_contents("$folder/postings", "tail", FILE_APPEND);
file_put_contents("$folder/doc_map", "abc");
$log = new UndoLog($folder);
$log->undo();
clearstatcache();
$this->assertEqual(file_get_contents("$folder/postings"), "0123456789",
"a file longer than its checkpoint is cut back to it");
$this->assertEqual(count($log->checkpoint_notes), 2,
"both files are reported");
$this->assertTrue(str_contains(implode(" ", $log->checkpoint_notes),
"doc_map is 3 bytes, not the 6 checkpointed"),
"a file shorter than its checkpoint is named");
}
/**
* mergedTiersAreNotReportedLostTestCase merges two partitions, so the
* dictionary's tier merge removes tier 0, whose files the bundle's
* checkpoint names. Opening the bundle's undo log afterward must not
* report those files as lost, and the tier must leave no folder
* behind under another name.
*/
public function mergedTiersAreNotReportedLostTestCase()
{
$this->removeBundle();
$bundle = $this->makeBundle();
foreach ([0, 1] as $partition) {
IndexDocumentBundle::buildClosedPartition(self::TEST_DIR,
$partition);
IndexDocumentBundle::mergeNextPartition(self::TEST_DIR);
}
$log = new UndoLog(self::TEST_DIR);
$log->undo();
$this->assertEqual($log->checkpoint_notes, [],
"no file the tier merge removed is reported as lost");
$this->assertEqual(glob($bundle->dictionary->folder . "/" .
LSMTree::REMOVING_PREFIX . "*"), [],
"the merged tier leaves no folder behind");
$this->removeBundle();
}
/**
* TEST_DIR is the folder of the index bundle the stop cases make
*/
const TEST_DIR = __DIR__ . "/test_files/undo_log_stop_test";
/**
* removeBundle deletes the index bundle the stop cases make.
*/
protected function removeBundle()
{
(new Model())->db->unlinkRecursive(self::TEST_DIR);
}
/**
* makeBundle builds the index bundle each stop case works on. It holds
* ten documents, each with the word "be", in partitions of four, so
* partitions 0 and 1 are closed.
*
* @return IndexDocumentBundle the bundle
*/
protected function makeBundle()
{
$bundle = new IndexDocumentBundle(self::TEST_DIR, false, "Stop Test",
4, 5);
$docs = [];
for ($i = 0; $i < 10; $i++) {
$docs[] = [
CC::DOC_ID => str_pad("$i", 8, "0", STR_PAD_LEFT) .
str_pad("$i", 8, "0", STR_PAD_LEFT) . "d" .
str_pad("$i", 7, "0", STR_PAD_LEFT),
CC::SUMMARY => [
CC::DESCRIPTION => "to$i be or$i not$i to$i be",
CC::HASH => str_pad("$i", 8, "0", STR_PAD_LEFT),
CC::TITLE => "Some$i Play$i",
CC::URL => "https://www.somewhere$i.com/"],
CC::PAGE => "Page $i"];
}
$bundle->addPages($docs, count($docs));
$bundle->forceSave();
return $bundle;
}
}
- Yioop Repo - Seekquarry