Adds checks before setting SAVE_PARTITION to 0, a=chris
Adds checks before setting SAVE_PARTITION to 0, a=chris
diff --git a/src/executables/QueueServer.php b/src/executables/QueueServer.php
index 84ff05e4a..36b62c05c 100644
--- a/src/executables/QueueServer.php
+++ b/src/executables/QueueServer.php
@@ -1762,8 +1762,16 @@ class QueueServer implements CrawlConstants
IndexDocumentBundle::NEXT_PARTITION_FILE;
$next_partition_to_add = (file_exists($next_partition_file)) ?
intval(file_get_contents($next_partition_file)) : 0;
- $save_partition = $this->index_archive->documents->parameters[
- "SAVE_PARTITION"] ?? 0;
+ $documents = $this->index_archive->documents;
+ $save_partition = $documents->parameters["SAVE_PARTITION"] ?? 0;
+ if ($save_partition == 0) { //double check
+ $documents = $this->index_archive->documents;
+ // this will read and if 0 recompute using a glob
+ $documents_parameters =
+ $this->index_archive->documents->getParameterInfo(
+ $documents->folder);
+ $save_partition = $documents_parameters['SAVE_PARTITION'] ?? 0;
+ }
L\crawlLog("Advance check details: Last:" .
$this->last_next_partition_to_add . " Next:" .
$next_partition_to_add . " Save:" . $save_partition);
diff --git a/src/library/PartitionDocumentBundle.php b/src/library/PartitionDocumentBundle.php
index 598d950c4..5ae09c11f 100644
--- a/src/library/PartitionDocumentBundle.php
+++ b/src/library/PartitionDocumentBundle.php
@@ -603,6 +603,9 @@ class PartitionDocumentBundle
}
/**
* Save the operating parameters of this PartitionDocumentBundle
+ * @param bool $force saveParameter() does a sanity check before
+ * writing either an empty parameter file or writing a SAVE_PARTITION
+ * with value 0. If $force is true, this check is skipped.
*/
public function saveParameters()
{
@@ -631,6 +634,11 @@ class PartitionDocumentBundle
$parameters["RECORD_COMPRESSOR"] ??= $parameters["COMPRESSOR"];
$parameters["BLOB_COMPRESSOR"] ??= $parameters["COMPRESSOR"];
}
+ if (empty($parameters['SAVE_PARTITION']) ||
+ $parameters['SAVE_PARTITION'] == 0) {
+ $parameters['SAVE_PARTITION'] =
+ max(count(glob("$folder/*" . self::INDEX_EXTENSION))-1, 0);
+ }
return $parameters;
} else {
return [];