Adding more error checking for ArcTool, a=chris
Adding more error checking for ArcTool, a=chris
diff --git a/src/executables/ArcTool.php b/src/executables/ArcTool.php
index 5d1dd43c2..c81228327 100755
--- a/src/executables/ArcTool.php
+++ b/src/executables/ArcTool.php
@@ -539,7 +539,7 @@ class ArcTool implements CrawlConstants
exit();
}
$shard_count_file = $path . "/reindex_count.txt";
- if ($start_shard == "continue") {
+ if (trim($start_shard) === "continue") {
if (file_exists($shard_count_file)) {
$start_shard = intval(file_get_contents($shard_count_file));
echo "Restarting reindex from $start_shard\n";
@@ -570,8 +570,12 @@ class ArcTool implements CrawlConstants
echo "\nShard $i of $num_shards\n";
$shard = new IndexShard($shard_name, $i,
C\NUM_DOCS_PER_GENERATION, true);
- $dictionary->addShardDictionary($shard);
- file_put_contents($shard_count_file, $i + 1);
+ if ($dictionary->addShardDictionary($shard)) {
+ file_put_contents($shard_count_file, $i + 1);
+ } else {
+ echo "Problem adding shard $i";
+ exit();
+ }
}
$max_tier = $dictionary->max_tier;
}
diff --git a/src/library/IndexDictionary.php b/src/library/IndexDictionary.php
index dcdecea25..8dd8800ec 100644
--- a/src/library/IndexDictionary.php
+++ b/src/library/IndexDictionary.php
@@ -209,10 +209,17 @@ class IndexDictionary implements CrawlConstants
$out_slot ="B";
}
crawlLog("Adding shard data to index dictionary files...");
- $index_shard->getShardHeader();
+ if (!$index_shard->getShardHeader()) {
+ crawlLog(" Problem reading shard header");
+ return false;
+ }
$base_offset = IndexShard::HEADER_LENGTH + $index_shard->prefixes_len;
$prefix_string = $index_shard->getShardSubstring(
IndexShard::HEADER_LENGTH, $index_shard->prefixes_len, false);
+ if (!$prefix_string) {
+ crawlLog(" Problem reading prefix string");
+ return false;
+ }
$next_offset = $base_offset;
$word_item_len = IndexShard:: WORD_KEY_LEN +
IndexShard:: WORD_DATA_LEN;
@@ -279,6 +286,7 @@ class IndexDictionary implements CrawlConstants
}
}
crawlLog("...Done Incremental Merging of Index Dictionary Tiers");
+ return true;
}
/**
* Merges for each first letter subdirectory, the $tier pair of files
diff --git a/src/library/IndexShard.php b/src/library/IndexShard.php
index 6e9d0adcf..dcd083247 100644
--- a/src/library/IndexShard.php
+++ b/src/library/IndexShard.php
@@ -1653,16 +1653,21 @@ class IndexShard extends PersistentStructure implements
/**
* If not already loaded, reads in from disk the fixed-length'd field
* variables of this IndexShard ($this->words_len, etc)
+ * @return bool whether was able to read in or not
*/
public function getShardHeader()
{
if (isset($this->num_docs) && $this->num_docs > 0) {
- return; // if $this->num_docs > 0 assume have read in
+ return true; // if $this->num_docs > 0 assume have read in
}
$header = substr($this->readBlockShardAtOffset(0, false),
0, self::HEADER_LENGTH);
+ if (!$header) {
+ return false;
+ }
self::headerToShardFields($header, $this);
$this->doc_info_offset = $this->file_len - $this->docids_len;
+ return true;
}
/**
* Used to store the length of a document as well as the number of