tweak memory limit of queue_server, tweak index shard, a=chris

Chris Pollett [2013-07-16 06:Jul:th]
tweak memory limit of queue_server, tweak index shard, a=chris
Filename
bin/queue_server.php
lib/index_archive_bundle.php
lib/index_shard.php
diff --git a/bin/queue_server.php b/bin/queue_server.php
index ee0a2c66b..df439b5af 100755
--- a/bin/queue_server.php
+++ b/bin/queue_server.php
@@ -48,7 +48,7 @@ if(!defined("UNIT_TEST_MODE")) {
     define("NO_CACHE", true);
 }

-ini_set("memory_limit","1600M"); //so have enough memory to crawl big pages
+ini_set("memory_limit","1800M"); //so have enough memory to crawl big pages

 /** Load in global configuration settings */
 require_once BASE_DIR.'/configs/config.php';
diff --git a/lib/index_archive_bundle.php b/lib/index_archive_bundle.php
index 22d02eba7..75640f2f7 100644
--- a/lib/index_archive_bundle.php
+++ b/lib/index_archive_bundle.php
@@ -248,7 +248,7 @@ class IndexArchiveBundle implements CrawlConstants
             memory_get_usage());

         if($current_num_docs + $add_num_docs > $this->num_docs_per_generation
-            || (0.75 * $memory_limit) < memory_get_usage() ) {
+            || (0.7 * $memory_limit) < memory_get_usage() ) {
             if($blocking == true) {
                 return -1;
             }
diff --git a/lib/index_shard.php b/lib/index_shard.php
index 8edd68871..e47d9b8a1 100644
--- a/lib/index_shard.php
+++ b/lib/index_shard.php
@@ -1198,13 +1198,15 @@ class IndexShard extends PersistentStructure implements
             }
             $i++;
         }
+        $this->words = array();
         if($tmp_string != "") {
             $rest_posts = substr($this->word_postings, $offset);
             $this->word_postings = substr($this->word_postings, 0,
                 $write_offset);
-            $this->word_postings .= $tmp_string. $rest_posts;
+            $this->word_postings .= $tmp_string;
+            unset($tmp_string);
+            $this->word_postings .= $rest_posts;
         }
-        $this->words = array();
         $this->last_flattened_words_count = $this->num_docs;
     }
ViewGit