_binary From 13e6c61d9a545ff83983ecac3af74b583406146d Mon Sep 17 00:00:00 2001
From: Shawn Tice <shawn.cameron.bird@gmail.com>
Date: Wed, 8 May 2013 18:15:29 -0700
Subject: [PATCH] Potential fix for index bundle iterators

A first attempt at trying to get index bundle  iterators to respect the
requested number of results passed in to nextPages when using a crawl mix. The
union iterator still appears to have problems because it sets results_per_block
to the sum of the results_per_block values for each iterator it contains.
---
 lib/index_bundle_iterators/doc_iterator.php          |  5 +++--
 lib/index_bundle_iterators/index_bundle_iterator.php |  2 +-
 lib/index_bundle_iterators/word_iterator.php         |  4 +++-
 models/phrase_model.php                              | 10 ++++++----
 4 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/index_bundle_iterators/doc_iterator.php b/lib/index_bundle_iterators/doc_iterator.php
index 5e26833..628d637 100755
--- a/lib/index_bundle_iterators/doc_iterator.php
+++ b/lib/index_bundle_iterators/doc_iterator.php
@@ -119,7 +119,8 @@ class DocIterator extends IndexBundleIterator
      * @param array $filter an array of hashes of domains to filter from
      *      results
      */
-    function __construct($index_name, &$filter = NULL)
+    function __construct($index_name, &$filter = NULL,
+        $results_per_block = IndexBundleIterator::RESULTS_PER_BLOCK)
     {
         if($filter != NULL) {
             $this->filter = & $filter;
@@ -127,7 +128,6 @@ class DocIterator extends IndexBundleIterator
             $this->filter = NULL;
         }
 
-
         $this->index_name =  $index_name;
         $index = IndexManager::getIndex($index_name);
         $info = $index->getArchiveInfo($index->dir_name);
@@ -136,6 +136,7 @@ class DocIterator extends IndexBundleIterator
         $this->num_generations = (isset($index->generation_info['ACTIVE'])) ?
             $index->generation_info['ACTIVE'] + 1 : 0;
 
+        $this->results_per_block = $results_per_block;
         $this->current_block_fresh = false;
 
         $this->reset();
diff --git a/lib/index_bundle_iterators/index_bundle_iterator.php b/lib/index_bundle_iterators/index_bundle_iterator.php
index b9dbb3f..95fef79 100644
--- a/lib/index_bundle_iterators/index_bundle_iterator.php
+++ b/lib/index_bundle_iterators/index_bundle_iterator.php
@@ -259,4 +259,4 @@ abstract class IndexBundleIterator implements CrawlConstants
         $this->results_per_block = $num;
      }
 }
-?>
\ No newline at end of file
+?>
diff --git a/lib/index_bundle_iterators/word_iterator.php b/lib/index_bundle_iterators/word_iterator.php
index ea85605..98e4a0a 100644
--- a/lib/index_bundle_iterators/word_iterator.php
+++ b/lib/index_bundle_iterators/word_iterator.php
@@ -159,7 +159,8 @@ class WordIterator extends IndexBundleIterator
      * @param array $filter an array of hashes of domains to filter from
      *      results
      */
-    function __construct($word_key, $index_name, $raw = false, &$filter = NULL)
+    function __construct($word_key, $index_name, $raw = false, &$filter = NULL,
+        $results_per_block = IndexBundleIterator::RESULTS_PER_BLOCK)
     {
         if($raw == false) {
             //get rid of out modfied base64 encoding
@@ -235,6 +236,7 @@ class WordIterator extends IndexBundleIterator
             }
         }
         $this->current_doc_offset = null;
+        $this->results_per_block = $results_per_block;
         if($this->dictionary_info !== false || $this->feed_info !== false) {
             $this->reset();
         }
diff --git a/models/phrase_model.php b/models/phrase_model.php
index 53ebeff..240ea48 100755
--- a/models/phrase_model.php
+++ b/models/phrase_model.php
@@ -954,7 +954,8 @@ class PhraseModel extends ParallelModel
             }
         }
         $query_iterator = $this->getQueryIterator($word_structs, $filter, $raw,
-             $queue_servers, $original_query, $save_timestamp_name);
+            $to_retrieve, $queue_servers, $original_query,
+            $save_timestamp_name);
         $num_retrieved = 0;
         $pages = array();
 
@@ -1237,7 +1238,8 @@ class PhraseModel extends ParallelModel
      *  query
      */
     function getQueryIterator($word_structs, &$filter, $raw = 0,
-        $queue_servers = array(), $original_query = "", $save_timestamp_name="")
+        $to_retrieve, $queue_servers = array(), $original_query = "",
+        $save_timestamp_name="")
     {
         $iterators = array();
         $total_iterators = 0;
@@ -1288,11 +1290,11 @@ class PhraseModel extends ParallelModel
                     if($distinct_word_keys[$i] == $doc_iterate_hash
                         || $distinct_word_keys[$i] == $doc_iterate_group_hash) {
                         $word_iterators[$i] = new DocIterator(
-                            $index_name, $filter);
+                            $index_name, $filter, $to_retrieve);
                     } else {
                         $word_iterators[$i] =
                             new WordIterator($distinct_word_keys[$i],
-                                $index_name, false, $filter);
+                                $index_name, false, $filter, $to_retrieve);
                     }
                     foreach ($word_keys as $index => $key) {
                         if(isset($distinct_word_keys[$i]) &&
-- 
1.8.1.5

