_binary From 7c91b09b97571bd48681225abfa97949b8782c0c Mon Sep 17 00:00:00 2001
From: Tim Chow <tph.chow@gmail.com>
Date: Fri, 6 Dec 2019 14:55:16 -0800
Subject: [PATCH] Fixes for newsfeed stuff. Removes some references to feed
 shards in ReverseIterator.

---
 src/library/IndexManager.php                  |  2 -
 src/library/IndexShard.php                    | 10 +--
 .../IntersectIterator.php                     |  2 -
 .../ReverseIterator.php                       | 77 ++-----------------
 src/models/PhraseModel.php                    |  5 +-
 tests/IndexShardTest.php                      | 14 ++--
 6 files changed, 17 insertions(+), 93 deletions(-)

diff --git a/src/library/IndexManager.php b/src/library/IndexManager.php
index 95df9617..6a7b2d45 100644
--- a/src/library/IndexManager.php
+++ b/src/library/IndexManager.php
@@ -88,11 +88,9 @@ class IndexManager implements CrawlConstants
             } else {
                 if ($index_name == "NewsFeed") {
                     $index_archive_name = self::feed_index_data_base_name;
-                    $index_name = 13;
                 } else {
                     $index_archive_name = self::index_data_base_name . $index_name;
                 }
-                $index_archive_name = self::index_data_base_name . $index_name;
                 if (file_exists(C\CRAWL_DIR.'/cache/' . $index_archive_name)) {
                     $tmp = new IndexArchiveBundle(
                         C\CRAWL_DIR.'/cache/' . $index_archive_name, null,
diff --git a/src/library/IndexShard.php b/src/library/IndexShard.php
index 49240695..419d4056 100644
--- a/src/library/IndexShard.php
+++ b/src/library/IndexShard.php
@@ -724,11 +724,6 @@ class IndexShard extends PersistentStructure implements CrawlConstants
     {
         $num_docs_so_far = 0;
         $results = [];
-        /* wd_len is a kludgy fix because word_docs_len can get out of sync
-           when things are file-based and am still tracking down why
-        */
-        $wd_len = (isset($this->file_len)) ?
-        $this->file_len - $this->docids_len : $this->word_docs_len;
         /*  For a reverse shard, the arguments for start offset and
             last offset are the same. It actually gets reversed here,
             where end:=start and last:=start.
@@ -736,7 +731,7 @@ class IndexShard extends PersistentStructure implements CrawlConstants
         $end = $start_offset >> 2;
         $last = $start_offset >> 2;
         $next = $next_offset >> 2;
-        $posting_end = $next;
+        $posting_start = $next;
         $total_posting_len = 0;
         $num_postings_so_far = 0;
         $stop = 0;
@@ -744,7 +739,7 @@ class IndexShard extends PersistentStructure implements CrawlConstants
             if ($next < $end) {
                 break;
             }
-            $posting_start = $next;
+            $posting_end = $next;
             // getPostingAtOffset will modify both start and end to the value of next
             // using addresses
             $posting = $this->getPostingAtOffset(
@@ -1048,7 +1043,6 @@ class IndexShard extends PersistentStructure implements CrawlConstants
         $posting_start = $current;
         $posting_end = $current;
         $pword = $this->getWordDocsWord($current << 2);
-
         $chr = ($pword >> 24) & 192;
         if (!$chr) {
             return packInt($pword);
diff --git a/src/library/index_bundle_iterators/IntersectIterator.php b/src/library/index_bundle_iterators/IntersectIterator.php
index 29d7316f..97b626bb 100644
--- a/src/library/index_bundle_iterators/IntersectIterator.php
+++ b/src/library/index_bundle_iterators/IntersectIterator.php
@@ -442,7 +442,6 @@ class IntersectIterator extends IndexBundleIterator
         }
         $last_changed = -1;
         $i = 0;
-        $j = 0;
         while($i != $last_changed) {
             if (time() > $time_out) {
                 return -1;
@@ -467,7 +466,6 @@ class IntersectIterator extends IndexBundleIterator
                 $i = 0;
                 $last_changed = max($last_changed, 0);
             }
-            $j++;
         }
         return 1;
     }
diff --git a/src/library/index_bundle_iterators/ReverseIterator.php b/src/library/index_bundle_iterators/ReverseIterator.php
index 5436146d..e2df4f41 100644
--- a/src/library/index_bundle_iterators/ReverseIterator.php
+++ b/src/library/index_bundle_iterators/ReverseIterator.php
@@ -86,17 +86,6 @@ class ReverseIterator extends IndexBundleIterator
      * @var array
      */
     public $dictionary_info;
-    /**
-     * File name (including path) of the feed shard for news items
-     * @var string
-     */
-    public $feed_shard_name;
-    /**
-     * Structure used to hold posting list start and stops for the query
-     * in the feed shard
-     * @var array
-     */
-    public $feed_info;
     /**
      * The total number of shards that have data for this word
      * @var int
@@ -189,47 +178,7 @@ class ReverseIterator extends IndexBundleIterator
         list($estimated_total, $this->dictionary_info) =
             IndexManager::getWordInfo($index_name, $word_key, $shift,
             -1, -1, C\NUM_DISTINCT_GENERATIONS, true);
-        $this->feed_shard_name = C\WORK_DIRECTORY . "/feeds/index";
-        if ((!C\nsdefined('NO_FEEDS') || !C\NO_FEEDS)
-            && file_exists($this->feed_shard_name)) {
-            $this->use_feeds = true;
-        } else {
-            $this->use_feeds = false;
-        }
-        if ($this->use_feeds) {
-            if (!isset($this->dictionary_info[-1])) {
-                $this->feed_info = false;
-                $this->feed_empty = true;
-            } else {
-                $this->feed_info = $this->dictionary_info[-1];
-                unset($this->dictionary_info[-1]);
-                $this->feed_empty = false;
-            }
-        } else {
-            $this->feed_info = false;
-            $this->feed_empty = true;
-        }
-        if (is_array($this->feed_info)) {
-            list(,$this->feed_start, $this->feed_end, $this->feed_count,) =
-                $this->feed_info;
-            $this->feed_info = [$this->feed_start, $this->feed_end,
-                $this->feed_count];
-        } else {
-            $this->feed_start = 0;
-            $this->feed_end = 0;
-            $this->feed_count = 0;
-        }
-        if ($this->feed_count > 0) {
-            $this->using_feeds = true;
-        } else {
-            $this->using_feeds = false;
-        }
-        if ($limit_feeds && $this->feed_count > self::LIMIT_FEEDS_COUNT) {
-            $this->feed_count = self::LIMIT_FEEDS_COUNT;
-            $this->feed_end = $this->feed_start +
-                IndexShard::POSTING_LEN * (self::LIMIT_FEEDS_COUNT - 1);
-        }
-        $this->num_docs = $this->feed_count + $estimated_total;
+        $this->num_docs = $estimated_total;
         if ($this->dictionary_info === false) {
             $this->empty = true;
         } else {
@@ -259,12 +208,6 @@ class ReverseIterator extends IndexBundleIterator
      */
     public function reset()
     {
-        if ($this->feed_count > 0) {
-            $this->using_feeds = true;
-        } else {
-            $this->using_feeds = false;
-        }
-        $no_feeds = $this->feed_empty || !$this->use_feeds;
         if (!$this->empty) {//we shouldn't be called when empty - but to be safe
             if ($this->start_generation < $this->num_generations-1) {
                 list($estimated_total, $this->dictionary_info) =
@@ -414,16 +357,10 @@ class ReverseIterator extends IndexBundleIterator
             $this->advanceGeneration($gen_doc_offset[0]);
             $this->next_offset = $this->current_offset;
         }
-        $using_feeds = $this->using_feeds && $this->use_feeds;
-        if ($using_feeds) {
-            $shard = IndexManager::getIndex("feed");
-            $last = $this->feed_end;
-        } else {
-            $index = IndexManager::getIndex($this->index_name, false);
-            $index->setCurrentShard($this->current_generation, true);
-            $shard = $index->getCurrentShard(false, false);
-            $start = $this->start_offset;
-        }
+        $index = IndexManager::getIndex($this->index_name, false);
+        $index->setCurrentShard($this->current_generation, true);
+        $shard = $index->getCurrentShard(false, false);
+        $start = $this->start_offset;
         if ($this->current_generation == $gen_doc_offset[0]) {
             $offset_pair = $shard->nextPostingOffsetDocOffset(
                  $start, $this->next_offset, $gen_doc_offset[1], false);
@@ -471,10 +408,6 @@ class ReverseIterator extends IndexBundleIterator
      */
     public function advanceGeneration($generation = null)
     {
-        if ($this->using_feeds && $this->use_feeds) {
-            $this->using_feeds = false;
-            $this->generation_pointer = -1;
-        }
         if ($generation === null) {
             $generation = $this->current_generation;
         }
diff --git a/src/models/PhraseModel.php b/src/models/PhraseModel.php
index d9073238..8ba8379c 100755
--- a/src/models/PhraseModel.php
+++ b/src/models/PhraseModel.php
@@ -1687,7 +1687,7 @@ class PhraseModel extends ParallelModel
                             $distinct_keys = [$distinct_word_keys[$i]];
                         }
                         $sum = 0;
-                        $tmp_word_iterators =[];
+                        $tmp_word_iterators = [];
                         $m = 0;
                         foreach ($distinct_keys as $distinct_key) {
                             $shift = (isset($distinct_key[1])) ?
@@ -1697,7 +1697,8 @@ class PhraseModel extends ParallelModel
                             // 13 is somewhat of a magic number right now
                             if ($index_name == 13) {
                                 $dir_name = C\CRAWL_DIR."/cache/"
-                                    .self::index_data_base_name.$index_name;
+                                    .self::feed_index_data_base_name;
+                                $index_name = "NewsFeed";
                             } else {
                                 $dir_name = C\CRAWL_DIR."/cache/"
                                     .self::index_data_base_name.$index_name;
diff --git a/tests/IndexShardTest.php b/tests/IndexShardTest.php
index 124b270d..4ab9de43 100644
--- a/tests/IndexShardTest.php
+++ b/tests/IndexShardTest.php
@@ -320,14 +320,14 @@ class IndexShardTest extends UnitTest
         $reversed = array_reverse($backward);
         $this->assertEqual($forward, $backward,
             "ReverseIndexShard returns a flipped version off a forward one");
-        $word = "media:news";
+        $word = "the";
         list($hash_key, $shift) =  L\allCrawlHashPaths($word, true)[0];
+        // set index_name to whatever IndexArchiveBundle to test.
         $index_name = 1573453725;
-        $index_name = 1575422839;
         $index_archive_name = "IndexData" . $index_name;
-        $index_archive_name = "IndexDataNewsFeed";
-        $index_name = "NewsFeed";
-        $results_limit = 200;
+        // Depending on the limit set, findDocsWithWord() will return
+        // different results, since the amount of docs per block will differ on direction
+        $results_limit = 200000;
         $total_results = 0;
         if (file_exists(C\CRAWL_DIR.'/cache/' . $index_archive_name)) {
             $info = IndexManager::getWordInfo($index_name, $hash_key, $shift, -1, 0, -1);
@@ -335,7 +335,7 @@ class IndexShardTest extends UnitTest
             $forward = [];
             if (isset($info[0][4])) {
                 $word_iterator = new WordIterator($info[0][4], 0, $index_name, true, null, $results_limit);
-                // $norm_docs = $word_iterator->findDocsWithWord();
+                $norm_docs = $word_iterator->findDocsWithWord();
                 $forward_offsets = [];
                 $offset = $word_iterator->currentGenDocOffsetWithWord();
                 array_push($forward_offsets, $offset);
@@ -356,7 +356,7 @@ class IndexShardTest extends UnitTest
             $this->assertTrue(isset($info[0][4]));
             if (isset($info[0][4])) {
                 $word_rev_iterator = new ReverseIterator($info[0][4], 0, $index_name, true, null, $results_limit);
-                // $rev_docs = $word_rev_iterator->findDocsWithWord();
+                $rev_docs = $word_rev_iterator->findDocsWithWord();
                 $backward_offsets = [];
                 $offset = $word_rev_iterator->currentGenDocOffsetWithWord();
                 array_push($backward_offsets, $offset);
-- 
2.20.1.windows.1

