_binary From e08371cb3a9d586a3f09510085e3b9207fcad8f3 Mon Sep 17 00:00:00 2001
From: Tim Chow <tph.chow@gmail.com>
Date: Sun, 10 Mar 2019 15:13:46 -0700
Subject: [PATCH] Updated IndexShard unit tests, r=chris

---
 tests/IndexShardTest.php | 127 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 112 insertions(+), 15 deletions(-)

diff --git a/tests/IndexShardTest.php b/tests/IndexShardTest.php
index 87e2f285..915c93d5 100644
--- a/tests/IndexShardTest.php
+++ b/tests/IndexShardTest.php
@@ -85,7 +85,7 @@ class IndexShardTest extends UnitTest
         ];
         $meta_ids = ["EEEEEEEE", "FFFFFFFF"];
         $this->test_objects['shard']->addDocumentWords($docid,
-            $offset, $word_counts, $meta_ids, ["EEEEEEEE"], true);
+            $offset, $word_counts, $meta_ids, true, false);
         $this->assertEqual($this->test_objects['shard']->len_all_docs, 8,
             "Len All Docs Correctly Counts Length of First Doc");
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
@@ -104,7 +104,7 @@ class IndexShardTest extends UnitTest
         ];
         $meta_ids = ["YYYYYYYY"];
         $this->test_objects['shard']->addDocumentWords($docid,
-            $offset, $word_counts, $meta_ids, [], true);
+            $offset, $word_counts, $meta_ids, true, false);
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('CCCCCCCC', true), 5);
         $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBCCCCCCCC"]),
@@ -120,6 +120,30 @@ class IndexShardTest extends UnitTest
             "Doc lookup by meta word works");
         $this->assertEqual(count($c_data), 1,
             "Doc lookup by meta word works has correct count");
+		// add a third document and check
+        $docid = "KKKKKKKK";
+        $doc_hash = "LLLLLLLL";
+        $doc_hosts_url = "MMMMMMMM";
+        $docid .= $doc_hash. $doc_hosts_url;
+        $offset = 10;
+        $word_counts = [
+			'BBBBBBBB' => [1, 3],
+            'DDDD' => [1, 4, 9],
+            'GG' => [6],
+        ];
+        $meta_ids = ["YYYYYYYY"];
+		$this->test_objects['shard']->addDocumentWords($docid,
+            $offset, $word_counts, $meta_ids, true, false);
+		$this->assertEqual($this->test_objects['shard']->len_all_docs, 18,
+            "Len All Docs Correctly Counts Length of all Docs");
+		$c_data = $this->test_objects['shard']->getPostingsSliceById(
+            L\crawlHashWord('BBBBBBBB', true), 5);
+        $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBCCCCCCCC"]),
+            "Doc lookup by meta word works");
+		$this->assertTrue(isset($c_data["KKKKKKKKLLLLLLLLMMMMMMMM"]),
+            "Doc lookup by meta word works");
+        $this->assertEqual(count($c_data), 2,
+            "Doc lookup by meta word works has correct count");
     }
     /**
      * Check if can store link documents into an index shard and retrieve them
@@ -137,7 +161,9 @@ class IndexShardTest extends UnitTest
         $this->test_objects['shard']->addDocumentWords($docid,
             $offset, $word_counts, $meta_ids);
         $this->assertEqual($this->test_objects['shard']->len_all_link_docs, 9,
-            "Len All Docs Correctly Counts Length of First Doc");
+            "Len All Link Docs Correctly Counts Length of First Link Doc");
+		$this->assertEqual($this->test_objects['shard']->len_all_docs, 0, 
+			"Link Docs don't add anything into Len All Docs");
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('MMMMMMMM', true), 5);
         $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBCCCCCCCC"]),
@@ -153,7 +179,7 @@ class IndexShardTest extends UnitTest
         ];
         $meta_ids = ["EEEEEEEE", "FFFFFFFF"];
         $this->test_objects['shard']->addDocumentWords($docid,
-            $offset, $word_counts, $meta_ids, true);
+            $offset, $word_counts, $meta_ids, false);
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('MMMMMMMM', true), 5);
         $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBCCCCCCCC"]),
@@ -162,6 +188,30 @@ class IndexShardTest extends UnitTest
             "Link Doc lookup by word works 2nd doc");
         $this->assertEqual(count($c_data), 2,
             "Link Doc lookup by word works has correct count");
+		$docid = "AAAAAAAA";
+        $doc_hash = "BBBBBBBB";
+        $docid .= $doc_hash."GGGGGGGG";
+        $offset = 21;
+        $word_counts = [
+            'DDDDDDDD' => [1, 4],
+            'LLLLLLLL' => [7, 9],
+            'NNNNNNNN' => [2],
+        ];
+        $meta_ids = ["EEEEEEEE", "FFFFFFFF"];
+        $this->test_objects['shard']->addDocumentWords($docid,
+            $offset, $word_counts, $meta_ids, false);
+        $c_data = $this->test_objects['shard']->getPostingsSliceById(
+            L\crawlHashWord('NNNNNNNN', true), 5);
+        $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBCCCCCCCC"]),
+            "Link Doc lookup by word works 1st of two");
+        $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBGGGGGGGG"]),
+            "Link Doc lookup by word works 2nd doc");
+			$this->assertFalse(isset($c_data["AAAAAAAABBBBBBBBEEEEEEEE"]),
+            "Link Doc lookup should not have third document");
+        $this->assertEqual(count($c_data), 2,
+            "Link Doc lookup by word works has correct count");
+		$this->assertEqual($this->test_objects['shard']->len_all_link_docs, 17,
+            "Len All Link Docs Correctly Counts Length of First Link Doc");
     }
     /**
      * Check that appending two index shards works correctly
@@ -177,8 +227,8 @@ class IndexShardTest extends UnitTest
             'DDDDDDDD' => [6],
         ];
         $meta_ids = ["EEEEEEEE", "FFFFFFFF"];
-        $this->test_objects['shard']->addDocumentWords($docid,
-            $offset, $word_lists, $meta_ids, true);
+        $this->test_objects['shard']->addDocumentWords($docid, //adding a document to the first shard
+            $offset, $word_lists, $meta_ids, true); 
         $docid = "KKKKKKKKGGGGGGGGHHHHHHHH";
         $offset = 20;
         $word_lists = [
@@ -186,7 +236,7 @@ class IndexShardTest extends UnitTest
             'DDDDDDDD' => [4],
         ];
         $meta_ids = [];
-        $this->test_objects['shard2']->addDocumentWords($docid,
+        $this->test_objects['shard2']->addDocumentWords($docid, //adding a document to the second shard
             $offset, $word_lists, $meta_ids);
         $docid = "GGGGGGGG";
         $offset = 6;
@@ -196,7 +246,7 @@ class IndexShardTest extends UnitTest
             'JJJJJJJJ' => [5],
         ];
         $meta_ids = ["KKKKKKKK"];
-        $this->test_objects['shard2']->addDocumentWords($docid,
+        $this->test_objects['shard2']->addDocumentWords($docid, //adding another document to the second shard
             $offset, $word_lists, $meta_ids);
         $this->test_objects['shard']->appendIndexShard(
             $this->test_objects['shard2']);
@@ -215,7 +265,7 @@ class IndexShardTest extends UnitTest
         $this->assertTrue(isset($c_data["KKKKKKKKGGGGGGGGHHHHHHHH"]),
             "Data from second shard present 1");
         $this->assertTrue(isset($c_data["GGGGGGGG"]),
-            "Data from third shard present 1");
+            "Data from second shard present 2");
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('EEEEEEEE', true), 5);
         $this->assertTrue(isset($c_data["AAAAAAAA"]),
@@ -227,19 +277,34 @@ class IndexShardTest extends UnitTest
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('ZZZZZZZZ', true), 5);
         $this->assertTrue(isset($c_data["KKKKKKKKGGGGGGGGHHHHHHHH"]),
-            "Data from second shard present 2");
+            "Data from second shard present 3");
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('IIIIIIII', true), 5);
         $this->assertTrue(isset($c_data["GGGGGGGG"]),
-            "Data from third shard present 2");
+            "Data from second shard present 4");
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('JJJJJJJJ', true), 5);
         $this->assertTrue(isset($c_data["GGGGGGGG"]),
-            "Data from third shard present 3");
+            "Data from second shard present 4");
         $c_data = $this->test_objects['shard']->getPostingsSliceById(
             L\crawlHashWord('KKKKKKKK', true), 5);
         $this->assertTrue(isset($c_data["GGGGGGGG"]),
-            "Data from third shard present 4");
+            "Data from second shard present 5");		
+		$docid = "HHHHHHHHDDDDDDDD";
+        $offset = 50;
+        $word_lists = [
+            'YYYYYYYY' => [3, 7],
+        ];
+        $meta_ids = [];		
+        $this->test_objects['shard3']->addDocumentWords($docid, //adding a document to the third shard
+            $offset, $word_lists, $meta_ids);
+		$this->test_objects['shard']->appendIndexShard(
+            $this->test_objects['shard3']);
+		$c_data = $this->test_objects['shard']->getPostingsSliceById(
+            L\crawlHashWord('YYYYYYYY', true), 5);
+        $this->assertTrue(isset($c_data["HHHHHHHHDDDDDDDD"]),
+            "Data from third shard present 5");
+			
     }
     /**
      * Check that changing document offsets works
@@ -317,7 +382,37 @@ class IndexShardTest extends UnitTest
             L\crawlHashWord('ZZZZZZZZ', true), 5);
         $this->assertEqual($c_data['CCCCCCCCFFFFFFFF']
                 [CrawlConstants::SUMMARY_OFFSET],
-                0,  "Summary offset matches predicted second word");
+                0,  "Summary offset matches predicted second word");			
+		// adding new document after changing offset once
+		$docid = "AAAAAAAASSSSSSSD";
+        $offset = 0;
+        $word_lists = [
+            'BBBBBBBB' => [1]
+        ];
+        $meta_ids = [];
+        $this->test_objects['shard']->addDocumentWords($docid,
+            $offset, $word_lists, $meta_ids);	
+		$new_doc_offsets = [
+            "QQQQQQQQEEEEEEEEFFFFFFFF" => 10,
+            "DDDDDDDD" => 6,
+        ];
+        $this->test_objects['shard']->changeDocumentOffsets($new_doc_offsets);
+        $c_data = $this->test_objects['shard']->getPostingsSliceById(
+            L\crawlHashWord('BBBBBBBB', true), 6);
+        $predicted_offsets = [
+            "AAAAAAAASSSSSSSS" => 5,
+            "CCCCCCCCFFFFFFFF" => 0,
+            "AAAAAAAAEEEEEEEEFFFFFFFF" => 10,
+            "QQQQQQQQEEEEEEEEFFFFFFFF" => 10,
+            "DDDDDDDD" => 6,
+        ];
+		foreach ($predicted_offsets as $key =>$offset) {
+            $this->assertTrue(isset($c_data[$key]),
+                "Summary key matches predicted $i");
+            $this->assertEqual($c_data[$key][CrawlConstants::SUMMARY_OFFSET],
+                $offset,  "Summary offset matches predicted $i");
+            $i++;
+        }
     }
     /**
      * Check that save and load work
@@ -387,9 +482,10 @@ class IndexShardTest extends UnitTest
         $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBCCCCCCCC"]),
             "String Load Doc lookup 2 by word works");
         // Check if save without dictionary preserves postings
-        $word_info = $this->test_objects['shard']->getWordInfo(
+		$word_info = $this->test_objects['shard']->getWordInfo(
             L\crawlHashWord('FFFFFFFF'));
         $this->test_objects['shard']->saveWithoutDictionary();
+		
         $shard = new IndexShard(C\WORK_DIRECTORY .
             "/shard.txt", 0, C\NUM_DOCS_PER_GENERATION, true);
         $c_data = $shard->getPostingsSlice($word_info[0],
@@ -397,4 +493,5 @@ class IndexShardTest extends UnitTest
         $this->assertTrue(isset($c_data["AAAAAAAABBBBBBBBCCCCCCCC"]),
             "Save without dictionary test works");
     }
+
 }
-- 
2.20.1.windows.1

