_binary From 6e853bdad6b7c1ca3ca5dd29ecc817e38930356e Mon Sep 17 00:00:00 2001
From: Gargi Sheguri <gargiegs@gmail.com>
Date: Sat, 21 Oct 2023 17:41:13 -0700
Subject: [PATCH] Most recent version of crawled page in SERP

---
 src/library/IndexDocumentBundle.php           |  58 ++++-
 .../index_bundle_iterators/WordIterator.php   | 201 +++++++++++++++---
 src/models/ParallelModel.php                  |   7 +-
 src/models/PhraseModel.php                    |   2 +-
 4 files changed, 238 insertions(+), 30 deletions(-)

diff --git a/src/library/IndexDocumentBundle.php b/src/library/IndexDocumentBundle.php
index 42661a6b6..eb1fc3075 100644
--- a/src/library/IndexDocumentBundle.php
+++ b/src/library/IndexDocumentBundle.php
@@ -80,6 +80,19 @@ class IndexDocumentBundle implements CrawlConstants
      * Length of TermIds used by this IndexDocumentBundle
      */
     const TERMID_LEN = 16;
+    /**
+     * Length of terms' bloom filter string in bytes
+     */
+    const WORDSFILTER_LEN = 125;
+    /**
+     * Length of terms' bloom filter string in bits
+     */
+    const WORDSFILTER_BITS_LEN = 1000;
+    /**
+     * Number of hash functions to use while constructing the
+     * terms' bloom filter string
+     */
+    const WORDSFILTER_HASHFN_COUNT = 3;
     /**
      * Partition i in an IndexDocumentBundle has a subfolder i
      * within self::POSITIONS_DOC_MAP_FOLDER. Within this subfolder i,
@@ -958,9 +971,15 @@ class IndexDocumentBundle implements CrawlConstants
         if (empty($doc_id)) {
             return "";
         }
+        $words = array_keys($word_lists);
+        if (count($words) > 300) {
+            $words = array_slice($words, 0, 300);
+        }
+        $words_filter = $this->storeWords($words);
         $this->addScoresDocMap($doc_id, $num_words,
             $url_info[self::SCORE], $host_keywords_end_pos, $title_end_pos,
-            $path_keywords_end_pos, $description_scores, $user_ranks);
+            $path_keywords_end_pos, $description_scores,
+            $user_ranks, $words_filter);
         $this->addTermPostingLists(0, $num_words,
             $word_lists, $meta_ids, $this->doc_map_counter);
         $this->doc_map_counter++;
@@ -1066,7 +1085,7 @@ class IndexDocumentBundle implements CrawlConstants
      */
     public function addScoresDocMap($doc_id, $num_words, $score,
         $host_keywords_end_pos, $title_end_pos, $path_keywords_end_pos,
-        $description_scores, $user_ranks)
+        $description_scores, $user_ranks, $words_filter = "")
     {
         $num_description_scores = count($description_scores);
         $preface_positions =
@@ -1082,9 +1101,40 @@ class IndexDocumentBundle implements CrawlConstants
             $out_rows[] = ["POS" => 0, "SCORE" => floatval($score)];
         }
         $entry = $this->doc_map_tools->pack($out_rows);
+        /* the doc_map entry is prepended with a string representing
+         * the bloom filter of terms in the document
+         */
+        $entry = $words_filter . $entry;
         $this->doc_map_tools->add($this->doc_map, $doc_id, $entry,
             PackedTableTools::ADD_MEM_TABLE_STRING);
     }
+    /**
+     * Creates a bloom filter string made up of the top 300 most
+     * important terms in the current document. This filter is used
+     * later to check if a term belongs to the document.
+     *
+     * @param array $words terms in document
+     * @return string term bloom filter, prepended with 't'
+     * to check for backward compatibility
+     */
+    public static function storeWords($words)
+    {
+        $hash_functions = self::WORDSFILTER_HASHFN_COUNT;
+        $size = self::WORDSFILTER_BITS_LEN;
+        $words_filter = str_repeat('0', self::WORDSFILTER_LEN);
+        foreach ($words as $word) {
+            $word = canonicalTerm($word);
+            for ($i = 0; $i < $hash_functions; $i++) {
+                $hash = crc32($word . $i) % $size;
+                $byte = (int)($hash / 8);
+                $bit = $hash % 8;
+                $ascii_char = ord($words_filter[$byte]);
+                $ascii_char |= (1 << $bit);
+                $words_filter[$byte] = chr($ascii_char);
+            }
+        }
+        return 't' . $words_filter;
+    }
     /**
      * Adds posting records associated to a document to the posting lists for
      * a partition.
@@ -1215,8 +1265,8 @@ class IndexDocumentBundle implements CrawlConstants
             64 => "video",
         ];
         /**
-         * This map is maintained for backward compatibility, i.e., for the $key values
-         * using the previous letter code format.
+         * This map is maintained for backward compatibility, i.e.,
+         * for the $key values using the previous letter code format.
          */
         $old_type_map = [
             "b" => "binary",
diff --git a/src/library/index_bundle_iterators/WordIterator.php b/src/library/index_bundle_iterators/WordIterator.php
index 0e4014443..6f5825c86 100644
--- a/src/library/index_bundle_iterators/WordIterator.php
+++ b/src/library/index_bundle_iterators/WordIterator.php
@@ -36,6 +36,7 @@ use seekquarry\yioop\library\IndexShard;
 use seekquarry\yioop\library\IndexDocumentBundle;
 use seekquarry\yioop\library\IndexManager;
 use seekquarry\yioop\library\PartitionDocumentBundle;
+use seekquarry\yioop\models\ParallelModel;
 
 /**
  * Used to iterate through the documents associated with a word in
@@ -204,6 +205,11 @@ class WordIterator extends IndexBundleIterator
      * @var string
      */
     public $word_key;
+    /**
+     * Whether the latest version of each document should be searched for
+     * @var boolean
+     */
+    public $latest_version;
     /**
      * Creates a word iterator with the given parameters.
      *
@@ -221,12 +227,15 @@ class WordIterator extends IndexBundleIterator
      *      could in theory open two read only versions of the same bundle but
      *      reading the results in different directions
      * @param array $ranking_factors field say how url, keywords, and
-     *     title words should influence relevance and doc rank calculations
+     *      title words should influence relevance and doc rank calculations
+     * @param boolean $latest_version whether the latest version of a url
+     *      is being searched for or not
+     *      (@see PhraseModel::lookupSummaryOffsetGeneration())
      */
     public function __construct($word_key, $index_name, $raw = false,
         $filter = null, $results_per_block =
         IndexBundleIterator::RESULTS_PER_BLOCK, $direction=self::ASCENDING,
-        $ranking_factors = [])
+        $ranking_factors = [], $latest_version = false)
     {
         if ($raw == false) {
             //get rid of our modified base64 encoding
@@ -242,6 +251,7 @@ class WordIterator extends IndexBundleIterator
         $this->current_doc_offset = null;
         $this->results_per_block = $results_per_block;
         $this->current_block_fresh = false;
+        $this->latest_version = $latest_version;
         $this->start_generation = ($direction == self::ASCENDING) ? 0 :
             "ACTIVE";
         foreach (["CLD_URL_BONUS" => C\CLD_URL_BONUS,
@@ -467,6 +477,48 @@ class WordIterator extends IndexBundleIterator
             $this->current_generation);
         return $key_postings;
     }
+    /**
+     * Get the positions file specs for the given base index folder.
+     *
+     * @param string $base_folder of index folder
+     * @return array [file handle, file size]
+     */
+    public function getPositionsFile($base_folder)
+    {
+        $positions_filename = $base_folder . "/" .
+            IndexDocumentBundle::POSITIONS_FILENAME;
+        $fh = (file_exists($positions_filename)) ?
+            fopen($positions_filename, "r") : false;
+        $file_size = filesize($positions_filename);
+        return [$fh, $file_size];
+    }
+    /**
+     * Get the positions entry associated with a particular posting.
+     *
+     * @param array $posting of positions entry
+     * @param int $positions_file_size of positions file
+     * @param resource $positions_fh of positions file
+     * @return array of positions
+     */
+    public function getPositionsList($posting,
+        $positions_file_size, $positions_fh)
+    {
+        if ($posting['POSITIONS_LEN'] > 0 && $posting['POSITIONS_LEN'] <
+            $positions_file_size && !empty($positions_fh)) {
+            if (fseek($positions_fh, $posting['POSITIONS_OFFSET']) >= 0) {
+                $encoded_positions = fread($positions_fh,
+                    $posting['POSITIONS_LEN']);
+                $position_list = L\decodePositionList(
+                    $encoded_positions, $posting['FREQUENCY']);
+            } else {
+                $position_list = [];
+            }
+        } else {
+            $position_list = [];
+        }
+
+        return $position_list;
+    }
     /**
      * Add to a set of postings from a partition scoring information, position
      * list information and info about the relative weights of given position
@@ -481,15 +533,10 @@ class WordIterator extends IndexBundleIterator
         $key_postings = [];
         $index = IndexManager::getIndex($this->index_name);
         $base_folder = $index->getPartitionBaseFolder($partition);
-        $doc_key_len = IndexDocumentBundle::DOCID_PART_LEN;
         $doc_map_filename = $base_folder . "/" .
             IndexDocumentBundle::DOC_MAP_FILENAME;
         $doc_map_tools = $index->doc_map_tools;
-        $positions_filename = $base_folder . "/" .
-            IndexDocumentBundle::POSITIONS_FILENAME;
-        $fh = (file_exists($positions_filename)) ?
-            fopen($positions_filename, "r") : false;
-        $file_size = filesize($positions_filename);
+        list($fh, $file_size) = $this->getPositionsFile($base_folder);
         $number_of_partitions = $this->total_number_of_partitions;
         $num_doc_keys = $doc_map_tools->countTableEntries($doc_map_filename);
         $is_ascending = ($this->direction == self::ASCENDING);
@@ -499,27 +546,110 @@ class WordIterator extends IndexBundleIterator
             max($this->total_num_docs, 1);
         foreach ($postings as $posting) {
             $posting[self::GENERATION] = $partition;
-            if ($posting['POSITIONS_LEN'] > 0 && $posting['POSITIONS_LEN'] <
-                $file_size && !empty($fh)) {
-                if (fseek($fh, $posting['POSITIONS_OFFSET']) >= 0) {
-                    $encoded_positions = fread($fh, $posting['POSITIONS_LEN']);
-                    $posting[self::POSITION_LIST] = L\decodePositionList(
-                        $encoded_positions, $posting['FREQUENCY']);
-                } else {
-                    $posting[self::POSITION_LIST] = [];
-                }
-            } else {
-                $posting[self::POSITION_LIST] = [];
-            }
+            $posting[self::POSITION_LIST] = $this->getPositionsList($posting,
+                $file_size, $fh);
             $doc_map_index = $posting['DOC_MAP_INDEX'];
             $entry = $doc_map_tools->findEntryAtIndexTableName(
                 $doc_map_filename, $doc_map_index);
             $docid_len = IndexDocumentBundle::DOCID_LEN;
+            $wordsfilter_len = IndexDocumentBundle::WORDSFILTER_LEN;
             if (strlen($entry) < $docid_len) {
                 continue;
             }
             $doc_key = substr($entry, 0, $docid_len);
-            $values = substr($entry, $docid_len);
+            /**
+             * For backward compatibility: only check for the latest
+             * crawled version of a page if $entry[24] == 't'
+             * (the beginning character of the term bloom
+             * filter string attached to doc_map entries).
+             */
+            $values = (strlen($entry) >= ($docid_len + $wordsfilter_len + 1) &&
+                $entry[$docid_len] == 't') ?
+                substr($entry, $docid_len + $wordsfilter_len + 1) :
+                substr($entry, $docid_len);
+            if ($this->latest_version && $entry[$docid_len] == 't') {
+                $url_hash = substr($doc_key, 0, 8);
+                $model_for_url_hash_lookup = new ParallelModel();
+                $page_versions = $model_for_url_hash_lookup->
+                    lookupSummaryOffsetGeneration(L\base64Hash($url_hash),
+                    $this->index_name, false, true);
+                if (key_exists('ROWS', $page_versions) &&
+                    count($page_versions['ROWS']) > 0) {
+                    $latest_postings_info =
+                        end($page_versions['ROWS'])['POSTINGS'];
+                    $latest_partition =
+                        end($page_versions['ROWS'])['PARTITION'];
+                    if (count($latest_postings_info) > 0) {
+                        $latest_posting = end($latest_postings_info);
+                        /**
+                         * Ensure that the discovered latest version
+                         * isn't the same as the current posting.
+                         */
+                        if ($partition != $latest_partition ||
+                            $latest_posting['DOC_MAP_INDEX'] !=
+                            $doc_map_index) {
+                            $latest_base_folder = $index->
+                            getPartitionBaseFolder($latest_partition);
+                            $latest_doc_map_filename = $latest_base_folder .
+                                "/" . IndexDocumentBundle::DOC_MAP_FILENAME;
+                            $latest_doc_map_index =
+                                $latest_posting['DOC_MAP_INDEX'];
+                            $latest_doc_map_entry = $doc_map_tools->
+                            findEntryAtIndexTableName($latest_doc_map_filename,
+                                $latest_doc_map_index);
+                            if (strlen($latest_doc_map_entry) < $docid_len) {
+                                continue;
+                            }
+                            $latest_doc_key = substr($latest_doc_map_entry, 0,
+                                $docid_len);
+                            $words_filter = substr($latest_doc_map_entry,
+                                $docid_len + 1, $wordsfilter_len);
+                            if (!$this->checkTermExists($words_filter)) {
+                                continue;
+                            } else {
+                                /**
+                                 * The current term id exists in the most recent
+                                 * version of the document; replace the current
+                                 * posting entries with the latest entry.
+                                 */
+                                $posting[self::GENERATION] =
+                                    $latest_partition;
+                                $posting['DOC_MAP_INDEX'] =
+                                    $latest_doc_map_index;
+                                $doc_key = $latest_doc_key;
+                                $values = substr($latest_doc_map_entry,
+                                    $docid_len + $wordsfilter_len + 1);
+                                $latest_term_postings = $this->
+                                    getGenerationPostings($latest_partition);
+                                $target_posting =
+                                    array_filter($latest_term_postings,
+                                    function ($p) use ($latest_doc_map_index)
+                                    {
+                                        return $p['DOC_MAP_INDEX'] ==
+                                            $latest_doc_map_index;
+                                    });
+                                if (count($target_posting) > 0) {
+                                    $posting['POSITIONS_LEN'] =
+                                        $target_posting[0]['POSITIONS_LEN'];
+                                    $posting['POSITIONS_OFFSET'] =
+                                        $target_posting[0]['POSITIONS_OFFSET'];
+                                    $posting['FREQUENCY'] =
+                                        $target_posting[0]['FREQUENCY'];
+                                    $latest_base_folder = $index->
+                                        getPartitionBaseFolder($partition);
+                                    list($latest_positions_fh,
+                                    $latest_positions_file_size) = $this->
+                                        getPositionsFile($latest_base_folder);
+                                    $posting[self::POSITION_LIST] =
+                                        $this->getPositionsList($posting,
+                                            $latest_positions_file_size,
+                                            $latest_positions_fh);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
             if (IndexDocumentBundle::isType($doc_key, "doc")) {
                 $posting[self::IS_DOC] = true;
             }
@@ -547,7 +677,7 @@ class WordIterator extends IndexBundleIterator
                 $posting[self::DOC_RANK] = log(
                     $remaining_partitions * $this->avg_items_per_partition +
                     $posting[self::SCORE], 10);
-                if(L\IndexDocumentBundle::isAHostDocId($doc_key)) {
+                if (L\IndexDocumentBundle::isAHostDocId($doc_key)) {
                     $posting[self::DOC_RANK] +=
                         (L\IndexDocumentBundle::isACldDocId($doc_key)) ?
                         $this->ranking_factors["CLD_URL_BONUS"] :
@@ -568,7 +698,7 @@ class WordIterator extends IndexBundleIterator
                  */
                 $doc_id_format = ord($doc_key[8 << 1] ?? 0) & 96;
                 if ($doc_id_format != 96) {
-                    if(L\IndexDocumentBundle::isAWikipediaPage($doc_key)) {
+                    if (L\IndexDocumentBundle::isAWikipediaPage($doc_key)) {
                         $posting[self::DOC_RANK] +=
                             $this->ranking_factors["WIKI_BONUS"];
                     }
@@ -620,6 +750,29 @@ class WordIterator extends IndexBundleIterator
         }
         return $key_postings;
     }
+    /**
+     * Check if the current term id exists in the term bloom filter
+     * associated with the doc_map entry.
+     *
+     * @param string $words_filter term bloom filter
+     * @return boolean exists or not
+     */
+    public function checkTermExists($words_filter)
+    {
+        $hash_functions = IndexDocumentBundle::WORDSFILTER_HASHFN_COUNT;
+        $size = IndexDocumentBundle::WORDSFILTER_BITS_LEN;
+        for ($i = 0; $i < $hash_functions; $i++) {
+            $hash = crc32($this->word_key . $i) % $size;
+            $byte = (int)($hash / 8);
+            $bit = $hash % 8;
+            $ascii_char = ord($words_filter[$byte]);
+            if (($ascii_char & (1 << $bit)) == 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /**
      * Normalizes the frequencies of a term within a document with respect to
      * the length of the document, the positions of the term with the document
@@ -1066,4 +1219,4 @@ class WordIterator extends IndexBundleIterator
         }
         return [$this->current_generation, $this->current_doc_offset];
     }
-}
+}
\ No newline at end of file
diff --git a/src/models/ParallelModel.php b/src/models/ParallelModel.php
index 1a4ddf86c..bb9d9124a 100755
--- a/src/models/ParallelModel.php
+++ b/src/models/ParallelModel.php
@@ -415,10 +415,12 @@ class ParallelModel extends Model
      * @param string $index_name index into which to do the lookup
      * @param bool $is_key whether the string is info:base64_hash_url or just a
      *     url
+     * @param boolean $latest_version_lookup whether the most recent version of a url
+     *     hash is being searched for or not
      * @return array (offset, generation) into the web archive bundle
      */
     public function lookupSummaryOffsetGeneration($url_or_key, $index_name = "",
-        $is_key = false)
+        $is_key = false, $latest_version_lookup = false)
     {
         if ($index_name == "") {
             $index_name = $this->index_name;
@@ -444,6 +446,9 @@ class ParallelModel extends Model
         $hash_key = ($is_key) ? $make_term_id($url_or_key, true) :
             $make_term_id($add_info . $url_or_key, true);
         $info = IndexManager::getWordInfo($index_name, $hash_key, 0);
+        if ($latest_version_lookup) {
+            return $info;
+        }
         if (!isset($info[0][4]) && empty($info['ROWS'][0])) {
             return false;
         }
diff --git a/src/models/PhraseModel.php b/src/models/PhraseModel.php
index 2bc42dad4..e1a554eea 100755
--- a/src/models/PhraseModel.php
+++ b/src/models/PhraseModel.php
@@ -1849,7 +1849,7 @@ class PhraseModel extends ParallelModel
                         $word_iterators[$i] =
                             new I\WordIterator($distinct_key_id,
                             $actual_index_name, true, $filter, $to_retrieve,
-                            $direction, $ranking_factors);
+                            $direction, $ranking_factors, true);
                         $min_group_override = true;
                     }
                     foreach ($word_keys as $index => $key) {
-- 
2.42.0

