_binary ÿþFrom 9660253e9c1882fbba8b40db0c45cf7389b3b6ae Mon Sep 17 00:00:00 2001
From: Tim Chow <tph.chow@gmail.com>
Date: Tue, 7 Apr 2020 10:33:12 -0700
Subject: [PATCH] IndexShard cutoff point for FeedsUpdateJob to prevent
 overflow

---
 src/library/media_jobs/FeedsUpdateJob.php | 37 ++++++++++++++++++++---
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/library/media_jobs/FeedsUpdateJob.php b/src/library/media_jobs/FeedsUpdateJob.php
index b6b9fdfa..949d2382 100644
--- a/src/library/media_jobs/FeedsUpdateJob.php
+++ b/src/library/media_jobs/FeedsUpdateJob.php
@@ -160,7 +160,7 @@ class FeedsUpdateJob extends MediaJob
             $this->updateFoundItemsOneGo($feeds_batch, self::OLD_ITEM_TIME);
             $limit += $feeds_one_go;
         }
-        $this->addFoundItemsInvertedIndex(self::OLD_ITEM_TIME);
+        $this->addFoundItemsShard(self::OLD_ITEM_TIME);
     }
     /**
      * Handles the request to get the  array of feed sources which hash to
@@ -611,7 +611,7 @@ class FeedsUpdateJob extends MediaJob
      *
      * @return bool whether job executed to complete
      */
-    public function addFoundItemsInvertedIndex($age)
+    public function addFoundItemsShard($age)
     {
         $items = $this->found_items;
         $pre_feeds = $this->tasks;
@@ -648,6 +648,9 @@ class FeedsUpdateJob extends MediaJob
             }
             return ($pub1 < $pub2) ? -1 : 1;
         });
+        // A single shard holds around 37000 items. We cut it off at
+        // 35000 just to be safe.
+        $limit = 35000;
         foreach ($items as $item) {
             L\crawlTimeoutLog(
                 "----..have added %s items to new item index.", $i);
@@ -702,9 +705,36 @@ class FeedsUpdateJob extends MediaJob
             $page[self::IMAGE_LINK] = $item['IMAGE_LINK'];
             $page[self::PUBDATE] = $item['PUBDATE'];
             $seen_sites[] = $page;
+            if ($seen_url_count > $limit) {
+                L\crawlLog("----..found more than $limit items. Saving to bundle first.");
+                $this->addFoundItemsInvertedIndex($tmp_shard, $seen_sites,
+                    $seen_url_count);
+                $tmp_shard = new IndexShard($tmp_shard_name);
+                $seen_url_count = 0;
+            }
+        }
+        if ($seen_url_count > 0) {
+            $this->addFoundItemsInvertedIndex($tmp_shard, $seen_sites,
+                $seen_url_count);
         }
         unset($term_counts['seen']);
         $this->addTermCountsTrendingTable($db, $term_counts);
+        set_error_handler(null);
+        set_error_handler(C\NS_CONFIGS . "yioop_error_handler");
+    }
+    /**
+     * Helper method for addFoundItemsShard(). Checks if the current
+     * shard is full or not and adds items to it.
+     * @param IndexShard $tmp_shard a temporary shard holding all necessary
+     *  information
+     * @param array $seen_sites of the sites and their corresponding hash
+     * @param int $seen_url_count of how many sites have been seen before
+     *  committing data to one shard
+     *
+     * @return bool whether job executed to complete
+     */
+    public function addFoundItemsInvertedIndex($tmp_shard, $seen_sites,
+            $seen_url_count) {
         L\crawlLog("----..adding items to IndexArchiveBundle");
         // 1. check if index shard is full or not. if it is, new gen
         $index_archive = $this->getFeedArchive();
@@ -733,12 +763,11 @@ class FeedsUpdateJob extends MediaJob
             $index_archive->addIndexData($tmp_shard);
         }
         $index_archive->forceSave();
+        $tmp_shard_name = C\WORK_DIRECTORY . "/data/tmp_index";
         if (file_exists($tmp_shard_name)) {
             unlink($tmp_shard_name);
         }
         unset($tmp_shard);
-        set_error_handler(null);
-        set_error_handler(C\NS_CONFIGS . "yioop_error_handler");
     }
     /**
      * Updates trending term counts based on the string from the current
-- 
2.20.1.windows.1

