Make low fetch activity warning more accurate

Chris Pollett [2023-09-10 06:Sep:th]
Make low fetch activity warning more accurate
Filename
src/configs/Config.php
src/controllers/FetchController.php
diff --git a/src/configs/Config.php b/src/configs/Config.php
index bd880e6cd..43fa4f4d4 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -986,8 +986,7 @@ nsconddefine('CRASH_LOG_NAME', LOG_DIR . "/YioopCrashes.log");
 /**
  * Maximum time a crawl daemon process can go before calling.
  * This is also the amount of time in log files of a Indexer or Scheduler
- * without a message before the process is deemed dead, and might be restarted
- * by other Queue Server process if possible.
+ * without a message before the process is deemed dead.
  * @see CrawlDaemon::processHandler
  */
 nsconddefine('PROCESS_TIMEOUT', 15 * ONE_MINUTE);
diff --git a/src/controllers/FetchController.php b/src/controllers/FetchController.php
index 8e7c04151..b8851123d 100755
--- a/src/controllers/FetchController.php
+++ b/src/controllers/FetchController.php
@@ -495,21 +495,32 @@ class FetchController extends Controller implements CrawlConstants
             $index_data =
                 substr($uploaded, $pos);
         }
+        $added_data = false;
         if (strlen($robot_data) > 0) {
             $messages_bundle->addMessages(MessagesBundle::ROBOTS_FOLDER,
                 $robot_data);
+            $added_data = true;
         }
         if (C\USE_ETAG_EXPIRES && strlen($cache_page_validation_data) > 0) {
             $messages_bundle->addMessages(MessagesBundle::ETAG_EXPIRES_FOLDER,
                 $cache_page_validation_data);
+            $added_data = true;
         }
         if (strlen($schedule_data) > 0) {
             $messages_bundle->addMessages(MessagesBundle::SCHEDULES_FOLDER,
                 $schedule_data);
+            $added_data = true;
         }
         if (strlen($index_data) > 0) {
             $messages_bundle->addMessages(MessagesBundle::INDEX_DATA_FOLDER,
                 $index_data);
+            $added_data = true;
+        }
+        if ($added_data) {
+            /* Time on $this->crawl_status_file_name used to determine if crawl
+               stalled. We just uploaded data so not stalled
+             */
+            touch($this->crawl_status_file_name, time());
         }
         return $logging;
     }
ViewGit