Make low fetch activity warning more accurate
Make low fetch activity warning more accurate
diff --git a/src/configs/Config.php b/src/configs/Config.php
index bd880e6cd..43fa4f4d4 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -986,8 +986,7 @@ nsconddefine('CRASH_LOG_NAME', LOG_DIR . "/YioopCrashes.log");
/**
* Maximum time a crawl daemon process can go before calling.
* This is also the amount of time in log files of a Indexer or Scheduler
- * without a message before the process is deemed dead, and might be restarted
- * by other Queue Server process if possible.
+ * without a message before the process is deemed dead.
* @see CrawlDaemon::processHandler
*/
nsconddefine('PROCESS_TIMEOUT', 15 * ONE_MINUTE);
diff --git a/src/controllers/FetchController.php b/src/controllers/FetchController.php
index 8e7c04151..b8851123d 100755
--- a/src/controllers/FetchController.php
+++ b/src/controllers/FetchController.php
@@ -495,21 +495,32 @@ class FetchController extends Controller implements CrawlConstants
$index_data =
substr($uploaded, $pos);
}
+ $added_data = false;
if (strlen($robot_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::ROBOTS_FOLDER,
$robot_data);
+ $added_data = true;
}
if (C\USE_ETAG_EXPIRES && strlen($cache_page_validation_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::ETAG_EXPIRES_FOLDER,
$cache_page_validation_data);
+ $added_data = true;
}
if (strlen($schedule_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::SCHEDULES_FOLDER,
$schedule_data);
+ $added_data = true;
}
if (strlen($index_data) > 0) {
$messages_bundle->addMessages(MessagesBundle::INDEX_DATA_FOLDER,
$index_data);
+ $added_data = true;
+ }
+ if ($added_data) {
+ /* Time on $this->crawl_status_file_name used to determine if crawl
+ stalled. We just uploaded data so not stalled
+ */
+ touch($this->crawl_status_file_name, time());
}
return $logging;
}