Dump waiting hosts when robots data dumped, a=chris
Dump waiting hosts when robots data dumped, a=chris
diff --git a/bin/queue_server.php b/bin/queue_server.php
index cd172a8af..6f7dc7654 100755
--- a/bin/queue_server.php
+++ b/bin/queue_server.php
@@ -1099,6 +1099,8 @@ class QueueServer implements CrawlConstants
$this->deleteRobotData();
} else {
crawlLog("... less than max age\n");
+ crawlLog("Number of Crawl-Delayed Hosts".count(
+ $this->waiting_hosts));
}
crawlLog("Checking for robots.txt files to process...");
@@ -1172,6 +1174,9 @@ class QueueServer implements CrawlConstants
crawlLog("... resetting robot bloom filters ...");
$this->web_queue->emptyRobotFilters();
+
+ crawlLog("...Clearing Waiting Hosts");
+ $this->waiting_hosts = array();
}
/**