Dump waiting hosts when robots data dumped, a=chris

Chris Pollett [2011-08-27 18:Aug:th]
Dump waiting hosts when robots data dumped, a=chris
Filename
bin/queue_server.php
diff --git a/bin/queue_server.php b/bin/queue_server.php
index cd172a8af..6f7dc7654 100755
--- a/bin/queue_server.php
+++ b/bin/queue_server.php
@@ -1099,6 +1099,8 @@ class QueueServer implements CrawlConstants
             $this->deleteRobotData();
         } else {
             crawlLog("... less than max age\n");
+            crawlLog("Number of Crawl-Delayed Hosts".count(
+                $this->waiting_hosts));
         }

         crawlLog("Checking for robots.txt files to process...");
@@ -1172,6 +1174,9 @@ class QueueServer implements CrawlConstants

         crawlLog("... resetting robot bloom filters ...");
         $this->web_queue->emptyRobotFilters();
+
+        crawlLog("...Clearing Waiting Hosts");
+        $this->waiting_hosts = array();
     }

     /**
ViewGit