Add extra log info when a quota'd site has exceed limit, a=chris
Add extra log info when a quota'd site has exceed limit, a=chris
diff --git a/src/executables/QueueServer.php b/src/executables/QueueServer.php
index 39adc724a..eb3e82edb 100755
--- a/src/executables/QueueServer.php
+++ b/src/executables/QueueServer.php
@@ -1225,9 +1225,7 @@ class QueueServer implements CrawlConstants, Join
}
$this->web_queue = null;
$this->index_archive = null;
-
gc_collect_cycles(); // garbage collect old crawls
-
if ($this->isAScheduler()) {
if ($this->crawl_type == self::WEB_CRAWL ||
!isset($this->crawl_type)) {
@@ -2621,6 +2619,8 @@ class QueueServer implements CrawlConstants, Join
$this->quota_sites[$site] = [$quota, $current_count + 1];
$flag = true;
} else {
+ L\crawlLog("Quota exceeded removing " .
+ "url:$url Quota Site:$site Count:$current_count Quota:$quota");
$flag = false;
}
if ($this->quota_clear_time + C\ONE_HOUR < time()) {
diff --git a/src/library/UrlParser.php b/src/library/UrlParser.php
index c1836247d..7b48005cd 100755
--- a/src/library/UrlParser.php
+++ b/src/library/UrlParser.php
@@ -844,9 +844,13 @@ class UrlParser
$count = count($sites);
for ($i = 0; $i < $count; $i++) {
$flag = UrlParser::isPathMemberRegexPaths($host, [$hosts[$i]]);
- if (!$flag) continue;
+ if (!$flag) {
+ continue;
+ }
$flag = UrlParser::isPathMemberRegexPaths($path, [$paths[$i]]);
- if ($flag) break;
+ if ($flag) {
+ break;
+ }
}
if ($return_rule && $flag) {
$flag = $sites[$i];