diff --git a/src/examples/QueryCacher.php b/src/examples/QueryCacher.php new file mode 100644 index 000000000..64c80e571 --- /dev/null +++ b/src/examples/QueryCacher.php @@ -0,0 +1,85 @@ +<?php +/** + * SeekQuarry/Yioop -- + * Open Source Pure PHP Search Engine, Crawler, and Indexer + * + * Copyright (C) 2009 - 2015 Chris Pollett chris@pollett.org + * + * LICENSE: + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * END LICENSE + * + * @author Chris Pollett chris@pollett.org + * @license http://www.gnu.org/licenses/ GPL3 + * @link http://www.seekquarry.com/ + * @copyright 2009 - 2015 + * @filesource + */ +namespace seekquarry\yioop\examples; + +/** + * Script to cache run a sequence of queries against a yioop instance + * so that they can be cached + */ +define("YIOOP_URL", "http://localhost/"); +define("TIME_BETWEEN_REQUEST_IN_SECONDS", 5); +define("QUERY_AGENT_NAME", "QUERY_CACHER"); + +if (empty($argv[1])) { + echo <<< EOD +QUERY_CACHER +============ +This program runs a sequence of queries against a Yioop Installation. +If file caching is turned on for that Yioop Installation, then those query +will be saved to its cache. To run this program, type a command like: +php QueryCacher.php file_name.txt +Here file_name.txt is the name of a text file with one query/line. +EOD; +exit(); +} else { + echo <<< EOD +QUERY_CACHER +============ +Now running a sequence of queries against the yioop installation at: + +EOD; + echo YIOOP_URL ."\n\n"; +} + +$queries = file($argv[1]); +$agent = curl_init(); +curl_setopt($agent, CURLOPT_USERAGENT, QUERY_AGENT_NAME); +curl_setopt($agent, CURLOPT_AUTOREFERER, true); +curl_setopt($agent, CURLOPT_FOLLOWLOCATION, true); +curl_setopt($agent, CURLOPT_SSL_VERIFYHOST, 0); +curl_setopt($agent, CURLOPT_SSL_VERIFYPEER, false); +curl_setopt($agent, CURLOPT_NOSIGNAL, true); +curl_setopt($agent, CURLOPT_RETURNTRANSFER, true); +curl_setopt($agent, CURLOPT_FAILONERROR, true); +curl_setopt($agent, CURLOPT_TIMEOUT, TIME_BETWEEN_REQUEST_IN_SECONDS); +curl_setopt($agent, CURLOPT_CONNECTTIMEOUT, + TIME_BETWEEN_REQUEST_IN_SECONDS); +curl_setopt($agents, CURLOPT_HTTPHEADER, ['Expect:']); + +$i = 1; +foreach($queries as $query) { + echo $i . " ". $query; + curl_setopt($agent, CURLOPT_URL, YIOOP_URL . "?q=". urlencode($query)); + $response = curl_exec($agent); + $i++; + sleep(TIME_BETWEEN_REQUEST_IN_SECONDS); +} +curl_close($agent); diff --git a/src/executables/Fetcher.php b/src/executables/Fetcher.php index a065e54b6..6ceeb0d88 100755 --- a/src/executables/Fetcher.php +++ b/src/executables/Fetcher.php @@ -541,7 +541,7 @@ class Fetcher implements CrawlConstants static $oldest_record_time = 0; static $last_record_time = 0; L\crawlLog("In Fetch Loop"); - L\crawlLog("PHP Version is use:" . phpversion()); + L\crawlLog("PHP Version in use: " . phpversion()); $prefix = $this->fetcher_num."-"; if (!file_exists(C\CRAWL_DIR."/{$prefix}temp")) { mkdir(C\CRAWL_DIR."/{$prefix}temp"); diff --git a/src/executables/MediaUpdater.php b/src/executables/MediaUpdater.php index 7c40c2474..2d0753414 100644 --- a/src/executables/MediaUpdater.php +++ b/src/executables/MediaUpdater.php @@ -135,7 +135,7 @@ class MediaUpdater implements CrawlConstants public function loop() { L\crawlLog("In Media Update Loop"); - L\crawlLog("PHP Version is use: " . phpversion()); + L\crawlLog("PHP Version in use: " . phpversion()); $info[self::STATUS] = self::CONTINUE_STATE; $local_archives = [""]; while (CrawlDaemon::processHandler()) { diff --git a/src/executables/Mirror.php b/src/executables/Mirror.php index a3d2258e4..afec42964 100644 --- a/src/executables/Mirror.php +++ b/src/executables/Mirror.php @@ -161,7 +161,7 @@ class Mirror implements CrawlConstants public function loop() { L\crawlLog("In Sync Loop"); - L\crawlLog("PHP Version is use:" . phpversion()); + L\crawlLog("PHP Version in use: " . phpversion()); $info[self::STATUS] = self::CONTINUE_STATE; while (CrawlDaemon::processHandler()) { $syncer_message_file = C\CRAWL_DIR . diff --git a/src/executables/QueueServer.php b/src/executables/QueueServer.php index 335ca58f4..6d8d4a5ab 100755 --- a/src/executables/QueueServer.php +++ b/src/executables/QueueServer.php @@ -371,7 +371,7 @@ class QueueServer implements CrawlConstants, Join if ($this->isAIndexer()) { $this->deleteOrphanedBundles(); } - L\crawlLog("PHP Version is use:" . phpversion()); + L\crawlLog("PHP Version in use: " . phpversion()); while (CrawlDaemon::processHandler()) { L\crawlLog("{$this->server_name} peak memory usage so far: ". memory_get_peak_usage()."!!"); diff --git a/src/library/cmsdetector/CmsBase.php b/src/library/cms_detectors/CmsBase.php similarity index 97% rename from src/library/cmsdetector/CmsBase.php rename to src/library/cms_detectors/CmsBase.php index c9b55fc9f..6de24328b 100644 --- a/src/library/cmsdetector/CmsBase.php +++ b/src/library/cms_detectors/CmsBase.php @@ -28,7 +28,7 @@ * @copyright 2009 - 2015 * @filesource */ -namespace seekquarry\yioop\library\cmsdetector; +namespace seekquarry\yioop\library\cms_detectors; /** * Class which is used to be the base for all framework detectors. It holds @@ -85,7 +85,8 @@ class CmsBase $elements = $xpath_doc->query("/html/head/*"); if (!is_null($elements)) { foreach ($elements as $head) { - $temp_result = array(); + $temp_result = []; + $temp_arr_result = []; $temp_result["tag"] = $head->nodeName; if($head->hasAttributes()) { $attributes = $head->attributes; diff --git a/src/library/cmsdetector/CmsDetector.php b/src/library/cms_detectors/CmsDetector.php similarity index 96% rename from src/library/cmsdetector/CmsDetector.php rename to src/library/cms_detectors/CmsDetector.php index 2570f34f2..61cd36fa4 100644 --- a/src/library/cmsdetector/CmsDetector.php +++ b/src/library/cms_detectors/CmsDetector.php @@ -28,7 +28,7 @@ * @copyright 2009 - 2015 * @filesource */ -namespace seekquarry\yioop\library\cmsdetector; +namespace seekquarry\yioop\library\cms_detectors; /** * Class which is used to run the each detector until it finds a match @@ -52,7 +52,7 @@ class CmsDetector public function checkFromContent($content) { $result = "UNKNOWN"; - $namespace = "seekquarry\yioop\library\cmsdetector"; + $namespace = "seekquarry\yioop\library\cms_detectors"; $systems = $this->getSystems(); foreach ($systems as $system_name) { $full_system_name = $namespace . "\\" . $system_name; @@ -86,4 +86,4 @@ class CmsDetector } return $result; } -} \ No newline at end of file +} diff --git a/src/library/cmsdetector/DrupalFrameworkDetector.php b/src/library/cms_detectors/DrupalFrameworkDetector.php similarity index 97% rename from src/library/cmsdetector/DrupalFrameworkDetector.php rename to src/library/cms_detectors/DrupalFrameworkDetector.php index d93089304..29e9c21bf 100644 --- a/src/library/cmsdetector/DrupalFrameworkDetector.php +++ b/src/library/cms_detectors/DrupalFrameworkDetector.php @@ -28,10 +28,11 @@ * @copyright 2009 - 2015 * @filesource */ -namespace seekquarry\yioop\library\cmsdetector; - -use seekquarry\yioop\library\cmsdetector\CmsBase; +namespace seekquarry\yioop\library\cms_detectors; +/** + * + */ class DrupalFrameworkDetector extends CmsBase { /** diff --git a/src/library/cmsdetector/WordpressFrameworkDetector.php b/src/library/cms_detectors/WordpressFrameworkDetector.php similarity index 97% rename from src/library/cmsdetector/WordpressFrameworkDetector.php rename to src/library/cms_detectors/WordpressFrameworkDetector.php index 71b55f0ae..c20d23d6a 100644 --- a/src/library/cmsdetector/WordpressFrameworkDetector.php +++ b/src/library/cms_detectors/WordpressFrameworkDetector.php @@ -28,10 +28,11 @@ * @copyright 2009 - 2015 * @filesource */ -namespace seekquarry\yioop\library\cmsdetector; - -use seekquarry\yioop\library\cmsdetector\CmsBase; +namespace seekquarry\yioop\library\cms_detectors; +/** + * + */ class WordpressFrameworkDetector extends CmsBase { /** diff --git a/src/library/processors/HtmlProcessor.php b/src/library/processors/HtmlProcessor.php index d8ac4db47..35579957b 100755 --- a/src/library/processors/HtmlProcessor.php +++ b/src/library/processors/HtmlProcessor.php @@ -37,7 +37,7 @@ use seekquarry\yioop\library\summarizers\CentroidSummarizer; use seekquarry\yioop\library\summarizers\GraphBasedSummarizer; use seekquarry\yioop\library\summarizers\ScrapeSummarizer; use seekquarry\yioop\library\summarizers\CentroidWeightedSummarizer; -use seekquarry\yioop\library\cmsdetector\CmsDetector; +use seekquarry\yioop\library\cms_detectors\CmsDetector; /** * Used to create crawl summary information @@ -614,4 +614,4 @@ class HtmlProcessor extends TextProcessor $text = preg_replace('/\</', ' <', $text); return strip_tags($text); } -} \ No newline at end of file +} diff --git a/src/views/layouts/WebLayout.php b/src/views/layouts/WebLayout.php index 24b85fec9..113f82fb0 100755 --- a/src/views/layouts/WebLayout.php +++ b/src/views/layouts/WebLayout.php @@ -116,7 +116,7 @@ class WebLayout extends Layout <?php $background_color = "#FFFFFF"; if (C\nsdefined('BACKGROUND_COLOR')) { - $background_color = isset($data['BACKGROUND_COLOR']) ? + $background_color = !empty($data['BACKGROUND_COLOR']) ? $data['BACKGROUND_COLOR'] : C\BACKGROUND_COLOR; ?> body @@ -126,7 +126,7 @@ class WebLayout extends Layout <?php } if (C\nsdefined('BACKGROUND_IMAGE') && C\BACKGROUND_IMAGE) { - $background_image = isset($data['BACKGROUND_IMAGE']) ? + $background_image = !empty($data['BACKGROUND_IMAGE']) ? $data['BACKGROUND_IMAGE'] : C\BACKGROUND_IMAGE; ?> body @@ -144,7 +144,7 @@ class WebLayout extends Layout } $foreground_color = "#FFFFFF"; if (C\nsdefined('FOREGROUND_COLOR')) { - $foreground_color = isset($data['FOREGROUND_COLOR']) ? + $foreground_color = !empty($data['FOREGROUND_COLOR']) ? $data['FOREGROUND_COLOR'] : C\FOREGROUND_COLOR; ?> .frame, @@ -166,19 +166,19 @@ class WebLayout extends Layout ?> .activity-menu h2 { - background-color: <?=isset($data['SIDEBAR_COLOR']) ? + background-color: <?=!empty($data['SIDEBAR_COLOR']) ? $data['SIDEBAR_COLOR'] : C\SIDEBAR_COLOR ?>; } .light-content, .mobile .light-content { - border: 16px solid <?=isset($data['SIDEBAR_COLOR']) ? + border: 16px solid <?=!empty($data['SIDEBAR_COLOR']) ? $data['SIDEBAR_COLOR'] : C\SIDEBAR_COLOR ?>; } <?php } if (C\nsdefined('TOPBAR_COLOR')) { - $top_color = (isset($data['TOPBAR_COLOR'])) ? + $top_color = (!empty($data['TOPBAR_COLOR'])) ? $data['TOPBAR_COLOR'] : C\TOPBAR_COLOR; ?> .display-ad p, diff --git a/tests/CmsDetectorTest.php b/tests/CmsDetectorTest.php index 0d8c20ce6..21af83c5a 100644 --- a/tests/CmsDetectorTest.php +++ b/tests/CmsDetectorTest.php @@ -32,7 +32,7 @@ namespace seekquarry\yioop\tests; use seekquarry\yioop\configs as C; use seekquarry\yioop\library\UnitTest; -use seekquarry\yioop\library\cmsdetector\CmsDetector; +use seekquarry\yioop\library\cms_detectors\CmsDetector; /** * Code used to test the CMS detector algorithms. @@ -67,7 +67,7 @@ class CmsDetectorTest extends UnitTest */ public function checkTestCase() { - $cms_detector_dir = C\PARENT_DIR.'/tests/test_files/cms_detector'; + $cms_detector_dir = C\PARENT_DIR . '/tests/test_files/cms_detector'; $file_names = file("$cms_detector_dir/cms_detector_input.txt"); $return_values_to_compare = file("$cms_detector_dir/cms_detector_results.txt");