diff --git a/src/configs/Config.php b/src/configs/Config.php index a58630062..fcb4939fd 100755 --- a/src/configs/Config.php +++ b/src/configs/Config.php @@ -595,7 +595,7 @@ if (!PROFILE) { */ if (defined("seekquarry\\yioop\\REDIRECTS_ON")) { nsconddefine('USER_AGENT', - 'Mozilla/5.0 (compatible; '.USER_AGENT_SHORT.'; +'.NAME_SERVER.')'); + 'Mozilla/5.0 (compatible; '.USER_AGENT_SHORT.'; +'.NAME_SERVER.'bot)'); } else { nsconddefine('USER_AGENT', 'Mozilla/5.0 (compatible; '. diff --git a/src/configs/PublicHelpPages.php b/src/configs/PublicHelpPages.php index 4d6f98ad1..1dcbde013 100644 --- a/src/configs/PublicHelpPages.php +++ b/src/configs/PublicHelpPages.php @@ -1076,27 +1076,63 @@ EOD; $help_pages["en-US"]["Authentication_Type"] = <<< EOD page_type=standard +page_alias= + page_border=solid-border -title=Authentication Type +toc=true + +title=Authentication+Type + +author= + +robots= + +description= + +alternative_path= + +page_header= + +page_footer= -END_HEAD_VARSThe Authentication Type field-set is used to control the protocol -used to log people into Yioop. +sort=aname -* Below is a list of Authentication types supported. -** '''Normal Authentication''', passwords are checked against stored as -salted hashes of the password; or -** '''ZKP (zero knowledge protocol) authentication''', the server picks -challenges at random and send these to the browser the person is logging in -from, the browser computes based on the password an appropriate response -according to the Fiat Shamir protocol.cThe password is never sent over the -internet and is not stored on the server. These are the main advantages of -ZKP, its drawback is that it is slower than Normal Authentication as to prove -who you are with a low probability of error requires several browser-server -exchanges. +END_HEAD_VARSThe '''Authentication and Access''' field-set is used to control people log into Yioop and how their sessions are maintained once logged in. +<br> + +The '''Authentication Type''' dropdown controls the protocol used for logging in: +* Below is a list of Authentication types supported. +** '''Normal Authentication''', passwords are checked against stored as +salted hashes of the password; or +** '''ZKP (zero knowledge protocol) authentication''', the server picks +challenges at random and send these to the browser the person is logging in +from, the browser computes based on the password an appropriate response +according to the Fiat Shamir protocol. The password is never sent over the +internet and is not stored on the server. These are the main advantages of +ZKP, its drawback is that it is slower than Normal Authentication as to prove +who you are with a low probability of error requires several browser-server +exchanges. + +* You should choose which authentication scheme you want before you create many +users as if you switch everyone will need to get a new password. +<br> + +The '''Timezone''' field controls the timezone used for dating posts and other events once a +user is logged in. +<br> + +The '''Token Name''' field controls the name of the token variable which appears in URLs that is used in conjunction +with cookies to determine if a user is logged. It is there to prevent cross-site request forgery attacks +on a Yioop website. +<br> + +The '''Session Name''' field controls the name of the cookie that will be stored in a user's browser to maintain a session once logged into Yioop. +<br> + +The '''Autologout''' dropdown specifies how long a session can be inactive before a user will be logged out. + -* You should choose which authentication scheme you want before you create many -users as if you switch everyone will need to get a new password. EOD; $help_pages["en-US"]["Bot_Configuration"] = <<< EOD page_type=standard @@ -1364,6 +1400,45 @@ to seed sites, etc. <br /> '''Page Importance''' gives each seed site an initial amount of cash. Yioop then crawls the seed sites. A given crawled page has its cash splits amongst the sites that it link to based on the link quality and whether it has been crawled yet. The sites with the most cash are crawled next and this process is continued. +EOD; +$help_pages["en-US"]["Crawl_Robot_Set-up"] = <<< EOD +page_type=standard + +page_alias= + +page_border=solid-border + +toc=true + +title= + +author= + +robots= + +description= + +alternative_path= + +page_header= + +page_footer= + +sort=aname + +END_HEAD_VARSThe '''Crawl Robot Set-up''' fieldset is used to provide websites that you crawl with information about who is crawling them. + +*The field '''Crawl Robot Name''' is used to say part of the USER-AGENT. It has the format: +<br> +Mozilla/5.0 (compatible; NAME_FROM_THIS_FIELD; YOUR_SITES_URL/bot) +<br> +The value set will be common for all fetcher traffic from the same queue server on site when downloading webpages. If you are doing crawls using multiple queue servers you should give the same value to each queue server. The value of YOUR_SITES_URL comes from the Server Settings - Name Server URL field. +*The '''Robot Instance''' field is used for web communication internal to a single yioop instance to help identify which queue server or fetcher under that queue server was involved. This string should be unique for each queue server in your Yioop set-up. The value of this string is written when logging requests between fetchers and queue servers and can be helpful in debugging. +*The '''Robot Description''' field is used to specify the Public bot wiki page. This page can also be accessed and edited under Manage Groups by clicking on the wiki link for the Public group and then editing its Bot page. This wiki page is what's display when someone goes to the URL:<br> +YOUR_SITES_URL/bot +<br> +The point of this page is to give web owners both contact info for your bot as well as a description of how your bot crawls web sites. + EOD; $help_pages["en-US"]["Create_Group"] = <<< EOD page_type=standard @@ -1471,6 +1546,40 @@ probably the most convenient database system to use with Yioop. If you think you are going to make use of Yioop's social functionality and have many users, feeds, and crawl mixes, using a system like Mysql or Postgres might be more appropriate. +EOD; +$help_pages["en-US"]["Debug_Display"] = <<< EOD +page_type=standard + +page_alias= + +page_border=solid-border + +toc=true + +title= + +author= + +robots= + +description= + +alternative_path= + +page_header= + +page_footer= + +sort=aname + +END_HEAD_VARSThe '''Debug Display''' fieldset consists of checkboxes which control the debugging features of Yioop that are enabled. + +*The '''Error Info''' checkbox controls whether or not PHP errors, warnings and notices are output from Yioop. Whether the output is then to the browser or to a log file is controlled by the php.ini of your php install. +*The '''Query Info''' checkbox controls whether or not Yioop appends to each page information about how long each database and search query took. +*The '''Test Info''' checkbox controls whether or not Yioop unit tests are visible from the Yioop site. If checked, the '''Test Info''' link takes one to the unit tests. + + + EOD; $help_pages["en-US"]["Disallowed_and_Sites_With_Quotas"] = <<< EOD page_type=standard @@ -2429,6 +2538,38 @@ END_HEAD_VARS'''Web Scrapers''' are used to help Y '''Extract Fields''' is used to specify a sequence of rules to extract to specific fields in the summary. Each rule should be on a line by itself and have the format: NAME_OF_SUMMARY_FIELD = SOME_XPATH. The meaning of such a rule compute the xpath on the original document and concatenate the text contents of the resulting nodes into NAME_OF_SUMMARY_FIELD in the summary. For example, SITE_NAME=//meta[@property='og:site_name']/@content would take the value of the content attribute of all meta tags with property attribute having value og:site_name, concatenate them as a string, and store the key SITE_NAME with value this string in the pages summary when it is indexed. +EOD; +$help_pages["en-US"]["Search_Access"] = <<< EOD +page_type=standard + +page_alias= + +page_border=solid-border + +toc=true + +title= + +author= + +robots= + +description= + +alternative_path= + +page_header= + +page_footer= + +sort=aname + +END_HEAD_VARSThe '''Search Access''' fieldset has checkboxes that control which interfaces can be used to get search query results from Yioop. + +* The '''Web''' checkbox controls whether or not a traditional web search through the Yioop instance's landing page can be done. +* The '''RSS''' checkbox controls whether or not search queries in RSS format are available. If so, there a query string of the form ?q=some_search_query&f=rss will output search results in rss format, a query string in RSS format will be output. This checkbox needs to be checked if you are using Yioop in a situation with multiple queue servers. This switch also enables queries of the form ?q=some_search_query&f=json, ?q=some_search_query&f=json&callback=some_function, and ?q=some_search_query&f=serial. These are respectively JSON format output, JSONP format output, and serialized PHP object format output. +* The '''API''' checkbox controls whether or not Yioop can be used as PHP library using the Yioop Search Function API to return search results. This is described in the [[https://www.seekquarry.com/p/Documentation#Embedding%20Yioop%20in%20an%20Existing%20Site|Embedding Yioop]] section of the Yioop Documentation. + EOD; $help_pages["en-US"]["Search_Results_Editor"] = <<< EOD page_type=standard @@ -2662,11 +2803,17 @@ robots= description= +alternative_path= + page_header= page_footer= -END_HEAD_VARSThe '''Test Page''' form is used to test how Yioop would process a given web page. To test a web page one copies and pastes the source of the web page (obtainable by doing View Source in a browser) into the textarea. Then one selects the mimetype of the page (usually, text/html) and submits the form to see the processing results. +sort=aname + +END_HEAD_VARSThe '''Test Page''' form is used to test how Yioop would process a given web page. To test a web page one +first selects the '''Method of Submission'''. This can either be by giving the URI of the webpage you would like to test, or by choosing a file to upload and test or by direct input. For direct input one +copies and pastes the source of the web page (obtainable by doing View Source in a browser) into the textarea. Then one selects the mimetype of the page (usually, text/html) and submits the form to see the processing results. EOD; $help_pages["en-US"]["Using_a_Classifier_or_Ranker"] = <<< EOD page_type=standard diff --git a/src/data/public_default.db b/src/data/public_default.db index 11803593b..d4d185a6b 100644 Binary files a/src/data/public_default.db and b/src/data/public_default.db differ diff --git a/src/library/CrawlConstants.php b/src/library/CrawlConstants.php index 9cf09dbb9..65136d8ba 100755 --- a/src/library/CrawlConstants.php +++ b/src/library/CrawlConstants.php @@ -65,7 +65,7 @@ interface CrawlConstants const network_crawllist_base_name = "NetworkCrawlList"; const statistics_base_name = "Statistics"; const index_closed_name = "IndexClosed"; - const fetch_batch_name = "FetchBatch"; //used to continue if fetcheer halted + const fetch_batch_name = "FetchBatch"; //used to continue if fetcher halted const fetch_crawl_info = "FetchInfo"; const fetch_closed_name = "FetchClosed"; const data_base_name = "At"; @@ -108,6 +108,8 @@ interface CrawlConstants const TO_CRAWL = 'y'; const INDEX = 'z'; const DESCRIPTION_SCORES = 'A'; + const HEIGHT = 'B'; + const WIDTH = 'C'; // codes available here const DOC_DEPTH = 'M'; const DOC_RANK = 'N'; diff --git a/src/library/PhraseParser.php b/src/library/PhraseParser.php index b0680a314..2d9393010 100755 --- a/src/library/PhraseParser.php +++ b/src/library/PhraseParser.php @@ -1102,9 +1102,23 @@ class PhraseParser $meta_ids[] = 'media:all'; if (!empty($site[CrawlConstants::IS_VIDEO])) { $meta_ids[] = "media:video"; + } else if (stripos($site[CrawlConstants::TYPE], + "image") !== false) { + $meta_ids[] = 'media:image'; + if (!empty($site[CrawlConstants::WIDTH]) && + !empty($site[CrawlConstants::HEIGHT])) { + $size = $site[CrawlConstants::WIDTH] * + $site[CrawlConstants::HEIGHT]; + if ($size < 100000) { + $meta_ids[] = 'media:image-small'; + } else if ($size < 400000) { + $meta_ids[] = 'media:image-medium'; + } else { + $meta_ids[] = 'media:image-large'; + } + } } else { - $meta_ids[] = (stripos($site[CrawlConstants::TYPE], - "image") !== false) ? 'media:image' : 'media:text'; + $meta_ids[] = 'media:text'; } if (!empty($site[CrawlConstants::IS_VR])) { $meta_ids[] = "media:vr"; diff --git a/src/library/processors/BmpProcessor.php b/src/library/processors/BmpProcessor.php index 9dfbe449b..6280786e3 100644 --- a/src/library/processors/BmpProcessor.php +++ b/src/library/processors/BmpProcessor.php @@ -90,15 +90,17 @@ class BmpProcessor extends ImageProcessor { if (is_string($page)) { $image = $this->imagecreatefrombmp($page); - $thumb_string = self::createThumb($image); + $summary = []; + $this->addWidthHeightSummary($summary, $page); $summary[self::TITLE] = ""; - $summary[self::DESCRIPTION] = "Image of ". + $summary[self::DESCRIPTION] = UrlParser::getDocumentFilename($url); $summary[self::LINKS] = []; $summary[self::PAGE] = "<html><body><div><img src='data:image/bmp;base64," . base64_encode($page)."' alt='".$summary[self::DESCRIPTION]. "' /></div></body></html>"; + $thumb_string = self::createThumb($image); $summary[self::THUMB] = 'data:image/jpeg;base64,'. base64_encode($thumb_string); } @@ -158,11 +160,15 @@ class BmpProcessor extends ImageProcessor } $x = 0; $y++; - if ($y > $height) break; + if ($y > $height) { + break; + } } $i_pos = $i << 1; - if (!isset($body[$i_pos + 5])) { break; } - $r =hexdec($body[$i_pos + 4] . $body[$i_pos + 5]); + if (!isset($body[$i_pos + 5])) { + break; + } + $r = hexdec($body[$i_pos + 4] . $body[$i_pos + 5]); $g = hexdec($body[$i_pos + 2] . $body[$i_pos + 3]); $b = hexdec($body[$i_pos].$body[$i_pos + 1]); $color = imagecolorallocate($image, $r, $g, $b); @@ -172,4 +178,4 @@ class BmpProcessor extends ImageProcessor unset($body); return $image; } -} \ No newline at end of file +} diff --git a/src/library/processors/GifProcessor.php b/src/library/processors/GifProcessor.php index b99d4015a..c88053b2c 100755 --- a/src/library/processors/GifProcessor.php +++ b/src/library/processors/GifProcessor.php @@ -73,22 +73,30 @@ class GifProcessor extends ImageProcessor */ public function process($page, $url) { - static $gif_num = 0; if (is_string($page)) { set_error_handler(null); - $gif_num++; $image = @imagecreatefromstring($page); set_error_handler(C\NS_CONFIGS . "yioop_error_handler"); - $thumb_string = self::createThumb($image); + $summary = []; + $this->addWidthHeightSummary($summary, $page); $summary[self::TITLE] = ""; $summary[self::DESCRIPTION] = - "Image of " . UrlParser::getDocumentFilename($url); + $xmp_data = $this->getXmpData($page); + if ($xmp_data) { + $summary[self::DESCRIPTION] = + UrlParser::getDocumentFilename($url) . "\nXMP Data\n". + $xmp_data; + } else { + $summary[self::DESCRIPTION] = + UrlParser::getDocumentFilename($url); + } $summary[self::LINKS] = []; $summary[self::PAGE] = "<html><body><div><img src='data:image/gif;base64,". base64_encode($page) . "' alt='" . $summary[self::DESCRIPTION]. "' /></div></body></html>"; + $thumb_string = self::createThumb($image); $summary[self::THUMB] = 'data:image/jpeg;base64,'. base64_encode($thumb_string); } diff --git a/src/library/processors/ImageProcessor.php b/src/library/processors/ImageProcessor.php index 9a8f67043..4f130996b 100755 --- a/src/library/processors/ImageProcessor.php +++ b/src/library/processors/ImageProcessor.php @@ -31,6 +31,7 @@ namespace seekquarry\yioop\library\processors; use seekquarry\yioop\configs as C; +use seekquarry\yioop\library as L; /** * Base abstract class common to all processors used to create crawl summary @@ -56,6 +57,46 @@ class ImageProcessor extends PageProcessor { return null; } + /** + * Given an $image_string determines if possible its width and height + * then assigns the values into the CrawlConstants:WIDTH, + * CrawlConstants:HEIGHT fields of $summary + * + * @param arrray &$summary to write the width and height into + * @param string $image_string the image represented as a character string + * @return array summary information including a thumbnail and a + * description (where the description is just the url) + */ + public function addWidthHeightSummary(&$summary, $image_string) + { + set_error_handler(null); + $image_info = @getimagesizefromstring($image_string); + set_error_handler(C\NS_CONFIGS . "yioop_error_handler"); + if (!empty($image_info[0]) && !empty($image_info[1])) { + list($summary[self::WIDTH], $summary[self::HEIGHT], ) = + $image_info; + } + } + /** + * Given an image try to extract and XMP info from it. + * + * @param string $image_string the image represented as a character string + * @return array XMP data converted from XML format to an array-like format + */ + public function getXmpData($image_string) + { + $xmp_data = ""; + if (function_exists("simplexml_load_string") && + preg_match('/\<x\:xmpmeta.+\<\/x\:xmpmeta\>/s', $image_string, + $match)) { + $xml_no_ns = preg_replace("/\<\/\w+\:/", "</", $match[0]); + $xml_no_ns = preg_replace("/\<\w+\:/", "<", $xml_no_ns); + $xmp_xml = simplexml_load_string($xml_no_ns); + $xmp_array = json_decode(json_encode($xmp_xml), true); + $xmp_data = print_r($xmp_array, true); + } + return $xmp_data; + } /** * Used to create a thumbnail from an image object * diff --git a/src/library/processors/JpgProcessor.php b/src/library/processors/JpgProcessor.php index c09c7c410..d52f463e3 100755 --- a/src/library/processors/JpgProcessor.php +++ b/src/library/processors/JpgProcessor.php @@ -81,7 +81,8 @@ class JpgProcessor extends ImageProcessor set_error_handler(null); $image = @imagecreatefromstring($page); set_error_handler(C\NS_CONFIGS . "yioop_error_handler"); - $thumb_string = self::createThumb($image); + $summary = []; + $this->addWidthHeightSummary($summary, $page); $summary[self::TITLE] = ""; $file_name = UrlParser::getDocumentFilename($url); if (function_exists("exif_read_data")) { @@ -94,16 +95,24 @@ class JpgProcessor extends ImageProcessor } $temp_file = $temp_dir . L\crawlHash($url) . ".jpg"; file_put_contents($temp_file, $page); + set_error_handler(null); $summary[self::DESCRIPTION] = "$file_name\nEXIF DATA\n". print_r(exif_read_data($temp_file), true); + set_error_handler(C\NS_CONFIGS . "yioop_error_handler"); } else { $summary[self::DESCRIPTION] = $file_name; } + $xmp_data = $this->getXmpData($page); + if ($xmp_data) { + $summary[self::DESCRIPTION] .= "\nXMP Data\n". + $xmp_data; + } $summary[self::LINKS] = []; $summary[self::PAGE] = "<html><body><div><img src='data:image/jpeg;base64," . base64_encode($page)."' alt='". $file_name . "' /></div></body></html>"; + $thumb_string = self::createThumb($image); $summary[self::THUMB] = 'data:image/jpeg;base64,'. base64_encode($thumb_string); } diff --git a/src/library/processors/PngProcessor.php b/src/library/processors/PngProcessor.php index 7097c2de8..f74a863d7 100755 --- a/src/library/processors/PngProcessor.php +++ b/src/library/processors/PngProcessor.php @@ -77,16 +77,25 @@ class PngProcessor extends ImageProcessor set_error_handler(null); $image = @imagecreatefromstring($page); set_error_handler(C\NS_CONFIGS . "yioop_error_handler"); - $thumb_string = self::createThumb($image); + $summary = []; + $this->addWidthHeightSummary($summary, $page); $summary[self::TITLE] = ""; - $summary[self::DESCRIPTION] = "Image of ". - UrlParser::getDocumentFilename($url); + $xmp_data = $this->getXmpData($page); + if ($xmp_data) { + $summary[self::DESCRIPTION] = + UrlParser::getDocumentFilename($url) . "\nXMP Data\n". + $xmp_data; + } else { + $summary[self::DESCRIPTION] = + UrlParser::getDocumentFilename($url); + } $summary[self::LINKS] = []; $summary[self::PAGE] = "<html><body><div><img src='data:image/png;base64,". base64_encode($page). "' alt='".$summary[self::DESCRIPTION]. "' /></div></body></html>"; + $thumb_string = self::createThumb($image); $summary[self::THUMB] = 'data:image/jpeg;base64,' . base64_encode($thumb_string); } diff --git a/src/library/processors/SvgProcessor.php b/src/library/processors/SvgProcessor.php index 66fe6760e..8acc74291 100644 --- a/src/library/processors/SvgProcessor.php +++ b/src/library/processors/SvgProcessor.php @@ -81,6 +81,7 @@ class SvgProcessor extends TextProcessor */ public function process($page, $url) { + $summary = []; if (is_string($page)) { $dom = self::dom($page); if (!$dom) { @@ -89,6 +90,15 @@ class SvgProcessor extends TextProcessor } if ($dom !== false && isset($dom->documentElement)) { $summary[self::TITLE] = self::title($dom, $page); + $svg = $dom->documentElement; + if ($svg->hasAttribute("width")) { + $width = $svg->getAttribute("width"); + $summary[self::WIDTH] = L\convertPixels($width); + } + if ($svg->hasAttribute("height")) { + $height = $svg->getAttribute("height"); + $summary[self::HEIGHT] = L\convertPixels($height); + } $summary[self::DESCRIPTION] = self::description($dom, $page); $summary[self::LINKS] = []; $summary[self::PAGE] = "<!DOCTYPE html>" . diff --git a/src/views/elements/ConfigureElement.php b/src/views/elements/ConfigureElement.php index c625b068d..13d7767cf 100644 --- a/src/views/elements/ConfigureElement.php +++ b/src/views/elements/ConfigureElement.php @@ -107,7 +107,9 @@ class ConfigureElement extends Element <?php if ($data['PROFILE']) { ?> <div class="top-margin"> <fieldset class="extra-wide-field"><legend><?= - tl('configure_element_debug_display') ?></legend> + tl('configure_element_debug_display') ?> <?= + $this->view->helper("helpbutton")->render( + "Debug Display", $data[C\CSRF_TOKEN]) ?></legend> <label for="error-info"><input id='error-info' type="checkbox" name="ERROR_INFO" value="<?= C\ERROR_INFO ?>" <?php if (($data['DEBUG_LEVEL'] & C\ERROR_INFO) == @@ -138,7 +140,9 @@ class ConfigureElement extends Element </div> <div class="top-margin"> <fieldset class="extra-wide-field"><legend><?= - tl('configure_element_site_access')?></legend> + tl('configure_element_site_access')?> <?= + $this->view->helper("helpbutton")->render( + "Search Access", $data[C\CSRF_TOKEN]) ?></legend> <label for="web-access"><input id='error-info' type="checkbox" name="WEB_ACCESS" value="true" <?php if ( $data['WEB_ACCESS']==true) { @@ -157,7 +161,9 @@ class ConfigureElement extends Element </fieldset> </div> <div class="top-margin"> - <fieldset><legend><?=tl('configure_element_crawl_robot')?></legend> + <fieldset><legend><?=tl('configure_element_crawl_robot')?> <?= + $this->view->helper("helpbutton")->render( + "Crawl Robot Set-up", $data[C\CSRF_TOKEN]) ?></legend> <div><b><label for="crawl-robot-name"><?= tl('configure_element_robot_name')?></label></b> <input type="text" id="crawl-robot-name"