<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\SyntheticWeb;
use seekquarry\yioop\library\UrlParser;
/**
* For crawlHash
*/
require_once __DIR__ . "/Utility.php";
/**
* The language and character set functions this file calls are loaded
* here.
*/
require_once __DIR__ . "/LocaleFunctions.php";
/**
* FetchUrl downloads pages for the crawler and for Yioop's own requests,
* over HTTP, HTTPS or Gopher, one page at a time or many at once.
*
* @author Chris Pollett
*/
class FetchUrl implements CrawlConstants
{
/**
* CHALLENGE_MARKERS are pieces of text that a bot-protection service
* puts in the page it sends in place of a site's own page, while it
* checks whether the visitor is a person. /_Incapsula_Resource is the
* script address of Imperva Incapsula, a service that guards sites
* such as www.pepsi.com. isBotChallenge looks for them.
*/
const CHALLENGE_MARKERS = ["/_Incapsula_Resource"];
/**
* MOST_CHALLENGE_TEXT is the most bytes of readable text a page holding
* one of the CHALLENGE_MARKERS may have and still count as a
* challenge. A site's own page that loads the same script has far
* more text than this.
*/
const MOST_CHALLENGE_TEXT = 256;
/**
* Most hosts whose cookies keepCookies holds at once; past it, the host
* whose cookies changed longest ago is dropped.
*/
const HOST_COOKIE_LIMIT = 10000;
/**
* WEB_BOT_AUTH_KEY_FILE is the file, in SECURITY_DIR, that holds the
* Ed25519 key pair the crawler signs requests with under Web Bot Auth,
* base64 encoded.
*/
const WEB_BOT_AUTH_KEY_FILE = "web_bot_auth.key";
/**
* WEB_BOT_AUTH_SECONDS is how long a Web Bot Auth signature is good for
* after it is made. Cloudflare asks for a short time, so a copied
* request cannot be sent again for long, and says a minute is enough.
*/
const WEB_BOT_AUTH_SECONDS = 60;
/**
* TLS12_CIPHERS_KEPT are the TLS 1.2 cipher suites, in OpenSSL's names,
* that every https connection offers: the two nearly every server
* accepts, so a connection can still be made whatever else is left
* out. varyTlsHello adds some of TLS12_CIPHERS_MAYBE to them.
*/
const TLS12_CIPHERS_KEPT = ["ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256"];
/**
* TLS12_CIPHERS_MAYBE are the TLS 1.2 cipher suites varyTlsHello offers
* or leaves out at random for each https connection
*/
const TLS12_CIPHERS_MAYBE = ["ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305", "ECDHE-RSA-AES128-SHA",
"ECDHE-RSA-AES256-SHA", "AES128-GCM-SHA256", "AES256-GCM-SHA384",
"AES128-SHA", "AES256-SHA"];
/**
* TLS13_CIPHERS_KEPT is the TLS 1.3 cipher suite every https connection
* offers, the one every TLS 1.3 server must accept
*/
const TLS13_CIPHERS_KEPT = ["TLS_AES_128_GCM_SHA256"];
/**
* TLS13_CIPHERS_MAYBE are the TLS 1.3 cipher suites varyTlsHello
* offers or leaves out at random for each https connection
*/
const TLS13_CIPHERS_MAYBE = ["TLS_AES_256_GCM_SHA384",
"TLS_CHACHA20_POLY1305_SHA256"];
/**
* TLS_GROUPS_KEPT are the key exchange groups (elliptic curves) every
* https connection offers
*/
const TLS_GROUPS_KEPT = ["X25519", "P-256"];
/**
* TLS_GROUPS_MAYBE are the key exchange groups varyTlsHello offers or
* leaves out at random for each https connection
*/
const TLS_GROUPS_MAYBE = ["P-384", "P-521"];
/**
* HTTP1_SHARE is the share of https connections, one in this many,
* that varyTlsHello asks to speak HTTP/1.1 rather than HTTP/2, which
* changes the protocol the connection's first message names
*/
const HTTP1_SHARE = 4;
/**
* THIS_MACHINE_CODE is the HTTP code getPages gives a crawl page whose
* host name led to an address of the fetcher's own machine. It is 421
* Misdirected Request, which says the request went to a server that
* cannot answer for the host. As an error other than 404 it stops the
* crawl of the host when it is the code of the host's robots.txt, and
* the Fetcher does not send it back to be fetched again.
*/
const THIS_MACHINE_CODE = 421;
/**
* $host_cookies holds the cookies each host has set during this crawl,
* as host => [cookie name => value], in the fetcher's memory only.
* keepCookies fills it from Set-Cookie headers, and cookieHeaderFor
* sends a host its own cookies back, never another host's.
* @var array
*/
public static $host_cookies = [];
/**
* $web_bot_auth_key_pair holds the signing key pair webBotAuthKeyPair
* read or made, so the key file is read once a process.
* @var string
*/
public static $web_bot_auth_key_pair = null;
/**
* $site_cookies holds the cookies the crawl's Site Cookies option gives,
* as domain => the text of a Cookie header, sent to that domain and its
* subdomains. They let
* a crawl pass a consent page that sets its cookie only by script when
* a button is clicked. setSiteCookies fills it from the option's lines.
* @var array
*/
public static $site_cookies = [];
/**
* $last_handler is the curl multi handle of the last getPages call,
* with the handles of any transfers still running when it stopped
* waiting. It is kept until
* the next call replaces it, since freeing a transfer that is still
* looking up its host's address waits for the lookup to end.
* @var \CurlMultiHandle
*/
public static $last_handler = null;
/**
* $local_ip_cache maps the host names of the machines in this Yioop
* cluster to their addresses, so each is looked up once.
*
* @var array
*/
static $local_ip_cache = [];
/**
* localHosts lists the host names and addresses that refer to the
* Yioop instance answering the current request. getPage, getPages,
* and isSelfUrl use this to decide whether a requested url points
* back at this instance itself, in which case the request is made
* through WebSite::processInternalRequest rather than over the
* network, as a network loopback to one's own url would deadlock a
* single-process web server. The list contains the usual loopback
* names, the current SERVER_NAME, the host of C\baseUrl() (this
* instance's configured url), together with the www-prefixed and
* bare variant of each domain, and the IP address the current
* request arrived on, so a machine configured by its bare IP still
* matches. C\p('NAME_SERVER')'s host is added only when this instance is
* the name server (the request arrived for that host); on a fetcher,
* where C\p('NAME_SERVER') names a remote host, it is left out so those
* requests still go over the network.
*
* @return array host names and addresses that refer to this instance
*/
public static function localHosts()
{
$local_hosts = ['localhost', '::1', '0.0.0.0', '127.0.0.1'];
$candidates = [];
if (!empty($_SERVER['SERVER_NAME'])) {
$candidates[] = $_SERVER['SERVER_NAME'];
}
$base_host = parse_url(C\baseUrl(), PHP_URL_HOST);
if (!empty($base_host)) {
$candidates[] = $base_host;
}
/*
When this instance is the name server, the name server's own
host names itself rather than a remote machine, so add it
too. We only do this when the request arrived for that host
(SERVER_NAME) or on the IP the name server resolves to, so a
fetcher, where NAME_SERVER is a remote host, is unaffected.
*/
if (C\nsdefined("NAME_SERVER")) {
$name_server_host = parse_url(C\p('NAME_SERVER'), PHP_URL_HOST);
if (!empty($name_server_host) &&
(($_SERVER['SERVER_NAME'] ?? "") == $name_server_host ||
in_array($name_server_host, $candidates) ||
C\nsdefined("HOSTS_NAME_SERVER"))) {
$candidates[] = $name_server_host;
}
}
foreach ($candidates as $candidate) {
$local_hosts[] = $candidate;
if (str_starts_with($candidate, "www.")) {
$local_hosts[] = substr($candidate, strlen("www."));
} else {
$local_hosts[] = "www." . $candidate;
}
}
/*
The IP address the current request actually arrived on is
unambiguously this machine, so a machine configured by its IP
(rather than a host name) is still recognized as ourselves and
answered in process instead of over a network loopback.
*/
if (!empty($_SERVER['SERVER_ADDR'])) {
$local_hosts[] = $_SERVER['SERVER_ADDR'];
}
return array_unique($local_hosts);
}
/**
* isSelfUrl tells if a url points back at this Yioop instance, so the
* caller can answer it in process with
* WebSite::processInternalRequest instead of sending a network
* request to oneself, which would deadlock a single-process web
* server. A url is ours when this process is its own web server, the
* url's host is one of localHosts(), and the url's port is the one
* the current request came in on.
*
* @param string $url the url to test
* @return bool true if the url refers to this instance
*/
public static function isSelfUrl($url)
{
if (!C\nsdefined("IS_OWN_WEB_SERVER")) {
return false;
}
$url_parts = @parse_url($url ?? "");
$host = empty($url_parts["host"]) ? false : $url_parts["host"];
if ($host === false || !in_array($host, self::localHosts())) {
return false;
}
$port = empty($url_parts["port"]) ? ($_SERVER['SERVER_PORT'] ?? "") :
$url_parts["port"];
return ($port == ($_SERVER['SERVER_PORT'] ?? ""));
}
/**
* reachesThisMachine says if an address is one a connection to which
* stays on this machine: a loopback address, 127.0.0.0/8 or ::1, or an
* unspecified one, 0.0.0.0/8 or ::, which Linux and macOS also connect
* to this machine. An IPv6 address holding an IPv4 one, as
* ::ffff:127.0.0.1, is judged by the IPv4 address. A name that is not
* an address gives false.
*
* @param string $address an IPv4 or IPv6 address
* @return bool whether a connection to the address stays on this machine
*/
public static function reachesThisMachine($address)
{
$packed = @inet_pton(trim($address ?? "", "[]"));
if ($packed === false) {
return false;
}
$mapped_prefix = str_repeat("\0", 10) . "\xff\xff";
if (strlen($packed) == 16 &&
str_starts_with($packed, $mapped_prefix)) {
$packed = substr($packed, strlen($mapped_prefix));
}
if (strlen($packed) == 4) {
return in_array(ord($packed[0]), [0, 127]);
}
return in_array($packed, [str_repeat("\0", 16),
str_repeat("\0", 15) . "\x01"], true);
}
/**
* leadsToThisMachine says if a crawl url was sent, or would be sent,
* to this machine though its host names another one. That is so when
* one of the addresses the url's host led to reaches this machine and
* the host is neither an address nor one of localHosts(). A site's
* name may lead here when its DNS record says 127.0.0.1, or when a
* name server that blocks it answers 0.0.0.0; either way the page
* that comes back is this machine's, not the site's. getPages refuses
* such urls when asked to with its $public_only argument.
*
* @param string $url the url asked for
* @param array $addresses the addresses its host led to, as the queue
* server gave or curl connected to
* @return bool whether the url is not to be fetched or kept
*/
public static function leadsToThisMachine($url, $addresses)
{
$host = @parse_url($url ?? "", PHP_URL_HOST);
if (empty($host) || in_array($host, self::localHosts()) ||
filter_var(trim($host, "[]"), FILTER_VALIDATE_IP) !== false) {
return false;
}
foreach ($addresses as $address) {
if (self::reachesThisMachine($address)) {
return true;
}
}
return false;
}
/**
* envProxyFor says if the environment names a proxy that curl sends
* urls of a scheme through: http_proxy for http (curl reads only the
* lowercase name there), https_proxy or HTTPS_PROXY for https, and
* all_proxy or ALL_PROXY for any scheme. getPages does not judge where
* such a url's host led by the address curl connected to, as that is
* the proxy's.
*
* @param string|bool $scheme the url's scheme, false if it has none
* @return bool whether curl may reach urls of the scheme by a proxy
*/
public static function envProxyFor($scheme)
{
$scheme = strtolower((string)$scheme);
$names = ["all_proxy", "ALL_PROXY", "{$scheme}_proxy"];
if ($scheme != "http") {
$names[] = strtoupper($scheme) . "_PROXY";
}
foreach ($names as $name) {
if (!empty(getenv($name))) {
return true;
}
}
return false;
}
/**
* refuseSite fills a site in getPages's array the way a fetch would,
* for a url leadsToThisMachine refused: code THIS_MACHINE_CODE, no
* page, and a header saying where the url's host led, so the cache
* view of the page tells why it has no text.
*
* @param array &$site the site to fill
* @param string $url the site's url, for the log
* @param string $value the component of the site that holds its page
* @param array $addresses the addresses the url's host led to
* @param string $header what curl wrote of the fetch, if it was made
*/
public static function refuseSite(&$site, $url, $value, $addresses,
$header = "")
{
$site[self::HTTP_CODE] = self::THIS_MACHINE_CODE;
$site[self::HEADER] = "* Not kept: the host led to " .
implode(", ", $addresses) . ", which is this machine\n" .
$header;
$site[$value] = "";
$site[self::TYPE] = "text/plain";
$site[self::ENCODING] = "UTF-8";
$site[self::SIZE] = 0;
$site[self::DNS_TIME] ??= 0;
$site[self::TOTAL_TIME] ??= 0;
$site[self::IP_ADDRESSES] = array_values($addresses);
$site[self::TIMESTAMP] = time();
unset($site[CrawlConstants::LOCATION]);
crawlLog(" $url led to " .
implode(", ", $addresses) . ", this machine, so is not kept");
}
/**
* syntheticGetPages is what a synthetic crawl downloads with in place
* of getPages, so a crawl of any size can run on one machine with no
* web behind it. For each site's url it works out the page's number,
* builds the page's html from the crawl's synthetic settings, and fills
* the same fields getPages fills, so everything after the fetch runs
* unchanged. A url that is not one of
* the made-up web's own is left with a 404, the way a fetch of a page
* that is not there would leave it.
*
* @param array $sites an array whose entries each have a url under $key
* @param array $settings the crawl's synthetic settings: graph_type,
* out_degree, seed, power_law_exponent, term_exponent, doc_length,
* length_spread, domain_link_probability, image_link_probability
* @param string $key the component of a site that holds its url
* @param string $value the component in which to store the made-up page
* @return array the sites, each filled with its made-up page
*/
public static function syntheticGetPages($sites, $settings,
$key = CrawlConstants::URL, $value = CrawlConstants::PAGE)
{
for ($i = 0; $i < count($sites); $i++) {
$url_parts = explode("###", $sites[$i][$key] ?? "");
$url = $url_parts[0];
$sites[$i][$key] = $url;
$sites[$i][self::TIMESTAMP] = time();
if (!empty($url_parts[2])) {
$sites[$i][self::URL_PARENT] = $url_parts[2];
}
$tuple = SyntheticWeb::pageTuple($url, $settings["graph_type"]);
if ($tuple == []) {
$sites[$i][self::HTTP_CODE] = "404";
$sites[$i][self::HEADER] = "HTTP/1.1 404 Not Found\r\n\r\n";
$sites[$i][$value] = "";
$sites[$i][self::TYPE] = "text/plain";
$sites[$i][self::ENCODING] = "UTF-8";
$sites[$i][self::SIZE] = 0;
$sites[$i][self::DNS_TIME] = 0;
$sites[$i][self::TOTAL_TIME] = 0;
$sites[$i][self::IP_ADDRESSES] =
[SyntheticWeb::ipAddress($url)];
continue;
}
if (SyntheticWeb::isImageUrl($url)) {
$body = SyntheticWeb::imageBytes($url);
$content_type = "image/jpeg";
$encoding = "";
} else {
$body = SyntheticWeb::pageHtml($tuple, $settings);
$content_type = "text/html";
$encoding = "UTF-8";
}
$sites[$i][self::HEADER] = "HTTP/1.1 200 OK\r\nContent-Type: " .
$content_type . "\r\n\r\n";
$sites[$i][$value] = $body;
$sites[$i][self::TYPE] = $content_type;
$sites[$i][self::ENCODING] = $encoding;
$sites[$i][self::SIZE] = strlen($body);
$sites[$i][self::DNS_TIME] = 0;
$sites[$i][self::TOTAL_TIME] = 0;
$sites[$i][self::HTTP_CODE] = 200;
$sites[$i][self::IP_ADDRESSES] =
[SyntheticWeb::ipAddress($url)];
}
return $sites;
}
/**
* varyTlsHello makes the first message of an https connection, the TLS
* ClientHello, differ from one connection to the next, so the JA3 and
* JA4 fingerprints a site may take of it, and block on, are not always
* the same. JA3 is a hash of the message's TLS version, cipher suites,
* extensions, groups and point formats, each in the order sent. JA4
* gives the TLS version, the number of cipher suites and extensions and
* the first protocol named for the connection, then hashes the sorted
* cipher suites and the sorted extensions with signature algorithms.
* So each connection offers the KEPT cipher suites and groups and a
* random choice of the MAYBE ones, all in a random order, and one in
* HTTP1_SHARE asks for HTTP/1.1 rather than HTTP/2. The order changes
* JA3; which suites are offered, and the protocol, change JA4.
* Cipher suite and group names are OpenSSL's, so curl built on another
* TLS library, as on some Windows and macOS builds, is left as it is.
* getPages calls it for each https url.
*
* @param \CurlHandle $handle the curl handle of the connection
*/
public static function varyTlsHello($handle)
{
static $is_openssl = null;
$is_openssl ??= str_starts_with(curl_version()["ssl_version"] ?? "",
"OpenSSL");
if (!$is_openssl) {
return;
}
$choose = function ($kept, $maybe) {
$offered = $kept;
foreach ($maybe as $name) {
if (mt_rand(0, 1) == 1) {
$offered[] = $name;
}
}
shuffle($offered);
return implode(":", $offered);
};
curl_setopt($handle, CURLOPT_SSL_CIPHER_LIST,
$choose(self::TLS12_CIPHERS_KEPT, self::TLS12_CIPHERS_MAYBE));
curl_setopt($handle, CURLOPT_TLS13_CIPHERS,
$choose(self::TLS13_CIPHERS_KEPT, self::TLS13_CIPHERS_MAYBE));
/* PHP names this option from 8.2 on */
if (defined("CURLOPT_SSL_EC_CURVES")) {
curl_setopt($handle, CURLOPT_SSL_EC_CURVES,
$choose(self::TLS_GROUPS_KEPT, self::TLS_GROUPS_MAYBE));
}
if (mt_rand(1, self::HTTP1_SHARE) == 1) {
curl_setopt($handle, CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_1_1);
}
}
/**
* isBotChallenge says whether a downloaded page is a bot-protection
* service's challenge rather than the site's own page. Such a page
* holds one of the CHALLENGE_MARKERS and almost no readable text,
* since it only runs a script that checks the visitor. The Fetcher
* calls it so a challenge is not indexed as the site's page. Yioop
* does not run the challenge's script: the page is fetched again
* later, when the site may answer with its own page.
*
* @param string $page the page as downloaded
* @return bool whether the page is a challenge
*/
public static function isBotChallenge($page)
{
if (!is_string($page)) {
return false;
}
$has_marker = false;
foreach (self::CHALLENGE_MARKERS as $marker) {
if (stripos($page, $marker) !== false) {
$has_marker = true;
break;
}
}
if (!$has_marker) {
return false;
}
$text = preg_replace(["/<script\b.*?<\/script>/is",
"/<style\b.*?<\/style>/is"], " ", $page);
$text = trim(preg_replace("/\s+/u", " ",
html_entity_decode(strip_tags($text ?? ""))) ?? "");
return strlen($text) < self::MOST_CHALLENGE_TEXT;
}
/**
* getPages downloads the pages of an array of sites at once, through
* curl's multi interface, and fills each site with its page and what
* its response said.
*
* @param array $sites an array containing urls of pages to request
* @param bool $timer flag, true means print timing statistics to log
* @param int $page_range_request maximum number of bytes to download/page
* 0 means download all
* @param string $temp_dir folder to store temporary ip header info
* @param string $key the component of $sites[$i] that has the value of
* a url to get defaults to URL
* @param string $value component of $sites[$i] in which to store the
* page that was gotten
* @param bool $minimal if true do a faster request of pages by not
* doing things like extract HTTP headers sent, etcs. Also will
* not use Expect header
* @param array $post_data data to be POST'd to each site
* @param bool $follow whether to follow redirects or not
* @param string $tor_proxy url of a proxy that knows how to download
* .onion urls
* @param array $proxy_servers if not [], then an array of proxy
* server to use rather than to directly download web pages from
* the current machine
* @param ?callable $while_waiting work to do while the downloads are
* waiting on the network, such as the fetcher compressing pages it
* already has. It is called once each time round the wait loop and
* returns whether it did any work; when it did, the loop checks
* the transfers again at once instead of waiting on their sockets.
* @param bool $public_only whether to refuse a url whose host name led
* to this machine, as leadsToThisMachine says. The Fetcher asks
* for it, so a site whose name leads here is not indexed with this
* machine's pages. A url the queue server gave an address for is
* refused before any connection is made. Otherwise the addresses
* curl connected to are checked once the fetch is done, except
* through a proxy, whose address is the one curl connects to.
*
* @return array an updated array with the contents of those pages
*/
public static function getPages($sites, $timer = false,
$page_range_request = C\PAGE_RANGE_REQUEST, $temp_dir = "",
$key = CrawlConstants::URL, $value = CrawlConstants::PAGE,
$minimal = false, $post_data = null, $follow = false, $tor_proxy = "",
$proxy_servers = [], $while_waiting = null, $public_only = false)
{
$root_errors = [];
/* should think of curl_multi_init as returning thread handler
curl_init returns a handle to an open connection and we can
then execute operations on this connection concurrent with other
handles by adding it to the group of handles managed by
the curl_multi_init thread handler. Removing a handle from
the curl_multi_init thread handler does not close the connection
it just means we can't do concurrent operations with it.
*/
$agent_handler = curl_multi_init();
$active = null;
$start_time = microtime(true);
if (empty($temp_dir)) {
$temp_dir = C\TEMP_DIR . "/";
if (!file_exists($temp_dir)) {
mkdir($temp_dir);
}
}
$memory_limit = memoryCeiling(C\MEMORY_FILL_FACTOR);
$memory_limit_high = memoryCeiling(C\MEMORY_FILL_FACTOR_HIGH);
if (memory_get_usage() > $memory_limit) {
self::$local_ip_cache = [];
}
//Set-up requests
$num_sites = count($sites);
$num_started = 0;
for ($i = 0; $i < $num_sites; $i++) {
$is_gopher = false;
$sites[$i][CrawlConstants::IS_GOPHER_URL] = $is_gopher;
if (!isset($sites[$i][$key])) {
continue;
}
$url_parts = @parse_url($sites[$i][$key]);
$host = (empty($url_parts["host"])) ? false :
$url_parts["host"];
$scheme = (empty($url_parts["scheme"])) ? false :
$url_parts["scheme"];
$is_local_host = ($host !== false &&
in_array($host, self::localHosts()));
if (self::isSelfUrl($sites[$i][$key])) {
$sites[$i][0] = "INTERNAL";
if (empty($GLOBALS['web_site'])) {
$sites[$i][$value] = "";
continue;
}
$web_site = $GLOBALS['web_site'];
$post_info = $post_data[$i] ?? null;
$sites[$i][$value] = $web_site->processInternalRequest(
$sites[$i][$key], !$minimal, $post_info);
continue;
}
list($sites[$i][$key], $url, $headers, $dns_resolve, $referer,
$ip_address) = self::prepareUrlHeaders($sites[$i][$key],
$proxy_servers, $temp_dir, !$minimal && !$follow && $post_data);
if (empty($url)) {
continue;
}
if ($public_only && !empty($dns_resolve) &&
self::leadsToThisMachine($url, [$ip_address])) {
self::refuseSite($sites[$i], $url, $value, [$ip_address]);
continue;
}
if ($headers == "gopher") {
$is_gopher = true;
$sites[$i][CrawlConstants::IS_GOPHER_URL] = $is_gopher;
$headers = [];
} else {
$cookie = self::cookieHeaderFor($sites[$i][$key]);
if ($cookie != "") {
$headers[] = "Cookie: $cookie";
}
}
$sites[$i][0] = curl_init();
/* curl's own cookie store, empty at the start of each fetch,
carries a cookie a site sets on a redirect to the next
request of the same fetch, as some sites need to answer.
It is started by naming an empty file to read cookies from:
the null device, which curl opens without the warning an
empty name writes into the verbose output. */
curl_setopt($sites[$i][0], CURLOPT_COOKIEFILE,
(PHP_OS_FAMILY === "Windows") ? "NUL" : "/dev/null");
if (!$minimal) {
/* curl keeps the request it sent, read back once the page
is in. Keeping it in memory, rather than in a file,
costs a fetch no open file. */
curl_setopt($sites[$i][0], CURLINFO_HEADER_OUT, true);
}
if (in_array(UrlParser::getHost($url),
C\USER_AGENT_ALTERNATIVE_HOSTS)) {
curl_setopt($sites[$i][0], CURLOPT_USERAGENT,
C\USER_AGENT_ALTERNATIVE);
} else {
curl_setopt($sites[$i][0], CURLOPT_USERAGENT, C\USER_AGENT);
}
curl_setopt($sites[$i][0], CURLOPT_IPRESOLVE,
CURL_IPRESOLVE_WHATEVER);
if (!empty($dns_resolve)) {
curl_setopt($sites[$i][0], CURLOPT_RESOLVE, [$dns_resolve]);
}
/* curl refuses an option holding a null byte, which a url
writes as %00 */
$url = str_replace("\0", "%00", $url);
curl_setopt($sites[$i][0], CURLOPT_URL, $url);
curl_setopt($sites[$i][0], CURLOPT_REFERER,
str_replace("\0", "%00", $referer));
$follow_site = $follow;
$url_parts = @parse_url($url);
if (!$follow && in_array($url_parts['path'] ?? "", ["", "/"]) &&
empty($url_parts['query'])) {
/* A site's root often redirects to a page of the same site,
as https://www.ibm.com/ does to /us-en. Following it lets
the root keep the page, so a search for the site lists
the root; the redirect is checked to stay on the same
host once the response is in. */
$follow_site = true;
$sites[$i]['ROOT_FOLLOW'] = $url;
}
if (strcmp(substr($url, -10), "robots.txt") == 0 ) {
$sites[$i]['ROBOT'] = true;
$follow_site = true; /*wikipedia redirects their robot page.
grr want to force this for robots pages
*/
}
curl_setopt($sites[$i][0], CURLOPT_FOLLOWLOCATION, $follow_site);
if ($scheme == "https" && defined("CURL_HTTP_VERSION_2_0")) {
curl_setopt($sites[$i][0], CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_2_0);
}
if ($scheme == "https") {
self::varyTlsHello($sites[$i][0]);
}
curl_setopt($sites[$i][0], CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($sites[$i][0], CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($sites[$i][0], CURLOPT_AUTOREFERER, true);
curl_setopt($sites[$i][0], CURLOPT_RETURNTRANSFER, true);
/*
A url whose host is one of ours but that was not answered
in process above (for example its port did not match the
listening port, or the web server object was not
available) is a loopback to ourselves. A single-process
server cannot answer it while busy with this request, so
fetch it with a short timeout rather than the full page
timeout, turning a would-be multi-second freeze of every
connection into a quick failure.
*/
$request_timeout = $is_local_host ? C\LOOPBACK_PAGE_TIMEOUT :
C\PAGE_TIMEOUT;
curl_setopt($sites[$i][0], CURLOPT_CONNECTTIMEOUT,
$request_timeout);
curl_setopt($sites[$i][0], CURLOPT_TIMEOUT, $request_timeout);
$proxied[$i] = self::envProxyFor($scheme);
if (stripos($url,'.onion') !== false && $tor_proxy != "") {
$proxied[$i] = true;
curl_setopt($sites[$i][0], CURLOPT_PROXY, $tor_proxy);
curl_setopt($sites[$i][0], CURLOPT_PROXYTYPE,
CURLPROXY_SOCKS5_HOSTNAME);
if ($timer) {
crawlLog("Using Tor proxy for $url..");
}
} else if ($proxy_servers != [] && !$is_gopher) {
$proxied[$i] = true;
$select_proxy = rand(0, count($proxy_servers) - 1);
$proxy_server = $proxy_servers[$select_proxy];
$proxy_parts = explode(":", $proxy_server);
$proxy_ip = $proxy_parts[0];
if (!empty($proxy_parts[1])) {
$proxy_port = $proxy_parts[1];
} else {
$proxy_port = "80";
}
$proxy_type = CURLPROXY_HTTP;
if (!empty($proxy_parts[2])) {
$pre_proxy_type = strtolower($proxy_parts[2]);
if ($pre_proxy_type != 'http') {
if ($pre_proxy_type == 'socks4') {
$proxy_type = CURLPROXY_SOCKS4;
} else if ($pre_proxy_type == 'socks4a') {
$proxy_type = CURLPROXY_SOCKS4A;
} else if ($pre_proxy_type == 'socks5') {
$proxy_type = CURLPROXY_SOCKS5;
} else if ($pre_proxy_type == 'socks5_hostname') {
$proxy_type = CURLPROXY_SOCKS5_HOSTNAME;
} else {
$proxy_type = $proxy_parts[2];
}
}
}
curl_setopt($sites[$i][0], CURLOPT_PROXY,
"$proxy_ip:$proxy_port");
curl_setopt($sites[$i][0], CURLOPT_PROXYTYPE,
$proxy_type);
if (!empty($proxy_parts[4])) {
// $proxy_parts[3] = username; $proxy_parts[4] = password
curl_setopt($sites[$i][0], CURLOPT_PROXYUSERPWD,
$proxy_parts[3] . ":" . $proxy_parts[4]);
curl_setopt($sites[$i][0], CURLOPT_PROXYAUTH,
CURLAUTH_BASIC);
}
if ($timer) {
crawlLog("Selecting proxy $select_proxy for $url");
}
}
if (!$minimal) {
curl_setopt($sites[$i][0], CURLOPT_HEADER, true);
}
//make lighttpd happier
if (!$is_gopher) {
$headers = array_merge($headers,
self::webBotAuthHeaders($url));
if (defined('CURLOPT_HEADEROPT')) {
curl_setopt($sites[$i][0], CURLOPT_HEADEROPT,
CURLHEADER_UNIFIED);
}
curl_setopt($sites[$i][0], CURLOPT_HTTPHEADER,
$headers);
}
curl_setopt($sites[$i][0], CURLOPT_ENCODING, "");
// ^ need to set for sites like at that use gzip
if ($page_range_request > 0 && empty(
$sites[$i][CrawlConstants::NO_RANGE])) {
curl_setopt($sites[$i][0], CURLOPT_RANGE, "0-".
$page_range_request);
} else if (!empty( $sites[$i][CrawlConstants::NO_RANGE])) {
crawlLog("No range used for $url");
}
if ($post_data != null) {
curl_setopt($sites[$i][0], CURLOPT_POST, true);
curl_setopt($sites[$i][0], CURLOPT_POSTFIELDS,
$post_data[$i]);
}
curl_multi_add_handle($agent_handler, $sites[$i][0]);
$num_started++;
}
if (!empty(self::$local_ip_cache['dirty'])) {
unset(self::$local_ip_cache['dirty']);
file_put_contents("$temp_dir/" . self::local_ip_cache_file,
serialize(self::$local_ip_cache));
}
if ($timer) {
crawlLog(" Init Get Pages " . (changeInMicrotime($start_time)));
}
$start_time = microtime(true);
$start = time();
//Wait for responses
$running = null;
$mrc_check = CURLM_CALL_MULTI_PERFORM;
$curlm_responses = [
CURLM_CALL_MULTI_PERFORM => "CURLM_CALL_MULTI_PERFORM",
CURLM_OK => "CURLM_OK",
CURLM_BAD_HANDLE => "CURLM_BAD_HANDLE",
CURLM_BAD_EASY_HANDLE => "CURLM_BAD_EASY_HANDLE",
CURLM_OUT_OF_MEMORY => "CURLM_OUT_OF_MEMORY",
CURLM_INTERNAL_ERROR => "CURLM_INTERNAL_ERROR",
CURLMSG_DONE => "CURLMSG_DONE"
];
set_error_handler(null);
$finished_by_second = [];
do {
$mrc = @curl_multi_exec($agent_handler, $running);
$finished_by_second[intval(microtime(true) - $start_time)] =
$num_started - $running;
if ($running && $mrc == CURLM_OK) {
$mrc_check = CURLM_OK;
if (\Fiber::getCurrent() !== null) {
/* Running inside the cooperative web server: hand
the loop back so it can serve other connections
while these transfers continue, rather than
freezing the whole single-threaded server on this
one fetch. The error handler is restored around
the suspend so work that runs in our place sees
the normal handler, not the null one this loop
installs. */
restore_error_handler();
\Fiber::suspend();
set_error_handler(null);
} else if ($while_waiting !== null && $while_waiting()) {
continue;
} else if (curl_multi_select($agent_handler, 0.05) === -1) {
/* curl_multi_select returns -1 when libcurl has no
socket to wait on yet, for example while a name is
still resolving; it returns at once in that case,
so without this brief sleep the loop would spin at
full CPU. When it does have sockets the 0.05s wait
keeps the loop from being too busy. */
usleep(50000);
}
}
} while (memory_get_usage() < $memory_limit_high &&
$mrc == $mrc_check && time() - $start < C\PAGE_TIMEOUT &&
$running > 0);
restore_error_handler();
if (time() - $start > C\PAGE_TIMEOUT && $timer) {
crawlLog(" TIMED OUT!!!");
}
if (memory_get_usage() > $memory_limit_high) {
crawlLog(" MEMORY EXHAUSTED!!!");
}
$mrc_message = (isset($curlm_responses[$mrc])) ? $curlm_responses[$mrc]
: $mrc;
/* These three lines are the trace of a batch of downloads, so
they go out only when timing is asked for, as the request time
line below does. A script that queries a distributed index
downloads its answers through here and drowned in them. */
if ($timer && php_sapi_name() == 'cli') {
crawlLog(" Final curl_multi_exec response $mrc: $mrc_message.");
crawlLog(" Final curl_multi running value $running.");
/* How many transfers had finished by the end of each second,
so the log shows how long a pass's transfers take. */
$finished_line = "";
foreach ($finished_by_second as $second => $finished) {
$finished_line .= " " . ($second + 1) . "s:$finished";
}
crawlLog(" Transfers finished by second of $num_started:" .
$finished_line);
$ceiling = ($memory_limit == PHP_INT_MAX) ?
"no memory_limit set in php.ini" :
"the process stops downloading at $memory_limit";
crawlLog(" Memory in use after the downloads " .
memory_get_usage() . " bytes; $ceiling");
}
if ($timer) {
crawlLog(" Page Request time ".(changeInMicrotime($start_time)));
}
$start_time = microtime(true);
//Process returned pages
for ($i = 0; $i < $num_sites; $i++) {
if ($timer && php_sapi_name() == 'cli') {
crawlTimeoutLog("FetchUrl initial processing of page %s of %s",
$i, $num_sites);
}
$ip_addresses = false;
if (!$minimal && ($sites[$i][0] ?? null) instanceof
\CurlHandle) {
$header = (string)curl_getinfo($sites[$i][0],
CURLINFO_HEADER_OUT);
$address = curl_getinfo($sites[$i][0], CURLINFO_PRIMARY_IP);
$ip_addresses = empty($address) ? false : [$address];
}
$is_gopher = false;
$is_internal = false;
if (!empty($sites[$i][0])) {
// Get Data and Message Code
if ($sites[$i][0] == 'INTERNAL') {
$is_internal = true;
$content = $sites[$i][$value];
} else {
$content = @curl_multi_getcontent($sites[$i][0]);
}
$is_gopher = $sites[$i][CrawlConstants::IS_GOPHER_URL];
/*
If the Transfer-encoding was chunked then the Range header
we sent was ignored. So we manually truncate the data
here
*/
if ($page_range_request > 0) {
$init_len = strlen($content);
$content = substr($content, 0, $page_range_request);
if (strlen($content) != $init_len) {
$sites[$i][CrawlConstants::CONTENT_SIZE] = $init_len;
}
}
if (empty($content) && !empty($header)) {
crawlLog(
"..Request {$sites[$i][$key]} returned no content");
crawlLog("..response header was " . $header);
}
if (isset($content) && !$minimal && !$is_gopher) {
self::keepCookies($sites[$i][$key], $content);
$site = self::parseHeaderPage($content, $value);
$sites[$i] = array_merge($sites[$i], $site);
if (isset($header)) {
$header = substr($header, 0,
strpos($header, "\x0D\x0A\x0D\x0A") + 4);
} else {
$header = "";
}
$sites[$i][CrawlConstants::HEADER] =
$header . $sites[$i][CrawlConstants::HEADER];
unset($header);
} else if (isset($content) && !$minimal && $is_gopher) {
$sites[$i][CrawlConstants::HEADER] =
$header;
$sites[$i][$value] = $content;
unset($header);
} else {
$sites[$i][$value] = $content;
}
if (!$minimal) {
if ($is_internal) {
$sites[$i][self::SIZE] = strlen($content);
$sites[$i][self::DNS_TIME] = 0;
$sites[$i][self::TOTAL_TIME] = 0;
$sites[$i][self::HTTP_CODE] =
(empty($sites[$i][self::HTTP_CODE])) ? "404" :
$sites[$i][self::HTTP_CODE];
} else {
$sites[$i][self::SIZE] = @curl_getinfo($sites[$i][0],
CURLINFO_SIZE_DOWNLOAD);
$sites[$i][self::DNS_TIME] =
@curl_getinfo($sites[$i][0],
CURLINFO_NAMELOOKUP_TIME);
$sites[$i][self::TOTAL_TIME] =
@curl_getinfo($sites[$i][0],
CURLINFO_TOTAL_TIME);
$sites[$i][self::HTTP_CODE] =
curl_getinfo($sites[$i][0], CURLINFO_HTTP_CODE);
if (!empty($sites[$i]['ROOT_FOLLOW']) &&
curl_getinfo($sites[$i][0],
CURLINFO_REDIRECT_COUNT) > 0) {
$final_url = curl_getinfo($sites[$i][0],
CURLINFO_EFFECTIVE_URL);
$same_host = strcasecmp(parse_url($final_url,
PHP_URL_HOST) ?? "", parse_url(
$sites[$i]['ROOT_FOLLOW'], PHP_URL_HOST) ??
"") == 0;
$http_code = $sites[$i][self::HTTP_CODE];
if (!$same_host || $http_code < 200 ||
$http_code >= 300) {
/* To another host, or not to a page: kept as
the redirect it was, with the code of the
first response the header records. */
preg_match('/HTTP\/[0-9.]+\s+(\d{3})/',
$sites[$i][CrawlConstants::HEADER] ?? "",
$first_status);
$sites[$i][self::HTTP_CODE] = intval(
$first_status[1] ?? 302);
$sites[$i][CrawlConstants::LOCATION] =
[$final_url];
$sites[$i][$value] = " ";
}
}
if (!empty($sites[$i]['ROOT_FOLLOW']) &&
intval($sites[$i][self::HTTP_CODE]) >= 400) {
$root_errors[$i] = $sites[$i][$key];
}
unset($sites[$i]['ROOT_FOLLOW']);
if (!$sites[$i][self::HTTP_CODE] && !$is_gopher) {
$sites[$i][self::HTTP_CODE] =
curl_error($sites[$i][0]);
} else if ($is_gopher) {
$sites[$i][self::HTTP_CODE] = 200;
}
}
if (!empty($ip_addresses)) {
$sites[$i][self::IP_ADDRESSES] = $ip_addresses;
} else {
$sites[$i][self::IP_ADDRESSES] = ["0.0.0.0"];
}
//Get Time, Mime type and Character encoding
$sites[$i][self::TIMESTAMP] = time();
if ($is_gopher) {
$path = UrlParser::getPath($sites[$i][self::URL]);
$filename =
UrlParser::getDocumentFilename(
$sites[$i][self::URL]);
if (isset($path[1])) {
$gopher_type = $path[1];
} else {
$gopher_type = 1;
}
if ($gopher_type == 1) {
$sites[$i][self::TYPE] = "text/gopher";
} else if (in_array($gopher_type,
[0, 3, 6])) {
$sites[$i][self::TYPE] = "text/plain";
if ($gopher_type == 6) {
$sites[$i][$value] = convert_uudecode(
$content);
}
} else if ($gopher_type == 'h') {
$sites[$i][self::TYPE] = "text/html";
} else if ($gopher_type == 'g') {
$sites[$i][self::TYPE] = "image/gif";
}
$path_info = pathinfo($filename);
if (!isset($sites[$i][self::TYPE]) &&
isset($path_info['extension'])) {
$sites[$i][self::TYPE] =
UrlParser::guessMimeTypeFromFileName($filename);
} else if (!isset($sites[$i][self::TYPE])) {
$sites[$i][self::TYPE] = "unknown";
}
} else if (!$is_internal) {
$type_parts = explode(";", curl_getinfo($sites[$i][0],
CURLINFO_CONTENT_TYPE));
$sites[$i][self::TYPE] =
strtolower(trim($type_parts[0]));
} else {
$sites[$i][self::TYPE] = "unknown";
}
if ($public_only && !$is_internal && !$is_gopher &&
empty($proxied[$i])) {
$addresses = array_values(array_filter(array_unique(
array_merge($ip_addresses ?: [],
[curl_getinfo($sites[$i][0],
CURLINFO_PRIMARY_IP)]))));
if (self::leadsToThisMachine($sites[$i][$key],
$addresses)) {
self::refuseSite($sites[$i], $sites[$i][$key],
$value, $addresses,
$sites[$i][CrawlConstants::HEADER] ?? "");
unset($root_errors[$i]);
}
}
if (!empty($referer)) {
$sites[$i][self::URL_PARENT] = $referer;
}
}
/*
Ideally should have line for all requests
However, this seems to cause curl to sometimes crash
by trying to free stuff twice on some linux systems
at crawl time. Not having it on other systems causes crashes
at query time
*/
if ($sites[$i][0] != 'INTERNAL') {
if ($minimal || !stristr(PHP_OS, "LINUX")) {
curl_multi_remove_handle($agent_handler, $sites[$i][0]);
}
unset($sites[$i][0]);
}
if (!empty($sites[$i]['ROBOT'])) {
if (isset($sites[$i][self::TYPE]) &&
$sites[$i][self::TYPE] != "text/plain" &&
isset($sites[$i][CrawlConstants::LOCATION]) &&
count($site[CrawlConstants::LOCATION]) > 0) {
$sites[$i][self::TYPE] = "text/plain";
$sites[$i][self::HTTP_CODE] = "200";
$tmp = wordwrap($sites[$i][$value], 80);
$tmp_parts = explode("\n", $tmp);
$tmp = "# Suspect server misconfiguration\n";
$tmp .= "# Assume shouldn't crawl this site.\n";
$tmp .= "# Pretending got following robots.txt.\n";
$tmp .= "User-agent: *\n";
$tmp .= "Disallow: /\n";
$tmp .= "# Original error code: ".
$sites[$i][self::HTTP_CODE]."\n";
$tmp .= "# Original content:\n";
foreach ($tmp_parts as $part) {
$tmp = "#" . $part . "\n";
}
$sites[$i][$value] = $tmp;
$sites[$i][self::HTTP_CODE] = "200";
unset($site[CrawlConstants::LOCATION]);
}
}
} //end big if
} //end for
/* A site's root that answers with an error, as www.oracle.com did
with a 403 page, may still have a working subsite for the default
locale, such as /us-en. The root then keeps that subsite's page,
as it keeps the page a same-host redirect leads to. */
$locale_parts = explode("-", strtolower(C\p('DEFAULT_LOCALE')));
$language = $locale_parts[0];
$region = $locale_parts[1] ?? $language;
$subsite_paths = array_unique(["$region-$language",
"$language-$region", $language]);
$subsite_sites = [];
foreach ($root_errors as $i => $root_url) {
foreach ($subsite_paths as $path) {
$subsite_sites[] = [$key => rtrim($root_url, "/") . "/$path",
'ROOT_INDEX' => $i];
}
}
if (!empty($subsite_sites)) {
$subsite_sites = self::getPages($subsite_sites, false,
$page_range_request, $temp_dir, $key, $value, $minimal,
null, false, "", [], null, $public_only);
}
foreach ($subsite_sites as $subsite) {
$i = $subsite['ROOT_INDEX'];
$code = intval($subsite[self::HTTP_CODE] ?? 0);
if (!isset($root_errors[$i]) || $code < 200 || $code >= 300 ||
!empty($subsite[CrawlConstants::LOCATION])) {
continue;
}
foreach ($subsite as $field => $field_value) {
if (!in_array($field, [$key, 'ROOT_INDEX'], true)) {
$sites[$i][$field] = $field_value;
}
}
unset($root_errors[$i]);
}
if ($timer) {
crawlLog(" Get Page Content time ".
(changeInMicrotime($start_time)));
}
/* Freeing the handles here would wait for every address lookup
still running at the time limit, which can take many seconds.
The next call comes after the pages are processed, which gives
the lookups time to end. */
self::$last_handler = $agent_handler;
return $sites;
}
/**
* webBotAuthKeyPair gives the name server's Web Bot Auth key pair, an
* Ed25519 pair, and holds it in $web_bot_auth_key_pair. It is read from
* WEB_BOT_AUTH_KEY_FILE in SECURITY_DIR, where the server's other keys
* are, and made there the first time, under a lock so processes starting
* at once share one pair. Only the name server calls it: a fetcher is
* sent the pair by the name server, so all of a crawl's fetchers sign
* with the key the name server publishes.
*
* @return string the key pair as sodium_crypto_sign_keypair makes it
*/
public static function webBotAuthKeyPair()
{
if (self::$web_bot_auth_key_pair !== null) {
return self::$web_bot_auth_key_pair;
}
if (!is_dir(C\SECURITY_DIR)) {
mkdir(C\SECURITY_DIR);
}
$key_path = C\SECURITY_DIR . "/" . self::WEB_BOT_AUTH_KEY_FILE;
$key_file = fopen($key_path, "c+");
flock($key_file, LOCK_EX);
$key_pair = base64_decode(trim(stream_get_contents($key_file)), true);
if ($key_pair === false ||
strlen($key_pair) != SODIUM_CRYPTO_SIGN_KEYPAIRBYTES) {
$key_pair = sodium_crypto_sign_keypair();
ftruncate($key_file, 0);
rewind($key_file);
fwrite($key_file, base64_encode($key_pair));
fflush($key_file);
chmod($key_path, 0600);
giveFileToWebUser($key_path);
}
flock($key_file, LOCK_UN);
fclose($key_file);
self::$web_bot_auth_key_pair = $key_pair;
return $key_pair;
}
/**
* webBotAuthAgent gives the address Web Bot Auth names as the crawler's
* Signature-Agent: the scheme, host and any port of the name server,
* whose web server serves the key directory. A name server reached by
* plain http gives the empty string, which turns Web Bot Auth off, since
* Cloudflare reads the directory only over https.
*
* @return string the name server's https address, or the empty string
*/
public static function webBotAuthAgent()
{
$url_parts = @parse_url(C\p('NAME_SERVER'));
if (strtolower($url_parts['scheme'] ?? "") != "https" ||
empty($url_parts['host'])) {
return "";
}
return "https://" . strtolower($url_parts['host']) .
((empty($url_parts['port']) || $url_parts['port'] == 443) ? "" :
":" . $url_parts['port']);
}
/**
* webBotAuthKeyId gives the name Web Bot Auth uses for a public key:
* the base64url SHA-256 thumbprint of its JSON Web Key, as Request for
* Comments (RFC) 7638 defines it, made from the key's members in name
* order.
*
* @param string $public_x the public key, base64url encoded without
* padding, as a JSON Web Key's x member holds it
* @return string the thumbprint, base64url encoded without padding
*/
public static function webBotAuthKeyId($public_x)
{
return rtrim(strtr(base64_encode(hash("sha256",
'{"crv":"Ed25519","kty":"OKP","x":"' . $public_x . '"}', true)),
"+/", "-_"), "=");
}
/**
* webBotAuthHeaders gives the Signature-Agent, Signature-Input and
* Signature headers that sign a request to $url under Web Bot Auth, as
* Cloudflare's documentation lays them out. The signature covers the
* request's authority, its host and any port that is not the default,
* and the Signature-Agent header, as RFC 9421 builds a signature base.
* Only https requests are signed, since a signature sent in the clear
* could be copied and sent again until it expires. Nothing is signed
* while this process holds no key pair, as before a fetcher has been
* sent one, or while the agent is empty.
*
* @param string $url the address the request goes to
* @param string $agent the https address whose key directory holds
* the public key; null for webBotAuthAgent's, as a test may not
* @param int $created the time the signature is made, now if -1
* @param string $nonce a random value that makes the signature unique,
* made here if empty
* @return array header lines to send with the request, or none
*/
public static function webBotAuthHeaders($url, $agent = null,
$created = -1, $nonce = "")
{
$agent ??= self::webBotAuthAgent();
$key_pair = self::$web_bot_auth_key_pair;
$url_parts = @parse_url($url);
if ($agent === "" || $key_pair === null ||
strtolower($url_parts['scheme'] ?? "") != "https" ||
empty($url_parts['host']) ||
!function_exists("sodium_crypto_sign_detached")) {
return [];
}
$authority = strtolower($url_parts['host']) .
((empty($url_parts['port']) || $url_parts['port'] == 443) ? "" :
":" . $url_parts['port']);
$created = ($created < 0) ? time() : $created;
$nonce = ($nonce === "") ? base64_encode(random_bytes(64)) : $nonce;
$public_x = rtrim(strtr(base64_encode(sodium_crypto_sign_publickey(
$key_pair)), "+/", "-_"), "=");
$params = '("@authority" "signature-agent");created=' . $created .
';keyid="' . self::webBotAuthKeyId($public_x) . '";alg="ed25519";' .
'expires=' . ($created + self::WEB_BOT_AUTH_SECONDS) .
';nonce="' . $nonce . '";tag="web-bot-auth"';
$quoted_agent = '"' . $agent . '"';
$signature = base64_encode(sodium_crypto_sign_detached(
"\"@authority\": $authority\n" .
"\"signature-agent\": $quoted_agent\n" .
"\"@signature-params\": $params",
sodium_crypto_sign_secretkey($key_pair)));
return ["Signature-Agent: $quoted_agent",
"Signature-Input: sig1=$params", "Signature: sig1=:$signature:"];
}
/**
* webBotAuthDirectory gives the response for
* /.well-known/http-message-signatures-directory, where a site checking
* a Web Bot Auth signature finds this crawler's public key. The body is
* a JSON Web Key Set holding the key. The response is signed with the
* same key over the authority the request named, so the directory
* cannot be served by someone else as theirs.
*
* @param string $authority the host, and port if any, the request for
* the directory was sent to
* @return array [header lines, body]
*/
public static function webBotAuthDirectory($authority)
{
$key_pair = self::webBotAuthKeyPair();
$public_x = rtrim(strtr(base64_encode(sodium_crypto_sign_publickey(
$key_pair)), "+/", "-_"), "=");
$created = time();
$params = '("@authority";req);alg="ed25519";keyid="' .
self::webBotAuthKeyId($public_x) . '";nonce="' .
base64_encode(random_bytes(64)) .
'";tag="http-message-signatures-directory";created=' . $created .
';expires=' . ($created + self::WEB_BOT_AUTH_SECONDS);
$signature = base64_encode(sodium_crypto_sign_detached(
"\"@authority\";req: " . strtolower($authority) . "\n" .
"\"@signature-params\": $params",
sodium_crypto_sign_secretkey($key_pair)));
return [["Content-Type: " .
"application/http-message-signatures-directory+json",
"Signature: sig1=:$signature:", "Signature-Input: sig1=$params",
"Cache-Control: max-age=86400"],
json_encode(["keys" => [["kty" => "OKP", "crv" => "Ed25519",
"x" => $public_x]]])];
}
/**
* prepareUrlHeaders splits a url the queue server wrote as
* url###ip_address###referer###Etag into what a request needs: the url,
* the headers, among them one for the Etag, a host:port:ip string for
* curl's address cache, and the referer. An address that is missing is
* looked up in a cache kept in $temp_dir. A missing referer is the
* empty string, and a missing Etag adds no header.
*
* @param string $url site to download with ip address at end potentially
* after ###
* @param array $proxy_servers if not empty an array of proxy servers
* used to crawl through
* @param string $temp_dir folder to store temporary ip header info
* @param bool $expect_header whether to send the Expect: 100-continue
* header
* @return array 6-tuple (orig url, url with replacement, http header array,
* column separated string with host:port:ip -- used for curl hashing,
* referer, the ip address of that string or "" when there is none)
*/
public static function prepareUrlHeaders($url, $proxy_servers = [],
$temp_dir = "", $expect_header = true)
{
if (empty($temp_dir)) {
$temp_dir = C\TEMP_DIR;
if (!file_exists($temp_dir)) {
mkdir($temp_dir);
}
}
$dns_resolve = "";
$dns_resolve_port = 443;
$dns_resolve_host = "";
$ip_address = "";
$url = str_replace("&", "&", $url);
$is_gopher = false;
if (strncmp($url, "gopher", 6) == 0) {
$is_gopher = true;
}
/* Check if an ETag was added by the queue server. If found, create
If-None_Match header with the ETag and add it to the headers. Remove
ETag from URL
*/
$if_none_match = "If-None-Match";
$etag = null;
if (C\USE_ETAG_EXPIRES && stristr($url, "ETag:")) {
$etag_parts = preg_split("/ETag\:/i", $url);
$etag_data = explode(" ", $etag_parts[1]);
$etag = $etag_data[1];
$pos = strrpos($url, "ETag:");
$url = substr_replace($url, "", $pos, strlen("ETag: ".$etag));
}
/* in queue_server we added the ip (if available)
after the url followed by ###
*/
$headers = [];
$referer = "";
$url_ip_parts = explode("###", $url);
if ($proxy_servers != [] || (isset($url_ip_parts[0]) &&
(stripos($url_ip_parts[0],'.onion') !== false)) ) {
$url_ip_parts = [$url_ip_parts[0]];
$url = $url_ip_parts[0];
}
if (count($url_ip_parts) > 1) {
if (!empty($url_ip_parts[2])) {
$referer = substr($url_ip_parts[2], 0, 4096);
unset($url_ip_parts[2]);
}
$ip_address = ltrim(urldecode($url_ip_parts[1]), "#");
unset($url_ip_parts[1]);
$url_parts = @parse_url($url);
if (!empty($url_parts["host"])) {
$dns_resolve_port = 80;
if (!empty($url_parts["port"])) {
$dns_resolve_port = $url_parts["port"];
} else if ($url_parts["scheme"] == "https") {
$dns_resolve_port = 443;
}
$dns_resolve_host = $url_parts["host"];
}
if ($ip_address == '@') {
if (empty(self::$local_ip_cache) &&
file_exists("$temp_dir/" . self::local_ip_cache_file) ) {
self::$local_ip_cache = unserialize(file_get_contents(
"$temp_dir/" . self::local_ip_cache_file) );
}
if (!is_array(self::$local_ip_cache)) {
self::$local_ip_cache = [];
}
if (!empty(self::$local_ip_cache[$url_parts['host']])) {
$ip_address = self::$local_ip_cache[$url_parts['host']];
} else if (!empty($url_parts['host'])) {
$ip_address = gethostbyname($url_parts['host']);
self::$local_ip_cache['dirty'] = true;
self::$local_ip_cache[$url_parts['host']] = $ip_address;
}
}
if (!empty($dns_resolve_host) && !empty($dns_resolve_port) &&
!empty($ip_address)) {
$dns_resolve =
"$dns_resolve_host:$dns_resolve_port:$ip_address";
}
$len = strlen(inet_pton($ip_address));
if (count($url_ip_parts) > 1) {
/*
since did array_pop above, implode may not be same as
original $url
*/
$url = implode("###", $url_ip_parts);
} else {
$url = $url_ip_parts[0];
}
$url_with_ip_if_possible = $url;
} else {
$url_with_ip_if_possible = $url;
}
if ($expect_header) {
$headers[] = 'Expect: 100-continue';
}
if (C\USE_ETAG_EXPIRES && $etag !== null) {
$etag_header = $if_none_match . ": " . $etag;
$headers[] = $etag_header;
}
if ($is_gopher) {
$headers = "gopher";
}
$results = [$url, $url_with_ip_if_possible, $headers, $dns_resolve,
$referer, ($dns_resolve === "") ? "" : $ip_address];
return $results;
}
/**
* computePageHash makes a hash of a page's contents, so that pages
* with the same content are found as duplicates. The Fetcher calls it
* on a text page's description and on any other page's bytes, and the
* index groups the pages of a partition whose hashes match, keeping
* one. A text page whose description has no letters or digits, as a
* page drawn by scripts has, would hash the same as every other such
* page, and the one kept would take the others' info: entries. So
* when a url is given, such a page is hashed by its url instead, and
* is a duplicate only of itself.
*
* @param string $page the page's description, for a text page, or its
* bytes
* @param string $type for now either text or not-text. If text
* then some tags are removed before computing the hash
* @param string $url the page's url, hashed in place of a text page's
* description when that holds no letters or digits
* @return string 8 byte hash to identify page contents
*/
public static function computePageHash($page, $type = "not-text",
$url = "")
{
if ($type != "text") {
return crawlHash($page, true);
}
/* to do dedup we strip script, noscript, and style tags
as well as their content, then we strip tags, get rid
of whitespace and hash. Text that is not UTF-8 hashes as a page
with no text, as the patterns read as UTF-8 that once took out
these elements gave nothing back for it.
*/
$dedup_string = "";
if (mb_check_encoding($page ?? "", "UTF-8")) {
$dedup_string = $page;
foreach (["script", "noscript", "style"] as $tag_name) {
$dedup_string = removeTagSpans($dedup_string, $tag_name, "");
}
}
$dedup_string_old = preg_replace('/\W+/u', '', $dedup_string ?? "");
$dedup_string = strip_tags($dedup_string_old);
if (empty($dedup_string)) {
$dedup_string = $dedup_string_old;
}
$dedup_string = preg_replace('/\W+/u', '', $dedup_string);
if ($dedup_string === "" && $url !== "") {
return crawlHash("$url EMPTY", true);
}
return crawlHash($dedup_string, true);
}
/**
* setSiteCookies reads the lines of the crawl's Site Cookies option,
* each a domain, a colon and the cookies to send it, such as
* example.com: euconsent=granted; lang=en. The Fetcher calls it with
* the option's lines before each batch of downloads.
*
* @param array $lines the option's lines, as the crawl options saved
* them, with HTML entities in place of quotes and ampersands
*/
public static function setSiteCookies($lines)
{
self::$site_cookies = [];
foreach ($lines ?? [] as $line) {
$line = html_entity_decode($line, ENT_QUOTES, "UTF-8");
$colon = strpos($line, ":");
if ($colon === false) {
continue;
}
$domain = strtolower(trim(substr($line, 0, $colon)));
$cookies = trim(substr($line, $colon + 1));
if ($domain != "" && $cookies != "") {
self::$site_cookies[$domain] = $cookies;
}
}
}
/**
* cookieHeaderFor gives the text of the Cookie header to send with a
* request for $url: the Site Cookies option's cookies for the url's
* host or a domain it is under, then the cookies the host itself set
* earlier in the crawl. getPages calls it for each url it fetches.
*
* @param string $url the url about to be fetched
* @return string the header's text, or "" when there is nothing to send
*/
public static function cookieHeaderFor($url)
{
$host = strtolower(parse_url($url, PHP_URL_HOST) ?? "");
if ($host == "") {
return "";
}
$parts = [];
foreach (self::$site_cookies as $domain => $cookies) {
if ($host == $domain || str_ends_with($host, ".$domain")) {
$parts[] = $cookies;
}
}
foreach (self::$host_cookies[$host] ?? [] as $name => $value) {
$parts[] = "$name=$value";
}
return implode("; ", $parts);
}
/**
* keepCookies reads the Set-Cookie headers of a response, including
* those of any redirects before it, and keeps each cookie for the host
* of $url, so later requests to that host carry them. A cookie with
* Max-Age at or below zero, or an Expires date past, is removed. getPages
* calls it for each response it reads.
*
* @param string $url the url that was fetched
* @param string $header_and_page the response, headers first
*/
public static function keepCookies($url, $header_and_page)
{
$host = strtolower(parse_url($url, PHP_URL_HOST) ?? "");
$end = strpos($header_and_page, "\r\n\r\n");
$headers = ($end === false) ? $header_and_page :
substr($header_and_page, 0, $end);
if ($host == "" || stripos($headers, "set-cookie:") === false ||
!preg_match_all('/^Set-Cookie:\s*([^=;\s]+)=([^;\r\n]*)(.*)$/mi',
$headers, $cookies, PREG_SET_ORDER)) {
return;
}
$kept = self::$host_cookies[$host] ?? [];
unset(self::$host_cookies[$host]);
foreach ($cookies as [, $name, $value, $attributes]) {
$gone = (preg_match('/max-age\s*=\s*(-?\d+)/i', $attributes,
$age) && intval($age[1]) <= 0) || (preg_match(
'/expires\s*=\s*([^;]+)/i', $attributes, $expires) &&
strtotime($expires[1]) !== false &&
strtotime($expires[1]) < time());
if ($gone) {
unset($kept[$name]);
} else {
$kept[$name] = trim($value);
}
}
if (!empty($kept)) {
self::$host_cookies[$host] = $kept;
}
if (count(self::$host_cookies) > self::HOST_COOKIE_LIMIT) {
array_shift(self::$host_cookies);
}
}
/**
* parseHeaderPage splits a downloaded http response into its headers
* and its page, and reads from the headers the server, its version,
* the operating system, the encoding and the date.
*
* @param string $header_and_page string of downloaded data
* @param string $value field to store the page portion of page
* @return array info array consisting of a header, page for an http
* response, as well as parsed from the header the server, server
* version, operating system, encoding, and date information.
*/
public static function parseHeaderPage($header_and_page,
$value = CrawlConstants::PAGE)
{
$cache_page_validators = ['ETAG' => -1, 'EXPIRES' => -1];
$new_offset = 0;
// header will include all redirect headers
$site = [];
$site[CrawlConstants::LOCATION] = [];
do {
$continue = false;
do {
$CRLFCRLF = strpos($header_and_page, "\x0D\x0A\x0D\x0A",
$new_offset);
$LFLF = strpos($header_and_page, "\x0A\x0A", $new_offset);
//either two CRLF (what spec says) or two LF's to be safe
$old_offset = $new_offset;
$header_offset = ($CRLFCRLF > 0) ? $CRLFCRLF : $LFLF;
$header_offset = ($header_offset) ? $header_offset : 0;
$new_offset = ($CRLFCRLF > 0) ? $header_offset + 4
: $header_offset + 2;
$new_offset = min($new_offset, max(
strlen($header_and_page) - 1, 0));
$test_pos = strpos($header_and_page, "\n", $new_offset);
$http_check = ltrim(strtoupper(
substr($header_and_page, $new_offset, $test_pos -
$new_offset)));
$http_check = substr($http_check, 0, 4);
//Sometimes in HTTP/2 have a HTTP/2 100 followed HTTP/2 200
} while ($http_check == "HTTP") ;
$redirect_pos = stripos($header_and_page, 'Location:', $old_offset);
$redirect_str = "Location:";
if ($redirect_pos === false) {
$redirect_pos =
stripos($header_and_page, 'Refresh:', $old_offset);
$redirect_str = "Refresh:";
}
if (isset($header_and_page[$redirect_pos - 1]) &&
ord($header_and_page[$redirect_pos - 1]) > 32) {
$redirect_pos = $new_offset; //ignore X-XRDS-Location header
} else if ($redirect_pos !== false && $redirect_pos < $new_offset){
$redirect_pos += strlen($redirect_str);
$pre_line = substr($header_and_page, $redirect_pos,
strpos($header_and_page, "\n", $redirect_pos) -
$redirect_pos);
$loc = @trim($pre_line);
if (strlen($loc) > 0) {
$site[CrawlConstants::LOCATION][] = @$loc;
}
$continue = true;
}
} while($continue);
if ($header_offset > 0) {
$site[CrawlConstants::HEADER] =
substr($header_and_page, 0, $header_offset);
$site[$value] = substr($header_and_page, $new_offset);
} else { //header message no body; maybe 301?
$site[CrawlConstants::HEADER] = $header_and_page;
$site[$value] = " ";
}
$lines = explode("\n", $site[CrawlConstants::HEADER]);
$first_line = array_shift($lines);
$response = preg_split("/(\s+)/", $first_line);
$site[CrawlConstants::HTTP_CODE] = isset($response[1]) ?
@trim($response[1]) : 0;
$site[CrawlConstants::ROBOT_METAS] = [];
foreach ($lines as $line) {
$line = trim($line);
if (stristr($line, 'Server:')) {
$server_parts = preg_split("/Server\:/i", $line);
$server_name_parts = @explode("/", $server_parts[1]);
$site[CrawlConstants::SERVER] = @trim($server_name_parts[0]);
if (isset($server_name_parts[1])) {
$version_parts = explode("(", $server_name_parts[1]);
$site[CrawlConstants::SERVER_VERSION] =
@trim($version_parts[0]);
if (isset($version_parts[1])) {
$os_parts = explode(")", $version_parts[1]);
$site[CrawlConstants::OPERATING_SYSTEM] =
@trim($os_parts[0]);
}
}
}
if (stristr($line, 'Content-type:')) {
list(,$mimetype,) = preg_split("/:|;/i", $line);
$site[CrawlConstants::TYPE] = trim($mimetype);
}
if (stristr($line, 'charset=')) {
$line_parts = preg_split("/charset\=/i", $line);
$site[CrawlConstants::ENCODING] =
strtoupper(@trim($line_parts[1]));
}
if (stristr($line, 'Last-Modified:')) {
$line_parts = preg_split("/Last\-Modified\:/i", $line);
$site[CrawlConstants::MODIFIED] =
strtotime(@trim($line_parts[1]));
}
if (stristr($line, 'X-Robots-Tag:')) { // robot directives pdfs etc
$line_parts = preg_split("/X\-Robots\-Tag\:/i", $line);
$robot_metas = explode(",", $line_parts[1]);
foreach ($robot_metas as $robot_meta) {
$site[CrawlConstants::ROBOT_METAS][] = strtoupper(
trim($robot_meta));
}
}
if (stristr($line, 'Content-Range:')) {
$line_parts = explode("/", $line);
if (!empty($line_parts[1])) {
$content_size = intval(trim($line_parts[1]));
if ($content_size > 0) {
$site[CrawlConstants::CONTENT_SIZE] = $content_size;
}
}
}
$canonical_regex = "/Link\:\s*\<\s*(http.*)\s*\>\s*\;\s*".
"rel\s*\=\s*(\"|')?canonical(\"|')?/";
// levenshtein gives notices on strings longer than 255
if (preg_match($canonical_regex, $line, $matches) &&
isset($site[CrawlConstants::URL]) && strlen($matches[1]) < 252
&& (strlen($site[CrawlConstants::URL]) >= 255 ||
levenshtein($matches[1], $site[CrawlConstants::URL]) > 3)) {
// for rel canonical headers
$site[CrawlConstants::LOCATION][] = $matches[1];
$site[CrawlConstants::ROBOT_METAS][] = 'NOFOLLOW';
}
if (C\USE_ETAG_EXPIRES && stristr($line, 'ETag:')) {
$line_parts = preg_split("/ETag\:/i", $line);
if (isset($line_parts[1])) {
$etag_data = explode(" ", $line_parts[1]);
if (isset($etag_data[1])) {
$etag = $etag_data[1];
$cache_page_validators['ETAG'] = substr($etag, 0, 255);
/* we are only storing first 255 chars of etag so
can use of BPlus trees TEXT field */
}
}
}
if (C\USE_ETAG_EXPIRES && stristr($line, 'Expires:')) {
$line_parts = preg_split("/Expires\:/i", $line);
$all_dates = $line_parts[1];
$date_parts = explode(",", $all_dates);
if (count($date_parts) == 2) {
$cache_page_validators['EXPIRES'] = strtotime(
$date_parts[1]);
} else if (count($date_parts) > 2) {
/*Encountered some pages with more than one Expires date
:O */
$timestamps = [];
for ($i = 1;$i < count($date_parts); $i += 2) {
$ds = strtotime($date_parts[$i]);
$timestamps[] = $ds;
}
$lowest = min($timestamps);
$cache_page_validators['EXPIRES'] = $lowest;
}
}
if (C\USE_ETAG_EXPIRES && !($cache_page_validators['ETAG'] == -1 &&
$cache_page_validators['EXPIRES'] == -1)) {
$site[CrawlConstants::CACHE_PAGE_VALIDATORS] =
$cache_page_validators;
}
}
/*
If the doc is HTML and it uses a http-equiv to set the encoding
then we override what the server says (if anything). As we
are going to convert to UTF-8 we remove the charset info
from the meta tag so cached pages will display correctly and
redirects without char encoding won't be given a different hash.
*/
if (!empty($site[CrawlConstants::TYPE]) && in_array($site[
CrawlConstants::TYPE], ['text/html',
'application/xhtml+xml', 'application/xml',
'text/xml'] ) ) {
$encoding_info = guessEncodingHtmlXml($site[$value], true);
if (is_array($encoding_info)) {
list($site[CrawlConstants::ENCODING], $start_charset, $len_c) =
$encoding_info;
$site[$value] = substr_replace($site[$value], "",
$start_charset, $len_c);
} else {
$site[CrawlConstants::ENCODING] = $encoding_info;
}
}
if (!isset($site[CrawlConstants::SERVER]) ) {
$site[CrawlConstants::SERVER] = "unknown";
}
return $site;
}
/**
* getPage downloads one url with curl and gives back its contents.
*
* @param string $site url of page to request
* @param array $post_data any data to be POST'd to the URL
* @param bool $check_for_errors whether or not to check the response
* for the words, NOTICE, WARNING, FATAL which might indicate an
* error on the server
* @param string $user_password username:password to use for connection if
* needed (optional)
* @param int $timeout how long to wait for page download to complete
* @param array $headers additional HTTP headers to ssend
* @return string the contents of what the curl request fetched
*/
public static function getPage($site, $post_data = null,
$check_for_errors = false, $user_password = null,
$timeout = C\SINGLE_PAGE_TIMEOUT, $headers = [])
{
static $agents = [];
$memory_limit = memoryCeiling(C\MEMORY_FILL_FACTOR);
if (memory_get_usage() > $memory_limit) {
$agents = [];
}
$MAX_AGENTS = 50;
$not_web_setting = (php_sapi_name() == 'cli' &&
!C\nsdefined("IS_OWN_WEB_SERVER"));
/* A server bound to every address calls itself 0.0.0.0 or ::,
and a name server set up under secure-start was given that
host. Linux and macOS connect to 0.0.0.0 as to this machine;
Windows refuses it, so such a host is sent to this machine's
loopback address everywhere. */
$site = preg_replace(['@^(\w+://)0\.0\.0\.0(?=[:/?#]|$)@',
'@^(\w+://)\[::\](?=[:/?#]|$)@'], ['${1}127.0.0.1',
'${1}[::1]'], (string)$site);
$url_parts = @parse_url($site);
$host = (empty($url_parts["host"])) ? false : $url_parts["host"];
$scheme = (empty($url_parts["scheme"])) ? false : $url_parts["scheme"];
if ($host !== false) {
if (self::isSelfUrl($site)) {
if (empty($GLOBALS['web_site'])) {
return "";
}
$web_site = $GLOBALS['web_site'];
$output = $web_site->processInternalRequest($site, false,
$post_data);
return $output;
}
if (count($agents) > $MAX_AGENTS) {
$agent_host = array_shift($agents);
if ($agent_host) {
unset($agent_host);
}
}
$host = ($post_data) ? "POST " . $host : "GET " . $host;
if (empty($agents[$host])) {
$agents[$host] = curl_init();
}
}
if ($not_web_setting) {
crawlLog(" Init curl request of a single page");
}
if (in_array(UrlParser::getHost($site),
C\USER_AGENT_ALTERNATIVE_HOSTS)) {
curl_setopt($agents[$host], CURLOPT_USERAGENT,
C\USER_AGENT_ALTERNATIVE);
} else {
curl_setopt($agents[$host], CURLOPT_USERAGENT, C\USER_AGENT);
}
curl_setopt($agents[$host], CURLOPT_URL, $site);
curl_setopt($agents[$host], CURLOPT_AUTOREFERER, true);
curl_setopt($agents[$host], CURLOPT_FOLLOWLOCATION, true);
/* curl http2 can cause problems when compiled against an older version
of openssl. This seems to mainly affect posting data, so for
now we use HTTP/1.1 to post.
*/
if ($post_data == null && $scheme == "https" &&
defined("CURL_HTTP_VERSION_2_0")) {
curl_setopt($agents[$host], CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_2_0);
}
// these next two lines should probably be modified for better security
curl_setopt($agents[$host], CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($agents[$host], CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($agents[$host], CURLOPT_NOSIGNAL, true);
curl_setopt($agents[$host], CURLOPT_RETURNTRANSFER, true);
curl_setopt($agents[$host], CURLOPT_FAILONERROR, true);
curl_setopt($agents[$host], CURLOPT_TIMEOUT, $timeout);
curl_setopt($agents[$host], CURLOPT_CONNECTTIMEOUT, $timeout);
if (!empty($headers)) {
curl_setopt($agents[$host], CURLOPT_HTTPHEADER, $headers);
}
if ($post_data != null) {
curl_setopt($agents[$host], CURLOPT_POST, true);
curl_setopt($agents[$host], CURLOPT_POSTFIELDS, $post_data);
} else {
/* Agents are cached, so a get request has to say so, or an
agent last used for a post stays a post and drops the
query string. */
curl_setopt($agents[$host], CURLOPT_HTTPGET, true);
}
if ($user_password != null) {
curl_setopt($agents[$host], CURLOPT_FAILONERROR, false);
curl_setopt($agents[$host], CURLOPT_USERPWD, $user_password);
curl_setopt($agents[$host], CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($agents[$host], CURLOPT_SSLVERSION,
CURL_SSLVERSION_TLSv1_2);
}
if ($not_web_setting) {
crawlLog(" Set curl options for single page request");
}
$time = time();
$response = curl_exec($agents[$host]);
if (!$response && $not_web_setting) {
crawlLog(" REQUEST FAILED!! Outputting curl_getinfo...");
crawlLog(print_r(curl_getinfo($agents[$host]), true));
}
if (time() - $time > C\PAGE_TIMEOUT && $not_web_setting) {
crawlLog(" Request took longer than page timeout!!");
crawlLog(" Either could not reach URL or website took too");
crawlLog(" long to respond.");
}
curl_setopt($agents[$host], CURLOPT_POSTFIELDS, "");
if ($not_web_setting) {
crawlLog(" Done curl exec");
}
if ($not_web_setting && $check_for_errors) {
self::checkResponseForErrors($response);
}
return $response;
}
/**
* checkResponseForErrors writes a getPage response to the crawl log when
* it holds the word NOTICE, WARNING or FATAL, which can mean an error on
* the server that sent it.
*
* @param string $response getPage response in which to check for errors
*/
public static function checkResponseForErrors($response)
{
if (preg_match("/\bNOTICE|WARNING|FATAL\b/i", $response)) {
crawlLog("There appears to have been an error in the server ".
"response. Response was:");
crawlLog(wordwrap($response));
}
}
}
- Yioop Repo - Seekquarry