<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\UrlParser;
/**
* For crawlHash
*/
require_once __DIR__ . "/Utility.php";
/**
* So know which part of speech tagger to use
*/
require_once __DIR__ . "/LocaleFunctions.php";
/**
*
* Code used to manage HTTP or Gopher requests from one or more URLS
*
* @author Chris Pollett
*/
class FetchUrl implements CrawlConstants
{
/**
* a small cache of dns - ip addresses lookups for machines that are
* part of this yioop cluster
*
* @var array
*/
static $local_ip_cache = [];
/**
* Computes the list of host names and addresses that refer to the
* Yioop instance answering the current request. getPage, getPages,
* and isSelfUrl use this to decide whether a requested url points
* back at this instance itself, in which case the request is made
* through WebSite::processInternalRequest rather than over the
* network, as a network loopback to one's own url would deadlock a
* single-process web server. The list contains the usual loopback
* names, the current SERVER_NAME, the host of C\baseUrl() (this
* instance's configured url), together with the www-prefixed and
* bare variant of each domain, and the IP address the current
* request arrived on, so a machine configured by its bare IP still
* matches. C\p('NAME_SERVER')'s host is added only when this instance is
* the name server (the request arrived for that host); on a fetcher,
* where C\p('NAME_SERVER') names a remote host, it is left out so those
* requests still go over the network.
*
* @return array host names and addresses that refer to this instance
*/
public static function localHosts()
{
$local_hosts = ['localhost', '::1', '0.0.0.0', '127.0.0.1'];
$candidates = [];
if (!empty($_SERVER['SERVER_NAME'])) {
$candidates[] = $_SERVER['SERVER_NAME'];
}
$base_host = parse_url(C\baseUrl(), PHP_URL_HOST);
if (!empty($base_host)) {
$candidates[] = $base_host;
}
/*
When this instance is the name server, the name server's own
host names itself rather than a remote machine, so add it
too. We only do this when the request arrived for that host
(SERVER_NAME) or on the IP the name server resolves to, so a
fetcher, where NAME_SERVER is a remote host, is unaffected.
*/
if (C\nsdefined("NAME_SERVER")) {
$name_server_host = parse_url(C\p('NAME_SERVER'), PHP_URL_HOST);
if (!empty($name_server_host) &&
(($_SERVER['SERVER_NAME'] ?? "") == $name_server_host ||
in_array($name_server_host, $candidates))) {
$candidates[] = $name_server_host;
}
}
foreach ($candidates as $candidate) {
$local_hosts[] = $candidate;
if (str_starts_with($candidate, "www.")) {
$local_hosts[] = substr($candidate, strlen("www."));
} else {
$local_hosts[] = "www." . $candidate;
}
}
/*
The IP address the current request actually arrived on is
unambiguously this machine, so a machine configured by its IP
(rather than a host name) is still recognized as ourselves and
answered in process instead of over a network loopback.
*/
if (!empty($_SERVER['SERVER_ADDR'])) {
$local_hosts[] = $_SERVER['SERVER_ADDR'];
}
return array_unique($local_hosts);
}
/**
* Decides whether a url points back at this Yioop instance, so the
* caller can answer it in process with
* WebSite::processInternalRequest instead of sending a network
* request to oneself, which would deadlock a single-process web
* server. A url is ours when this process is its own web server, the
* url's host is one of localHosts(), and the url's port is the one
* the current request came in on.
*
* @param string $url the url to test
* @return bool true if the url refers to this instance
*/
public static function isSelfUrl($url)
{
if (!C\nsdefined("IS_OWN_WEB_SERVER")) {
return false;
}
$url_parts = @parse_url($url ?? "");
$host = empty($url_parts["host"]) ? false : $url_parts["host"];
if ($host === false || !in_array($host, self::localHosts())) {
return false;
}
$port = empty($url_parts["port"]) ? ($_SERVER['SERVER_PORT'] ?? "") :
$url_parts["port"];
return ($port == ($_SERVER['SERVER_PORT'] ?? ""));
}
/**
* Make multi_curl requests for an array of sites with urls or onion urls
*
* @param array $sites an array containing urls of pages to request
* @param bool $timer flag, true means print timing statistics to log
* @param int $page_range_request maximum number of bytes to download/page
* 0 means download all
* @param string $temp_dir folder to store temporary ip header info
* @param string $key the component of $sites[$i] that has the value of
* a url to get defaults to URL
* @param string $value component of $sites[$i] in which to store the
* page that was gotten
* @param bool $minimal if true do a faster request of pages by not
* doing things like extract HTTP headers sent, etcs. Also will
* not use Expect header
* @param array $post_data data to be POST'd to each site
* @param bool $follow whether to follow redirects or not
* @param string $tor_proxy url of a proxy that knows how to download
* .onion urls
* @param array $proxy_servers if not [], then an array of proxy
* server to use rather than to directly download web pages from
* the current machine
*
* @return array an updated array with the contents of those pages
*/
public static function getPages($sites, $timer = false,
$page_range_request = C\PAGE_RANGE_REQUEST, $temp_dir = "",
$key = CrawlConstants::URL, $value = CrawlConstants::PAGE,
$minimal = false, $post_data = null, $follow = false, $tor_proxy = "",
$proxy_servers = [])
{
/* should think of curl_multi_init as returning thread handler
curl_init returns a handle to an open connection and we can
then execute operations on this connection concurrent with other
handles by adding it to the group of handles managed by
the curl_multi_init thread handler. Removing a handle from
the curl_multi_init thread handler does not close the connection
it just means we can't do concurrent operations with it.
*/
$agent_handler = curl_multi_init();
$active = null;
$start_time = microtime(true);
if (empty($temp_dir)) {
$temp_dir = C\TEMP_DIR . "/";
if (!file_exists($temp_dir)) {
mkdir($temp_dir);
}
}
$memory_limit = memoryCeiling(C\MEMORY_FILL_FACTOR);
$memory_limit_high = memoryCeiling(C\MEMORY_FILL_FACTOR_HIGH);
if (memory_get_usage() > $memory_limit) {
self::$local_ip_cache = [];
}
//Set-up requests
$num_sites = count($sites);
for ($i = 0; $i < $num_sites; $i++) {
$is_gopher = false;
$sites[$i][CrawlConstants::IS_GOPHER_URL] = $is_gopher;
if (!isset($sites[$i][$key])) {
continue;
}
$url_parts = @parse_url($sites[$i][$key]);
$host = (empty($url_parts["host"])) ? false :
$url_parts["host"];
$scheme = (empty($url_parts["scheme"])) ? false :
$url_parts["scheme"];
$is_local_host = ($host !== false &&
in_array($host, self::localHosts()));
if (self::isSelfUrl($sites[$i][$key])) {
$sites[$i][0] = "INTERNAL";
if (empty($GLOBALS['web_site'])) {
$sites[$i][$value] = "";
continue;
}
$web_site = $GLOBALS['web_site'];
$post_info = $post_data[$i] ?? null;
$sites[$i][$value] = $web_site->processInternalRequest(
$sites[$i][$key], !$minimal, $post_info);
continue;
}
list($sites[$i][$key], $url, $headers, $dns_resolve, $referer) =
self::prepareUrlHeaders($sites[$i][$key], $proxy_servers,
$temp_dir, !$minimal && !$follow && $post_data);
if (empty($url)) {
continue;
}
if ($headers == "gopher") {
$is_gopher = true;
$sites[$i][CrawlConstants::IS_GOPHER_URL] = $is_gopher;
$headers = [];
}
$sites[$i][0] = curl_init();
if (!$minimal) {
$ip_holder[$i] = fopen("$temp_dir/tmp$i.txt", 'w+');
curl_setopt($sites[$i][0], CURLOPT_STDERR, $ip_holder[$i]);
curl_setopt($sites[$i][0], CURLOPT_VERBOSE, true);
}
if (in_array(UrlParser::getHost($url),
C\USER_AGENT_ALTERNATIVE_HOSTS)) {
curl_setopt($sites[$i][0], CURLOPT_USERAGENT,
C\USER_AGENT_ALTERNATIVE);
} else {
curl_setopt($sites[$i][0], CURLOPT_USERAGENT, C\USER_AGENT);
}
curl_setopt($sites[$i][0], CURLOPT_IPRESOLVE,
CURL_IPRESOLVE_WHATEVER);
if (!empty($dns_resolve)) {
curl_setopt($sites[$i][0], CURLOPT_RESOLVE, [$dns_resolve]);
}
curl_setopt($sites[$i][0], CURLOPT_URL, $url);
curl_setopt($sites[$i][0], CURLOPT_REFERER, $referer);
if (strcmp(substr($url, -10), "robots.txt") == 0 ) {
$sites[$i]['ROBOT'] = true;
$follow = true; /*wikipedia redirects their robot page. grr
want to force this for robots pages
*/
}
curl_setopt($sites[$i][0], CURLOPT_FOLLOWLOCATION, $follow);
if ($scheme == "https" && defined("CURL_HTTP_VERSION_2_0")) {
curl_setopt($sites[$i][0], CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_2_0);
}
curl_setopt($sites[$i][0], CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($sites[$i][0], CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($sites[$i][0], CURLOPT_AUTOREFERER, true);
curl_setopt($sites[$i][0], CURLOPT_RETURNTRANSFER, true);
/*
A url whose host is one of ours but that was not answered
in process above (for example its port did not match the
listening port, or the web server object was not
available) is a loopback to ourselves. A single-process
server cannot answer it while busy with this request, so
fetch it with a short timeout rather than the full page
timeout, turning a would-be multi-second freeze of every
connection into a quick failure.
*/
$request_timeout = $is_local_host ? C\LOOPBACK_PAGE_TIMEOUT :
C\PAGE_TIMEOUT;
curl_setopt($sites[$i][0], CURLOPT_CONNECTTIMEOUT,
$request_timeout);
curl_setopt($sites[$i][0], CURLOPT_TIMEOUT, $request_timeout);
if (stripos($url,'.onion') !== false && $tor_proxy != "") {
curl_setopt($sites[$i][0], CURLOPT_PROXY, $tor_proxy);
curl_setopt($sites[$i][0], CURLOPT_PROXYTYPE,
CURLPROXY_SOCKS5_HOSTNAME);
if ($timer) {
crawlLog("Using Tor proxy for $url..");
}
} else if ($proxy_servers != [] && !$is_gopher) {
$select_proxy = rand(0, count($proxy_servers) - 1);
$proxy_server = $proxy_servers[$select_proxy];
$proxy_parts = explode(":", $proxy_server);
$proxy_ip = $proxy_parts[0];
if (!empty($proxy_parts[1])) {
$proxy_port = $proxy_parts[1];
} else {
$proxy_port = "80";
}
$proxy_type = CURLPROXY_HTTP;
if (!empty($proxy_parts[2])) {
$pre_proxy_type = strtolower($proxy_parts[2]);
if ($pre_proxy_type != 'http') {
if ($pre_proxy_type == 'socks4') {
$proxy_type = CURLPROXY_SOCKS4;
} else if ($pre_proxy_type == 'socks4a') {
$proxy_type = CURLPROXY_SOCKS4A;
} else if ($pre_proxy_type == 'socks5') {
$proxy_type = CURLPROXY_SOCKS5;
} else if ($pre_proxy_type == 'socks5_hostname') {
$proxy_type = CURLPROXY_SOCKS5_HOSTNAME;
} else {
$proxy_type = $proxy_parts[2];
}
}
}
curl_setopt($sites[$i][0], CURLOPT_PROXY,
"$proxy_ip:$proxy_port");
curl_setopt($sites[$i][0], CURLOPT_PROXYTYPE,
$proxy_type);
if (!empty($proxy_parts[4])) {
// $proxy_parts[3] = username; $proxy_parts[4] = password
curl_setopt($sites[$i][0], CURLOPT_PROXYUSERPWD,
$proxy_parts[3] . ":" . $proxy_parts[4]);
curl_setopt($sites[$i][0], CURLOPT_PROXYAUTH,
CURLAUTH_BASIC);
}
if ($timer) {
crawlLog("Selecting proxy $select_proxy for $url");
}
}
if (!$minimal) {
curl_setopt($sites[$i][0], CURLOPT_HEADER, true);
}
//make lighttpd happier
if (!$is_gopher) {
if (defined('CURLOPT_HEADEROPT')) {
curl_setopt($sites[$i][0], CURLOPT_HEADEROPT,
CURLHEADER_UNIFIED);
}
curl_setopt($sites[$i][0], CURLOPT_HTTPHEADER,
$headers);
}
curl_setopt($sites[$i][0], CURLOPT_ENCODING, "");
// ^ need to set for sites like at that use gzip
if ($page_range_request > 0 && empty(
$sites[$i][CrawlConstants::NO_RANGE])) {
curl_setopt($sites[$i][0], CURLOPT_RANGE, "0-".
$page_range_request);
} else if (!empty( $sites[$i][CrawlConstants::NO_RANGE])) {
crawlLog("No range used for $url");
}
if ($post_data != null) {
curl_setopt($sites[$i][0], CURLOPT_POST, true);
curl_setopt($sites[$i][0], CURLOPT_POSTFIELDS,
$post_data[$i]);
}
curl_multi_add_handle($agent_handler, $sites[$i][0]);
}
if (!empty(self::$local_ip_cache['dirty'])) {
unset(self::$local_ip_cache['dirty']);
file_put_contents("$temp_dir/" . self::local_ip_cache_file,
serialize(self::$local_ip_cache));
}
if ($timer) {
crawlLog(" Init Get Pages " . (changeInMicrotime($start_time)));
}
$start_time = microtime(true);
$start = time();
//Wait for responses
$running = null;
$mrc_check = CURLM_CALL_MULTI_PERFORM;
$curlm_responses = [
CURLM_CALL_MULTI_PERFORM => "CURLM_CALL_MULTI_PERFORM",
CURLM_OK => "CURLM_OK",
CURLM_BAD_HANDLE => "CURLM_BAD_HANDLE",
CURLM_BAD_EASY_HANDLE => "CURLM_BAD_EASY_HANDLE",
CURLM_OUT_OF_MEMORY => "CURLM_OUT_OF_MEMORY",
CURLM_INTERNAL_ERROR => "CURLM_INTERNAL_ERROR",
CURLMSG_DONE => "CURLMSG_DONE"
];
set_error_handler(null);
do {
$mrc = @curl_multi_exec($agent_handler, $running);
if ($running && $mrc == CURLM_OK) {
$mrc_check = CURLM_OK;
if (\Fiber::getCurrent() !== null) {
/* Running inside the cooperative web server: hand
the loop back so it can serve other connections
while these transfers continue, rather than
freezing the whole single-threaded server on this
one fetch. The error handler is restored around
the suspend so work that runs in our place sees
the normal handler, not the null one this loop
installs. */
restore_error_handler();
\Fiber::suspend();
set_error_handler(null);
} else if (curl_multi_select($agent_handler, 0.05) === -1) {
/* curl_multi_select returns -1 when libcurl has no
socket to wait on yet, for example while a name is
still resolving; it returns at once in that case,
so without this brief sleep the loop would spin at
full CPU. When it does have sockets the 0.05s wait
keeps the loop from being too busy. */
usleep(50000);
}
}
} while (memory_get_usage() < $memory_limit_high &&
$mrc == $mrc_check && time() - $start < C\PAGE_TIMEOUT &&
$running > 0);
restore_error_handler();
if (time() - $start > C\PAGE_TIMEOUT && $timer) {
crawlLog(" TIMED OUT!!!");
}
if (memory_get_usage() > $memory_limit_high) {
crawlLog(" MEMORY EXHAUSTED!!!");
}
$mrc_message = (isset($curlm_responses[$mrc])) ? $curlm_responses[$mrc]
: $mrc;
if (php_sapi_name() == 'cli') {
crawlLog(" Final curl_multi_exec response $mrc: $mrc_message.");
crawlLog(" Final curl_multi running value $running.");
crawlLog(" Final curl_multi memory usage " . memory_get_usage() .
". Max allowed: $memory_limit");
}
if ($timer) {
crawlLog(" Page Request time ".(changeInMicrotime($start_time)));
}
$start_time = microtime(true);
//Process returned pages
for ($i = 0; $i < $num_sites; $i++) {
if ($timer && php_sapi_name() == 'cli') {
crawlTimeoutLog("FetchUrl initial processing of page %s of %s",
$i, $num_sites);
}
if (!$minimal && isset($ip_holder[$i]) ) {
rewind($ip_holder[$i]);
$header = fread($ip_holder[$i], 8192);
$ip_addresses = self::getCurlIp($header);
}
$is_gopher = false;
$is_internal = false;
if (!empty($sites[$i][0])) {
// Get Data and Message Code
if ($sites[$i][0] == 'INTERNAL') {
$is_internal = true;
$content = $sites[$i][$value];
} else {
$content = @curl_multi_getcontent($sites[$i][0]);
}
$is_gopher = $sites[$i][CrawlConstants::IS_GOPHER_URL];
/*
If the Transfer-encoding was chunked then the Range header
we sent was ignored. So we manually truncate the data
here
*/
if ($page_range_request > 0) {
$init_len = strlen($content);
$content = substr($content, 0, $page_range_request);
if (strlen($content) != $init_len) {
$sites[$i][CrawlConstants::CONTENT_SIZE] = $init_len;
}
}
if (empty($content) && !empty($header)) {
crawlLog(
"..Request {$sites[$i][$key]} returned no content");
crawlLog("..response header was " . $header);
}
if (isset($content) && !$minimal && !$is_gopher) {
$site = self::parseHeaderPage($content, $value);
$sites[$i] = array_merge($sites[$i], $site);
if (isset($header)) {
$header = substr($header, 0,
strpos($header, "\x0D\x0A\x0D\x0A") + 4);
} else {
$header = "";
}
$sites[$i][CrawlConstants::HEADER] =
$header . $sites[$i][CrawlConstants::HEADER];
unset($header);
} else if (isset($content) && !$minimal && $is_gopher) {
$sites[$i][CrawlConstants::HEADER] =
$header;
$sites[$i][$value] = $content;
unset($header);
} else {
$sites[$i][$value] = $content;
}
if (!$minimal) {
if ($is_internal) {
$sites[$i][self::SIZE] = strlen($content);
$sites[$i][self::DNS_TIME] = 0;
$sites[$i][self::TOTAL_TIME] = 0;
$sites[$i][self::HTTP_CODE] =
(empty($sites[$i][self::HTTP_CODE])) ? "404" :
$sites[$i][self::HTTP_CODE];
} else {
$sites[$i][self::SIZE] = @curl_getinfo($sites[$i][0],
CURLINFO_SIZE_DOWNLOAD);
$sites[$i][self::DNS_TIME] =
@curl_getinfo($sites[$i][0],
CURLINFO_NAMELOOKUP_TIME);
$sites[$i][self::TOTAL_TIME] =
@curl_getinfo($sites[$i][0],
CURLINFO_TOTAL_TIME);
$sites[$i][self::HTTP_CODE] =
curl_getinfo($sites[$i][0], CURLINFO_HTTP_CODE);
if (!$sites[$i][self::HTTP_CODE] && !$is_gopher) {
$sites[$i][self::HTTP_CODE] =
curl_error($sites[$i][0]);
} else if ($is_gopher) {
$sites[$i][self::HTTP_CODE] = 200;
}
}
if (!empty($ip_addresses)) {
$sites[$i][self::IP_ADDRESSES] = $ip_addresses;
} else {
$sites[$i][self::IP_ADDRESSES] = ["0.0.0.0"];
}
//Get Time, Mime type and Character encoding
$sites[$i][self::TIMESTAMP] = time();
if ($is_gopher) {
$path = UrlParser::getPath($sites[$i][self::URL]);
$filename =
UrlParser::getDocumentFilename(
$sites[$i][self::URL]);
if (isset($path[1])) {
$gopher_type = $path[1];
} else {
$gopher_type = 1;
}
if ($gopher_type == 1) {
$sites[$i][self::TYPE] = "text/gopher";
} else if (in_array($gopher_type,
[0, 3, 6])) {
$sites[$i][self::TYPE] = "text/plain";
if ($gopher_type == 6) {
$sites[$i][$value] = convert_uudecode(
$content);
}
} else if ($gopher_type == 'h') {
$sites[$i][self::TYPE] = "text/html";
} else if ($gopher_type == 'g') {
$sites[$i][self::TYPE] = "image/gif";
}
$path_info = pathinfo($filename);
if (!isset($sites[$i][self::TYPE]) &&
isset($path_info['extension'])) {
$sites[$i][self::TYPE] =
UrlParser::guessMimeTypeFromFileName($filename);
} else if (!isset($sites[$i][self::TYPE])) {
$sites[$i][self::TYPE] = "unknown";
}
} else if (!$is_internal) {
$type_parts = explode(";", curl_getinfo($sites[$i][0],
CURLINFO_CONTENT_TYPE));
$sites[$i][self::TYPE] =
strtolower(trim($type_parts[0]));
} else {
$sites[$i][self::TYPE] = "unknown";
}
if (!empty($referer)) {
$sites[$i][self::URL_PARENT] = $referer;
}
}
/*
Ideally should have line for all requests
However, this seems to cause curl to sometimes crash
by trying to free stuff twice on some linux systems
at crawl time. Not having it on other systems causes crashes
at query time
*/
if ($sites[$i][0] != 'INTERNAL') {
if ($minimal || !stristr(PHP_OS, "LINUX")) {
curl_multi_remove_handle($agent_handler, $sites[$i][0]);
}
unset($sites[$i][0]);
}
if (!empty($sites[$i]['ROBOT'])) {
if (isset($sites[$i][self::TYPE]) &&
$sites[$i][self::TYPE] != "text/plain" &&
isset($sites[$i][CrawlConstants::LOCATION]) &&
count($site[CrawlConstants::LOCATION]) > 0) {
$sites[$i][self::TYPE] = "text/plain";
$sites[$i][self::HTTP_CODE] = "200";
$tmp = wordwrap($sites[$i][$value], 80);
$tmp_parts = explode("\n", $tmp);
$tmp = "# Suspect server misconfiguration\n";
$tmp .= "# Assume shouldn't crawl this site.\n";
$tmp .= "# Pretending got following robots.txt.\n";
$tmp .= "User-agent: *\n";
$tmp .= "Disallow: /\n";
$tmp .= "# Original error code: ".
$sites[$i][self::HTTP_CODE]."\n";
$tmp .= "# Original content:\n";
foreach ($tmp_parts as $part) {
$tmp = "#" . $part . "\n";
}
$sites[$i][$value] = $tmp;
$sites[$i][self::HTTP_CODE] = "200";
unset($site[CrawlConstants::LOCATION]);
}
}
} //end big if
} //end for
if ($timer) {
crawlLog(" Get Page Content time ".
(changeInMicrotime($start_time)));
}
if (!empty($agent_handler)) {
unset($agent_handler);
}
return $sites;
}
/**
* Given an extended url in the format url###ip_address###referer###Etag,
* a list of proxy servers, and a temporary directly for ip lookup.
* returns a modified url, returns an array of headers for the etag and to
* keep lighttpd happy, returns a dns resolve string used by curl for IP
* caching, and returns a separated out referer string. If some of the
* the fields in the extended url are not present such as ip, it attempts
* to compute them by looking in a cache in temp_dir. For other values,
* such as referer, if not present, an empty string is return, and for
* Etag the header is just not added to the array of headers returned.
*
* @param string $url site to download with ip address at end potentially
* after ###
* @param array $proxy_servers if not empty an array of proxy servers
* used to crawl through
* @param string $temp_dir folder to store temporary ip header info
* @param bool $expect_header whether to send the Expect: 100-continue
* header
* @return array 5-tuple (orig url, url with replacement, http header array,
* column separated string with host:port:ip -- used for curl hashing,
* referer)
*/
public static function prepareUrlHeaders($url, $proxy_servers = [],
$temp_dir = "", $expect_header = true)
{
if (empty($temp_dir)) {
$temp_dir = C\TEMP_DIR;
if (!file_exists($temp_dir)) {
mkdir($temp_dir);
}
}
$dns_resolve = "";
$dns_resolve_port = 443;
$dns_resolve_host = "";
$url = str_replace("&", "&", $url);
$is_gopher = false;
if (strncmp($url, "gopher", 6) == 0) {
$is_gopher = true;
}
/* Check if an ETag was added by the queue server. If found, create
If-None_Match header with the ETag and add it to the headers. Remove
ETag from URL
*/
$if_none_match = "If-None-Match";
$etag = null;
if (C\USE_ETAG_EXPIRES && stristr($url, "ETag:")) {
$etag_parts = preg_split("/ETag\:/i", $url);
$etag_data = explode(" ", $etag_parts[1]);
$etag = $etag_data[1];
$pos = strrpos($url, "ETag:");
$url = substr_replace($url, "", $pos, strlen("ETag: ".$etag));
}
/* in queue_server we added the ip (if available)
after the url followed by ###
*/
$headers = [];
$referer = "";
$url_ip_parts = explode("###", $url);
if ($proxy_servers != [] || (isset($url_ip_parts[0]) &&
(stripos($url_ip_parts[0],'.onion') !== false)) ) {
$url_ip_parts = [$url_ip_parts[0]];
$url = $url_ip_parts[0];
}
if (count($url_ip_parts) > 1) {
if (!empty($url_ip_parts[2])) {
$referer = substr($url_ip_parts[2], 0, 4096);
unset($url_ip_parts[2]);
}
$ip_address = ltrim(urldecode($url_ip_parts[1]), "#");
unset($url_ip_parts[1]);
$url_parts = @parse_url($url);
if (!empty($url_parts["host"])) {
$dns_resolve_port = 80;
if (!empty($url_parts["port"])) {
$dns_resolve_port = $url_parts["port"];
} else if ($url_parts["scheme"] == "https") {
$dns_resolve_port = 443;
}
$dns_resolve_host = $url_parts["host"];
}
if ($ip_address == '@') {
if (empty(self::$local_ip_cache) &&
file_exists("$temp_dir/" . self::local_ip_cache_file) ) {
self::$local_ip_cache = unserialize(file_get_contents(
"$temp_dir/" . self::local_ip_cache_file) );
}
if (!is_array(self::$local_ip_cache)) {
self::$local_ip_cache = [];
}
if (!empty(self::$local_ip_cache[$url_parts['host']])) {
$ip_address = self::$local_ip_cache[$url_parts['host']];
} else if (!empty($url_parts['host'])) {
$ip_address = gethostbyname($url_parts['host']);
self::$local_ip_cache['dirty'] = true;
self::$local_ip_cache[$url_parts['host']] = $ip_address;
}
}
if (!empty($dns_resolve_host) && !empty($dns_resolve_port) &&
!empty($ip_address)) {
$dns_resolve =
"$dns_resolve_host:$dns_resolve_port:$ip_address";
}
$len = strlen(inet_pton($ip_address));
if (count($url_ip_parts) > 1) {
/*
since did array_pop above, implode may not be same as
original $url
*/
$url = implode("###", $url_ip_parts);
} else {
$url = $url_ip_parts[0];
}
$url_with_ip_if_possible = $url;
} else {
$url_with_ip_if_possible = $url;
}
if ($expect_header) {
$headers[] = 'Expect: 100-continue';
}
if (C\USE_ETAG_EXPIRES && $etag !== null) {
$etag_header = $if_none_match . ": " . $etag;
$headers[] = $etag_header;
}
if ($is_gopher) {
$headers = "gopher";
}
$results = [$url, $url_with_ip_if_possible, $headers, $dns_resolve,
$referer];
return $results;
}
/**
* Computes a hash of a string containing page data for use in
* deduplication of pages with similar content
*
* @param string &$page reference to web page data
* @param string $type for now either text or not-text. If text
* then some tags are removed before computing the hash
* @return string 8 byte hash to identify page contents
*/
public static function computePageHash($page, $type = "not-text")
{
if ($type != "text") {
return crawlHash($page, true);
}
/* to do dedup we strip script, noscript, and style tags
as well as their content, then we strip tags, get rid
of whitespace and hash
*/
$strip_array = ['@<script[^>]*?>.*?</script>@siu',
'@<noscript[^>]*?>.*?</noscript>@siu',
'@<style[^>]*?>.*?</style>@siu'];
$dedup_string = preg_replace($strip_array, '', $page ?? "") ?? "";
$dedup_string_old = preg_replace('/\W+/u', '', $dedup_string ?? "");
$dedup_string = strip_tags($dedup_string_old);
if (empty($dedup_string)) {
$dedup_string = $dedup_string_old;
}
$dedup_string = preg_replace('/\W+/u', '', $dedup_string);
return crawlHash($dedup_string, true);
}
/**
* Splits an http response document into the http headers sent
* and the web page returned. Parses out useful information from
* the header and return an array of these two parts and the useful info.
*
* @param string $header_and_page string of downloaded data
* @param string $value field to store the page portion of page
* @return array info array consisting of a header, page for an http
* response, as well as parsed from the header the server, server
* version, operating system, encoding, and date information.
*/
public static function parseHeaderPage($header_and_page,
$value = CrawlConstants::PAGE)
{
$cache_page_validators = ['ETAG' => -1, 'EXPIRES' => -1];
$new_offset = 0;
// header will include all redirect headers
$site = [];
$site[CrawlConstants::LOCATION] = [];
do {
$continue = false;
do {
$CRLFCRLF = strpos($header_and_page, "\x0D\x0A\x0D\x0A",
$new_offset);
$LFLF = strpos($header_and_page, "\x0A\x0A", $new_offset);
//either two CRLF (what spec says) or two LF's to be safe
$old_offset = $new_offset;
$header_offset = ($CRLFCRLF > 0) ? $CRLFCRLF : $LFLF;
$header_offset = ($header_offset) ? $header_offset : 0;
$new_offset = ($CRLFCRLF > 0) ? $header_offset + 4
: $header_offset + 2;
$new_offset = min($new_offset, max(
strlen($header_and_page) - 1, 0));
$test_pos = strpos($header_and_page, "\n", $new_offset);
$http_check = ltrim(strtoupper(
substr($header_and_page, $new_offset, $test_pos -
$new_offset)));
$http_check = substr($http_check, 0, 4);
//Sometimes in HTTP/2 have a HTTP/2 100 followed HTTP/2 200
} while ($http_check == "HTTP") ;
$redirect_pos = stripos($header_and_page, 'Location:', $old_offset);
$redirect_str = "Location:";
if ($redirect_pos === false) {
$redirect_pos =
stripos($header_and_page, 'Refresh:', $old_offset);
$redirect_str = "Refresh:";
}
if (isset($header_and_page[$redirect_pos - 1]) &&
ord($header_and_page[$redirect_pos - 1]) > 32) {
$redirect_pos = $new_offset; //ignore X-XRDS-Location header
} else if ($redirect_pos !== false && $redirect_pos < $new_offset){
$redirect_pos += strlen($redirect_str);
$pre_line = substr($header_and_page, $redirect_pos,
strpos($header_and_page, "\n", $redirect_pos) -
$redirect_pos);
$loc = @trim($pre_line);
if (strlen($loc) > 0) {
$site[CrawlConstants::LOCATION][] = @$loc;
}
$continue = true;
}
} while($continue);
if ($header_offset > 0) {
$site[CrawlConstants::HEADER] =
substr($header_and_page, 0, $header_offset);
$site[$value] = substr($header_and_page, $new_offset);
} else { //header message no body; maybe 301?
$site[CrawlConstants::HEADER] = $header_and_page;
$site[$value] = " ";
}
$lines = explode("\n", $site[CrawlConstants::HEADER]);
$first_line = array_shift($lines);
$response = preg_split("/(\s+)/", $first_line);
$site[CrawlConstants::HTTP_CODE] = isset($response[1]) ?
@trim($response[1]) : 0;
$site[CrawlConstants::ROBOT_METAS] = [];
foreach ($lines as $line) {
$line = trim($line);
if (stristr($line, 'Server:')) {
$server_parts = preg_split("/Server\:/i", $line);
$server_name_parts = @explode("/", $server_parts[1]);
$site[CrawlConstants::SERVER] = @trim($server_name_parts[0]);
if (isset($server_name_parts[1])) {
$version_parts = explode("(", $server_name_parts[1]);
$site[CrawlConstants::SERVER_VERSION] =
@trim($version_parts[0]);
if (isset($version_parts[1])) {
$os_parts = explode(")", $version_parts[1]);
$site[CrawlConstants::OPERATING_SYSTEM] =
@trim($os_parts[0]);
}
}
}
if (stristr($line, 'Content-type:')) {
list(,$mimetype,) = preg_split("/:|;/i", $line);
$site[CrawlConstants::TYPE] = trim($mimetype);
}
if (stristr($line, 'charset=')) {
$line_parts = preg_split("/charset\=/i", $line);
$site[CrawlConstants::ENCODING] =
strtoupper(@trim($line_parts[1]));
}
if (stristr($line, 'Last-Modified:')) {
$line_parts = preg_split("/Last\-Modified\:/i", $line);
$site[CrawlConstants::MODIFIED] =
strtotime(@trim($line_parts[1]));
}
if (stristr($line, 'X-Robots-Tag:')) { // robot directives pdfs etc
$line_parts = preg_split("/X\-Robots\-Tag\:/i", $line);
$robot_metas = explode(",", $line_parts[1]);
foreach ($robot_metas as $robot_meta) {
$site[CrawlConstants::ROBOT_METAS][] = strtoupper(
trim($robot_meta));
}
}
if (stristr($line, 'Content-Range:')) {
$line_parts = explode("/", $line);
if (!empty($line_parts[1])) {
$content_size = intval(trim($line_parts[1]));
if ($content_size > 0) {
$site[CrawlConstants::CONTENT_SIZE] = $content_size;
}
}
}
$canonical_regex = "/Link\:\s*\<\s*(http.*)\s*\>\s*\;\s*".
"rel\s*\=\s*(\"|')?canonical(\"|')?/";
// levenshtein gives notices on strings longer than 255
if (preg_match($canonical_regex, $line, $matches) &&
isset($site[CrawlConstants::URL]) && strlen($matches[1]) < 252
&& (strlen($site[CrawlConstants::URL]) >= 255 ||
levenshtein($matches[1], $site[CrawlConstants::URL]) > 3)) {
// for rel canonical headers
$site[CrawlConstants::LOCATION][] = $matches[1];
$site[CrawlConstants::ROBOT_METAS][] = 'NOFOLLOW';
}
if (C\USE_ETAG_EXPIRES && stristr($line, 'ETag:')) {
$line_parts = preg_split("/ETag\:/i", $line);
if (isset($line_parts[1])) {
$etag_data = explode(" ", $line_parts[1]);
if (isset($etag_data[1])) {
$etag = $etag_data[1];
$cache_page_validators['ETAG'] = substr($etag, 0, 255);
/* we are only storing first 255 chars of etag so
can use of BPlus trees TEXT field */
}
}
}
if (C\USE_ETAG_EXPIRES && stristr($line, 'Expires:')) {
$line_parts = preg_split("/Expires\:/i", $line);
$all_dates = $line_parts[1];
$date_parts = explode(",", $all_dates);
if (count($date_parts) == 2) {
$cache_page_validators['EXPIRES'] = strtotime(
$date_parts[1]);
} else if (count($date_parts) > 2) {
/*Encountered some pages with more than one Expires date
:O */
$timestamps = [];
for ($i = 1;$i < count($date_parts); $i += 2) {
$ds = strtotime($date_parts[$i]);
$timestamps[] = $ds;
}
$lowest = min($timestamps);
$cache_page_validators['EXPIRES'] = $lowest;
}
}
if (C\USE_ETAG_EXPIRES && !($cache_page_validators['ETAG'] == -1 &&
$cache_page_validators['EXPIRES'] == -1)) {
$site[CrawlConstants::CACHE_PAGE_VALIDATORS] =
$cache_page_validators;
}
}
/*
If the doc is HTML and it uses a http-equiv to set the encoding
then we override what the server says (if anything). As we
are going to convert to UTF-8 we remove the charset info
from the meta tag so cached pages will display correctly and
redirects without char encoding won't be given a different hash.
*/
if (!empty($site[CrawlConstants::TYPE]) && in_array($site[
CrawlConstants::TYPE], ['text/html',
'application/xhtml+xml', 'application/xml',
'text/xml'] ) ) {
$encoding_info = guessEncodingHtmlXml($site[$value], true);
if (is_array($encoding_info)) {
list($site[CrawlConstants::ENCODING], $start_charset, $len_c) =
$encoding_info;
$site[$value] = substr_replace($site[$value], "",
$start_charset, $len_c);
} else {
$site[CrawlConstants::ENCODING] = $encoding_info;
}
}
if (!isset($site[CrawlConstants::SERVER]) ) {
$site[CrawlConstants::SERVER] = "unknown";
}
return $site;
}
/**
* Computes the IP address from http get-responser header
*
* @param string $header contains complete transcript of HTTP get/response
* @return string IPv4 address as a string of dot separated quads.
*/
public static function getCurlIp($header)
{
if (preg_match_all('/Trying\s+(\d+\.\d+\.\d+\.\d+|'.
'[a-f\d]{0,4}(\:[a-f\d]{0,4}){5})(\:\d{1,5})?.*(\.\.\.)/',
$header, $matches)) {
$out_addresses = [];
$addresses = array_unique($matches[1]);
foreach ($addresses as $address) {
$num = @inet_pton($address);
if ($num !== false) {
$out_addresses[] = $address;
}
}
if ($out_addresses != []) {
return $out_addresses;
}
return false;
} else {
return false;
}
}
/**
* Make a curl request for the provided url
*
* @param string $site url of page to request
* @param array $post_data any data to be POST'd to the URL
* @param bool $check_for_errors whether or not to check the response
* for the words, NOTICE, WARNING, FATAL which might indicate an
* error on the server
* @param string $user_password username:password to use for connection if
* needed (optional)
* @param int $timeout how long to wait for page download to complete
* @param array $headers additional HTTP headers to ssend
* @return string the contents of what the curl request fetched
*/
public static function getPage($site, $post_data = null,
$check_for_errors = false, $user_password = null,
$timeout = C\SINGLE_PAGE_TIMEOUT, $headers = [])
{
static $agents = [];
$memory_limit = memoryCeiling(C\MEMORY_FILL_FACTOR);
if (memory_get_usage() > $memory_limit) {
$agents = [];
}
$MAX_AGENTS = 50;
$not_web_setting = (php_sapi_name() == 'cli' &&
!C\nsdefined("IS_OWN_WEB_SERVER"));
$url_parts = @parse_url($site);
$host = (empty($url_parts["host"])) ? false : $url_parts["host"];
$scheme = (empty($url_parts["scheme"])) ? false : $url_parts["scheme"];
if ($host !== false) {
if (self::isSelfUrl($site)) {
if (empty($GLOBALS['web_site'])) {
return "";
}
$web_site = $GLOBALS['web_site'];
$output = $web_site->processInternalRequest($site, false,
$post_data);
return $output;
}
if (count($agents) > $MAX_AGENTS) {
$agent_host = array_shift($agents);
if ($agent_host) {
unset($agent_host);
}
}
$host = ($post_data) ? "POST " . $host : "GET " . $host;
if (empty($agents[$host])) {
$agents[$host] = curl_init();
}
}
if ($not_web_setting) {
crawlLog(" Init curl request of a single page");
}
if (in_array(UrlParser::getHost($site),
C\USER_AGENT_ALTERNATIVE_HOSTS)) {
curl_setopt($agents[$host], CURLOPT_USERAGENT,
C\USER_AGENT_ALTERNATIVE);
} else {
curl_setopt($agents[$host], CURLOPT_USERAGENT, C\USER_AGENT);
}
curl_setopt($agents[$host], CURLOPT_URL, $site);
curl_setopt($agents[$host], CURLOPT_AUTOREFERER, true);
curl_setopt($agents[$host], CURLOPT_FOLLOWLOCATION, true);
/* curl http2 can cause problems when compiled against an older version
of openssl. This seems to mainly affect posting data, so for
now we use HTTP/1.1 to post.
*/
if ($post_data == null && $scheme == "https" &&
defined("CURL_HTTP_VERSION_2_0")) {
curl_setopt($agents[$host], CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_2_0);
}
// these next two lines should probably be modified for better security
curl_setopt($agents[$host], CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($agents[$host], CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($agents[$host], CURLOPT_NOSIGNAL, true);
curl_setopt($agents[$host], CURLOPT_RETURNTRANSFER, true);
curl_setopt($agents[$host], CURLOPT_FAILONERROR, true);
curl_setopt($agents[$host], CURLOPT_TIMEOUT, $timeout);
curl_setopt($agents[$host], CURLOPT_CONNECTTIMEOUT, $timeout);
if (!empty($headers)) {
curl_setopt($agents[$host], CURLOPT_HTTPHEADER, $headers);
}
if ($post_data != null) {
curl_setopt($agents[$host], CURLOPT_POST, true);
curl_setopt($agents[$host], CURLOPT_POSTFIELDS, $post_data);
} else {
// since we are caching agents, need to do this so doesn't get stuck
// as post request and so query string ignored for get requests
curl_setopt($agents[$host], CURLOPT_HTTPGET, true);
}
if ($user_password != null) {
curl_setopt($agents[$host], CURLOPT_FAILONERROR, false);
curl_setopt($agents[$host], CURLOPT_USERPWD, $user_password);
curl_setopt($agents[$host], CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($agents[$host], CURLOPT_SSLVERSION,
CURL_SSLVERSION_TLSv1_2);
}
if ($not_web_setting) {
crawlLog(" Set curl options for single page request");
}
$time = time();
$response = curl_exec($agents[$host]);
if (!$response && $not_web_setting) {
crawlLog(" REQUEST FAILED!! Outputting curl_getinfo...");
crawlLog(print_r(curl_getinfo($agents[$host]), true));
}
if (time() - $time > C\PAGE_TIMEOUT && $not_web_setting) {
crawlLog(" Request took longer than page timeout!!");
crawlLog(" Either could not reach URL or website took too");
crawlLog(" long to respond.");
}
curl_setopt($agents[$host], CURLOPT_POSTFIELDS, "");
if ($not_web_setting) {
crawlLog(" Done curl exec");
}
if ($not_web_setting && $check_for_errors) {
self::checkResponseForErrors($response);
}
return $response;
}
/**
* Given the results of a getPage call, check whether or not the response
* had the words NOTICE, WARNING, FATAL which might indicate an error on
* the server. If it does, then the $response string is sent to the
* crawlLog
*
* @param string $response getPage response in which to check for errors
*/
public static function checkResponseForErrors($response)
{
if (preg_match("/\bNOTICE|WARNING|FATAL\b/i", $response)) {
crawlLog("There appears to have been an error in the server ".
"response. Response was:");
crawlLog(wordwrap($response));
}
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software