<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\atto as A;
use seekquarry\yioop\library\certificates as LC;
use seekquarry\yioop\library\storage_formats as LSF;
use seekquarry\yioop\models as M;
/**
* Used to make using Yioop through this index page rather than as part of
* a library
*/
const BASE_INDEX_ENTRY = true;
/**
* This will in turn load Config.php
*/
require_once __DIR__ . "/src/library/Utility.php";
/**
* passthruYioopRequest hands the request now being served to the copy
* of Yioop that answers it. Two index.php files take part: this one,
* which a web server reaches and which starts the server when Yioop
* runs as its own, and the one under src, which works out the reply.
* This function tidies the address first, sending the reader to the
* plain form where the one asked for holds a needless step in its
* path, and it records that addresses on this site are written the
* short way rather than as a query.
*
* @return bool always true once the inner index page has handled the
* request
*/
function passthruYioopRequest()
{
$web_site = $GLOBALS['web_site'];
$uri = $_SERVER['REQUEST_URI'];
$new_uri = preg_replace("@/(\.?/)+@", "/", $uri);
if (!empty($web_site) && $new_uri != $uri) {
$web_site->header("Location: $new_uri");
A\webExit();
}
require_once __DIR__ . "/src/index.php";
bootstrap($web_site);
return true;
}
/**
* processCommandLine reads the words typed after the script name when
* Yioop is run as its own web server, and works out from them which
* command was asked for and which address to listen on. It rewrites
* those words into the fixed places the rest of this file reads them
* from, so every later step sees one shape whatever was typed.
*/
function processCommandLine()
{
global $argv;
define("seekquarry\\yioop\\configs\\IS_OWN_WEB_SERVER", true);
/*
secure-terminal / secure-start / secure-stop drive the TLS
launcher (bind 80 and 443 with a managed certificate) instead
of the single plain port. secure-terminal runs in this
process; secure-start daemonizes by re-spawning index.php as a
'child' (see CrawlDaemon::start), which drops the secure-*
word. So the secure listen address is set to the self-naming
marker 'secure-80-443' below: CrawlDaemon forwards it to the
child, which reads it back here as argv[3] to re-enable secure
mode, and it also names the daemon (startup line reads
"secure-80-443-index") so it is clear this is the secure
server and on which ports. The secure block ignores the
address otherwise. secure-stop maps to the plain stop, which
signals the secure daemon through the same lock and stop file.
The choice is recorded in a constant the listen-construction
step reads.
*/
$secure_marker = 'secure-80-443';
$is_secure_server = (!empty($argv[1]) && in_array($argv[1],
['secure-terminal', 'secure-start', 'secure-stop',
'secure-restart'], true)) ||
(!empty($argv[1]) && $argv[1] === $secure_marker) ||
(isset($argv[3]) && $argv[3] === $secure_marker);
define("seekquarry\\yioop\\configs\\SECURE_SERVER_MARKER",
$secure_marker);
if (!empty($argv[1])) {
if ($argv[1] === 'secure-terminal') {
$argv[1] = 'terminal';
} else if ($argv[1] === 'secure-start') {
$argv[1] = 'start';
} else if ($argv[1] === 'secure-stop') {
$argv[1] = 'stop';
} else if ($argv[1] === 'secure-restart') {
$argv[1] = 'restart';
}
}
$address = 8080;
$sleep = 0;
$session_file = "";
if (empty($argv[1])) {
echo "Yioop web server commands:\n\n";
echo " php index.php start [port] " .
"start in the background on a port\n";
echo " (default 8080)\n";
echo " php index.php terminal [port] " .
"run in this terminal, not in the\n";
echo " background\n";
echo " php index.php restart [port] " .
"restart a running server, or start\n";
echo " one if none is running\n";
echo " php index.php stop " .
"stop the running server\n";
echo " php index.php status " .
"show which Yioop processes are\n";
echo " running and on which " .
"ports\n\n";
echo "Add 443 as the port to serve HTTPS on the standard " .
"secure port.\n";
echo "The secure- forms bind both 80 and 443 with a managed " .
"certificate:\n";
echo " sudo php index.php secure-start\n";
echo " sudo php index.php secure-terminal\n";
echo " sudo php index.php secure-restart\n";
echo " sudo php index.php secure-stop\n";
exit();
} else {
if ($argv[1] == 'stop' || $argv[1] == 'status' ||
$argv[1] == 'supervisor' || $argv[1] == 'worker' ||
$argv[1] == 'mailsend') {
return;
}
$possible_address = $argv[1];
$possible_sleep = 0;
if (empty($argv[2])) {
if (in_array($argv[1], ['start', 'terminal', 'child', 'restart'])) {
$possible_address = 8080;
} else {
$argv[1] = 'start';
}
} else {
$possible_sleep = $argv[2];
if (in_array($argv[1], ['start', 'terminal', 'child', 'restart'])) {
$possible_address = $argv[2];
if (empty($argv[3])) {
$possible_sleep = 0;
if ($argv[2] == 'none') {
$possible_address = 8080;
}
} else {
$possible_sleep = $argv[3];
if ($argv[2] == 'none') {
$possible_address = $argv[3];
if (empty($argv[4])) {
$possible_sleep = 0;
} else {
$possible_sleep = $argv[4];
}
if (!empty($argv[5])) {
$session_file = $argv[5];
}
} else if (!empty($argv[4])) {
$session_file = $argv[4];
}
}
} else if (!empty($argv[3])) {
if (intval($possible_sleep) > 0) {
$argv[1] = 'restart';
} else {
$argv[1] = 'start';
}
$session_file = $argv[3];
}
}
$tmp_address = intval($possible_address);
if ($tmp_address > 0 && $tmp_address < 65535) {
$address = $possible_address;
} else {
$url_parts = @parse_url($possible_address);
if(!empty($url_parts)) {
$address = $possible_address;
}
}
if (intval($possible_sleep) > 0 ) {
$sleep = $possible_sleep;
}
}
/* A bare "443" port (terminal 443 / start 443) means the user
wants HTTPS on the standard secure port. Turn on secure serving
for that one port: a self-signed certificate is written if none
exists and the port is served over TLS. This is the single-port
counterpart to secure-terminal / secure-start, which instead
bind both 80 and 443 for the full public set-up (HTTP-to-HTTPS
redirect and the ACME challenge on 80). Here only the one port
the user named is served. */
$is_single_secure_port = !$is_secure_server &&
intval($address) === 443;
define("seekquarry\\yioop\\configs\\IS_SECURE_SERVER",
$is_secure_server);
define("seekquarry\\yioop\\configs\\SECURE_SINGLE_PORT",
$is_single_secure_port);
$argv[2] = 'none';
$argv[3] = ($is_secure_server) ? $secure_marker : $address;
$argv[4] = $sleep;
$argv[5] = $session_file;
}
/**
* $web_site is the WebSite object that answers Yioop's requests, with the
* routes set up below. Run from the command line, it listens for web
* connections itself.
*/
/* When this run of the server began. A stop or a restart asked for
before this moment belongs to the server this one replaced; one asked
for after it is this server's to carry out. */
$server_began = time();
$web_site = new A\WebSite('.');
/* The server reads no settings of its own, so what the site wants of it
is handed over here. Left alone, a failure while a request is handled
is caught and only that connection is let go; asked for, it is let out
and ends the server, which is what someone tracking such a failure
down wants to see. */
$web_site->no_wrap_requests = (bool)C\NO_WRAP_REQUESTS;
$web_site->get('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->head('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->post('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->put('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->delete('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->options('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->propfind('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->mkcol('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->move('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->lock('/*', 'seekquarry\\yioop\\passthruYioopRequest');
$web_site->unlock('/*', 'seekquarry\\yioop\\passthruYioopRequest');
/*
The conversation refresh can be pushed over a held-open WebSocket when
Yioop serves itself through this command-line server. The route is
registered here, at startup, so it is in place before any upgrade
request is read; its handler lives in src/index.php, loaded the same
lazy way passthruYioopRequest loads it, so the socket work sits with the
rest of the messaging code rather than in this launcher.
*/
$web_site->ws('/ws-messages', function ($ws) use ($web_site) {
require_once __DIR__ . "/src/index.php";
\seekquarry\yioop\messageWsConnect($ws, $web_site);
});
if ($web_site->isCli()) {
/* HOSTS_NAME_SERVER says this web server is the name server itself:
its machine list names it as BASE_URL. A worker then runs a request
to the name server's address inside itself, rather than sending it
back to this server and waiting on its own queue. */
if ((new M\MachineModel())->hostsNameServer()) {
define("seekquarry\\yioop\\configs\\HOSTS_NAME_SERVER", true);
}
if ($argv[1] == 'worker') {
/* Started by the web server to run requests for it: the routes
above are built as usual, then requests come from the link to
the server rather than from a port. A request brings the
server's settings, which turn on logging to files, so the log
is named index here, as the server's is: otherwise the lines
the worker logs go to message.log. This first line goes to
the worker's output, as no request has come yet. */
L\crawlLog("Web server worker " . getmypid() . " started", "index");
$web_site->serveAsWorker();
exit();
}
if ($argv[1] == 'mailsend') {
\seekquarry\yioop\library\mail\SmtpClient::runSendWorker(
$argv[2] ?? "", intval($argv[3] ?? 0));
exit();
}
if ($argv[1] == 'supervisor') {
/* This run of index.php is the one process that keeps root.
The secure start begins it once the web server is on its
way, and it binds nothing and listens to nobody. Every few
seconds it compares what should be running against what is,
and starts what is missing. That covers both things a server
which has become WWW_USER can no longer do: bring back a
mail server that wants a port below 1024, and put up a
replacement for itself after a restart. What should be
running is read from where it was already recorded, the
ACTIVE_PROCESS rows Manage Machines writes and the port file
the server writes, so nothing here needs a channel of its
own. */
L\CrawlDaemon::$name = "index";
L\CrawlDaemon::$subname = "";
L\CrawlDaemon::$mode = 'supervisor';
$_SERVER["LOG_TO_FILES"] = true;
L\crawlLog("\n\nInitialize logger..", "index");
$address = empty($argv[2]) ? "" : $argv[2];
$terminal_id = intval($argv[3] ?? 0);
$began = time();
$machine_model = new M\MachineModel();
L\crawlLog("Supervisor running as root, watching for services " .
"that should be running on " . $address);
$server_missing_passes = 0;
while (true) {
sleep(C\SUPERVISOR_SLEEP_TIME);
clearstatcache();
$stop_file = L\CrawlDaemon::GLOBAL_STOP_FILE_NAME;
if (file_exists($stop_file) &&
intval(file_get_contents($stop_file)) >= $began) {
L\crawlLog("Supervisor stopping, since everything was " .
"asked to stop");
break;
}
if ($terminal_id > 0 &&
!L\CrawlDaemon::isProcessAlive($terminal_id)) {
L\crawlLog("Supervisor stopping, since the terminal " .
"server it watched has ended");
break;
}
$running = L\CrawlDaemon::statuses();
if (empty($running["index"]) && !empty($address) &&
$terminal_id == 0) {
/* Waiting a second pass leaves room for a restart run
from the command line, which is root as well, to put
the replacement up itself. Two servers binding the
same ports would leave one of them dead. */
$server_missing_passes++;
if ($server_missing_passes > 1) {
L\crawlLog("Supervisor starting a web server on " .
$address . ", since none is running");
preg_match_all('/\d+/', $address, $ports);
foreach ($ports[0] as $port) {
foreach (L\CrawlDaemon::portHolders($port) as
$holder => $command) {
L\crawlLog("Supervisor: port $port is held " .
"by process $holder: $command");
}
}
/* A server that stopped for a restart it could
not do itself, having given up root, saved its
sessions first; the replacement is handed them,
so no one is signed out by the restart. An old
file from an earlier restart is not used. */
$session_path = C\WORK_DIRECTORY .
"/PreviousSession.txt";
$sessions_to_keep = (file_exists($session_path) &&
time() - filemtime($session_path) <
C\ONE_MINUTE * 5) ? $session_path : "";
L\CrawlDaemon::start("index", "none",
'"' . $address . '" "0" "' . $sessions_to_keep .
'"', -3);
$server_missing_passes = 0;
}
} else {
$server_missing_passes = 0;
}
foreach ($machine_model->getNameServerProcessTypes() as $type) {
if (!in_array($type, L\CrawlDaemon::RESTART_DAEMONS,
true) || !empty($running[$type])) {
continue;
}
L\crawlLog("Supervisor starting " . $type .
", which is switched on but not running");
L\CrawlDaemon::start($type, "none", "", -3);
}
}
exit();
}
if ($argv[1] == 'status') {
/* Report what Yioop processes are running and, for the web
server, which port(s) it is listening on. Reads the same
lock files the server uses to track running daemons, so the
report reflects what is actually alive right now. */
$running = L\CrawlDaemon::statuses();
$labels = ["index" => "Web server",
"MediaUpdater" => "Media Updater",
"MailServer" => "Mail Server",
"TurnServer" => "Turn Server",
"QueueServer" => "Queue Server", "Fetcher" => "Fetcher"];
if (empty($running)) {
echo "No Yioop processes appear to be running.\n";
} else {
echo "Running Yioop processes:\n";
foreach ($running as $name => $instances) {
$label = $labels[$name] ?? $name;
$line = " " . $label;
$count = count($instances);
if ($count > 1) {
$line .= " x" . $count;
}
echo $line . "\n";
}
}
exit();
}
$web_site->setTimer(10, function () use ($web_site, $server_began) {
if (!C\PROFILE) {
return;
}
$start_time = $server_began;
$global_stop_file = L\CrawlDaemon::GLOBAL_STOP_FILE_NAME;
if (file_exists($global_stop_file)) {
$stop_time = intval(file_get_contents($global_stop_file));
if ($stop_time > $start_time) {
echo "Stopping Server...";
$web_site->stop = true;
$web_site->restart = false;
/* clear this server's lock as part of honoring the
stop, so a follow-up start is not refused for
PROCESS_TIMEOUT because a request handled between
the stop command's unlink and shutdown recreated
the lock with a fresh timestamp
*/
L\CrawlDaemon::releaseLock();
} else {
unlink($global_stop_file);
}
}
$restart_request_file = C\DATA_DIR . "/RestartServer.txt";
if (file_exists($restart_request_file)) {
$request_time = intval(
file_get_contents($restart_request_file));
unlink($restart_request_file);
L\crawlLog("Restart request read: asked at $request_time, this " .
"server first ticked at $start_time");
if ($request_time > $start_time) {
/* A command-line restart asked for this. Take the
background daemons down and record them for relaunch,
then restart this server the same way the admin
Restart Server action does, so there is one restart
path to maintain. */
L\crawlLog("Restart asked for from the command line: " .
"stopping the services");
/* The moment the restart was asked for is handed on,
so that stopping the services cannot end one that
something else has started since. */
L\CrawlDaemon::saveDaemonStateForRestart($request_time);
$session_path = C\WORK_DIRECTORY .
"/PreviousSession.txt";
L\crawlLog("Restart: saving sessions to $session_path");
if (!is_writable(dirname($session_path)) ||
(file_exists($session_path) &&
!is_writable($session_path))) {
L\crawlLog("Restart: $session_path cannot be written " .
"by this process; restarting without the sessions");
} else {
$web_site->saveSessionsForRestart($session_path);
}
/* A server that has dropped to WWW_USER cannot bind
ports 80 and 443 again, so it stops without asking to
relaunch itself and leaves that to the command, which
runs as root and watches for the server to go. Asking
anyway spawned a process that could not bind and died
with its output in /dev/null. */
$can_rebind = !function_exists("posix_geteuid") ||
posix_geteuid() === 0;
if ($can_rebind) {
L\crawlLog("Restart: stopping this server to " .
"relaunch it");
$web_site->restart = "restart " . $session_path;
} else {
L\crawlLog("Restart: stopping this server; the " .
"restart command starts the new one, since this " .
"process cannot bind its ports again");
}
$web_site->stop = true;
}
}
$log_msg = "Current Memory Usage: " . memory_get_usage() .
" Peak usage:" . memory_get_peak_usage() . " " .
"index-caches indexes=" . count(L\IndexManager::$indexes) .
" urls=" . (string)count(L\IndexManager::$urls_cache) .
" info=" . (string)count(L\IndexManager::$info_cache) .
" max-posting-read=" .
LSF\IndexDocumentBundle::$max_postings_read . "B";
L\crawlLog($log_msg);
});
if (file_exists(C\LOG_DIR)) {
// don't use logging until after log directory built on first run
$web_site->middleware(function () use ($web_site)
{
$lock_file = L\CrawlDaemon::getLockFileName("index", '');
/* in first run case, lock file won't be created by outer
instance as folder not created yet; skip the recreate
while stopping so a shutdown-window request does not
resurrect a lock the stop command just removed
*/
if (empty($web_site->stop) && !file_exists($lock_file)) {
L\CrawlDaemon::writeLockFile($lock_file, time());
}
/* A request from another Yioop machine carries a session
that is the timestamp hashed with this site's key, which
a reader cannot forge. Marking it here lets the web
server leave the fetchers and the archive sync at a whole
turn of its write loop, while the files going to one
block of reader IP addresses divide a turn between
them. */
if (!empty($_REQUEST['session']) &&
!empty($_REQUEST['time'])) {
$_SERVER['MACHINE_REQUEST'] = true;
}
$_SERVER['REQUEST_ARRIVED'] = microtime(true);
});
$web_site->logResponses(function ($status, $size) use ($web_site) {
if (($_SERVER['REMOTE_ADDR'] ?? "") === "0.0.0.0") {
return;
}
$took = "";
$arrived = $_SERVER['REQUEST_ARRIVED'] ??
$_SERVER['REQUEST_TIME_FLOAT'] ?? 0;
if (!empty($arrived)) {
$milliseconds = round((microtime(true) - $arrived) *
C\MILLISECONDS_PER_SECOND);
$mark = C\REQUEST_SLOWEST_MARK;
foreach (C\REQUEST_SPEED_MARKS as $under => $word) {
if ($milliseconds < $under) {
$mark = $word;
break;
}
}
$took = " " . $milliseconds . "ms" . $mark;
}
$who = empty($_SESSION['USER_NAME']) ? "" :
" " . $_SESSION['USER_NAME'];
$fibers = ($web_site->cooperative_scheduler === null) ? 0 :
count($web_site->cooperative_scheduler->tasks);
$ran_in = isset($_SERVER['ATTO_WORKER']) ?
" worker=" . ($_SERVER['ATTO_WORKER'] + 1) : "";
$access_line = $_SERVER['REMOTE_ADDR'] . $who . " " .
$_SERVER['REQUEST_URI'] . " " .
$web_site->requestProtocol() . ":" .
($_SERVER['SERVER_PORT'] ?? "") . " " . $status . " " .
$size . "B" . $took . " fibers=" . $fibers . $ran_in;
L\crawlLog($access_line);
});
register_shutdown_function(function () {
/* If the always-on server dies on a fatal error, a memory
exhaustion most of all, record which request was in flight
and the peak memory reached. A request that exhausts memory
while still building its reply never reaches the response
logger above, so without this the crashing request can only
be guessed at from the surrounding arrival lines. */
$fatal = error_get_last();
if ($fatal === null || !in_array($fatal['type'],
[E_ERROR, E_PARSE, E_CORE_ERROR, E_COMPILE_ERROR])) {
return;
}
$remote = $_SERVER['REMOTE_ADDR'] ?? "?";
$uri = $_SERVER['REQUEST_URI'] ?? "?";
L\crawlLog("Fatal while serving " . $remote . " " . $uri .
" peak_memory=" . memory_get_peak_usage() . " " .
$fatal['message']);
});
}
$session_info = null;
if (!empty($argv[5]) && file_exists($argv[5])) {
$session_info = unserialize(file_get_contents($argv[5]));
if ($argv[1] == 'restart') {
if(!empty($session_info['SESSIONS']['TERMINAL'])) {
$argv[1] = 'start';
} else {
$argv[1] = 'child';
}
}
} else if ($argv[1] == 'restart') {
/* Command-line restart. If a server is already running, leave
the work to it: drop a request file that it picks up on its
next tick and then restarts itself the same way the admin
Restart Server action does (stopping and relaunching the
background daemons, handing sessions across so people stay
signed in). Talking to it through a file rather than a web
request avoids opening an unauthenticated restart endpoint.
If nothing is running, say so and start a fresh server. */
$index_lock = L\CrawlDaemon::getLockFileName("index", '');
$index_lock_alt = L\CrawlDaemon::getLockFileName("index", '0');
$running = false;
foreach ([$index_lock, $index_lock_alt] as $lock) {
if (L\CrawlDaemon::lockHeldByLiveProcess($lock)) {
$running = true;
break;
}
}
if ($running) {
$restart_request_file = C\DATA_DIR . "/RestartServer.txt";
file_put_contents($restart_request_file, time());
chmod($restart_request_file, 0777);
echo "Asked the running server to restart.\n";
/* Follow along until the restart finishes. First wait for
the running server to notice and remove the request file
(it polls on a timer, so this can take a few seconds);
that confirms the restart is under way. Then print each
restart phase the server records (stopping services,
waiting for and restarting the mail server, and so on)
as it changes, and finish once the phase has settled
back to empty, which the server does when it is serving
again. A generous overall cap stops the wait if no
progress is seen. */
$last_phase = "";
$waited = 0;
$max_wait = 120;
$picked_up = false;
$clear_streak = 0;
$done = false;
$started_here = false;
$index_lock = L\CrawlDaemon::getLockFileName("index", "");
$old_process_id = L\CrawlDaemon::lockProcessId($index_lock);
$relaunched = [];
$daemon_labels = ["MediaUpdater" => "Media Updater",
"MailServer" => "Mail Server",
"TurnServer" => "Turn Server"];
while ($waited < $max_wait) {
sleep(1);
$waited++;
clearstatcache(true, $restart_request_file);
if (!$picked_up) {
if (!file_exists($restart_request_file)) {
$picked_up = true;
echo " Restart under way...\n";
}
continue;
}
/* The server records which background services it is
bringing back, and it writes that list after it has
taken the request file away. Reading the list only at
the moment the request file went therefore usually
read nothing, and the services came back without
being named. So it is looked for on every pass until
it is found, since the fresh server takes it away
again once it has relaunched them. */
$marker = C\DATA_DIR . "/RestartDaemons.txt";
clearstatcache(true, $marker);
if (empty($relaunched) && file_exists($marker)) {
foreach (array_filter(array_map('trim',
explode("\n", (string)file_get_contents(
$marker)))) as $entry) {
$name = explode(" ", $entry)[0];
$relaunched[] = $daemon_labels[$name] ?? $name;
}
}
$phase = L\CrawlDaemon::getRestartStatus();
if ($phase !== "") {
$clear_streak = 0;
if ($phase !== $last_phase) {
echo " " . $phase . "\n";
$last_phase = $phase;
}
} else {
$clear_streak++;
}
/* a couple of clear readings in a row with a server
other than the old one holding the lock means the
new server is up and serving */
clearstatcache(true, $index_lock);
$new_process_id = L\CrawlDaemon::lockProcessId($index_lock);
$held = L\CrawlDaemon::lockHeldByLiveProcess($index_lock);
if ($clear_streak >= 3 && $held &&
$new_process_id !== false &&
$new_process_id != $old_process_id) {
$done = true;
break;
}
/* The old server has gone and nothing replaced it. A
server that dropped to WWW_USER cannot bind ports 80
and 443 again, so this command, which has the rights
to, starts the new one. This is read whatever the
phase file says: that file is written by the server
that has just stopped, so it can be left holding an
old phase with nothing running to clear it. */
if ($picked_up && !$held && !$started_here) {
echo " Starting the web server back up...\n";
L\CrawlDaemon::setRestartStatus("");
/* Started from here rather than through init, so
that this command keeps running and can say what
came back. init's start ends the process that
calls it, which left the reader with the daemon's
own startup lines as the last word and no list of
what had been brought back. */
$started_here = true;
$restart_options = "";
for ($word = 3; $word < count($argv); $word++) {
$restart_options .= '"' . $argv[$word] . '" ';
}
L\CrawlDaemon::start("index", "none",
trim($restart_options), -3);
}
}
if ($done) {
echo "Restart complete; the web server is back up.\n";
if (!empty($relaunched)) {
/* Each service is looked at once, as things stand
when the server is back. The list read earlier
says which were asked for, not which came back,
so naming them all would name one that never
started. Nothing is waited for here: a service
that takes a moment longer than the server is
said to be starting, and the reader gets their
prompt back at once. */
$came_back = [];
$starting = [];
$did_not_start = [];
foreach ($relaunched as $service) {
$daemon = array_search($service, $daemon_labels);
if ($daemon === false) {
$daemon = $service;
}
$state = L\CrawlDaemon::daemonState($daemon);
if ($state == L\CrawlDaemon::DAEMON_RUNNING) {
$came_back[] = $service;
} else if ($state ==
L\CrawlDaemon::DAEMON_STARTING) {
$starting[] = $service;
} else {
$did_not_start[] = $service;
}
}
if (!empty($came_back)) {
echo "These background services are running " .
"again: " . implode(", ", $came_back) .
".\n";
}
if (!empty($starting)) {
echo "These are starting up: " .
implode(", ", $starting) . ".\n";
}
foreach ($did_not_start as $service) {
/* A reader who has just restarted their site
wants to know what to do next, so the name of
the file to look in is written out rather
than left to be worked out. */
$daemon = array_search($service, $daemon_labels);
if ($daemon === false) {
$daemon = $service;
}
echo "The " . $service . " did not come back " .
"up.\n";
echo " See what is running with: php " .
"index.php status\n";
echo " The reason should be at the end of " .
"this file:\n";
echo " " . C\LOG_DIR . "/" . $daemon .
".log\n";
}
}
} else if (!$picked_up) {
echo "The running server did not pick up the restart " .
"request; check the server log.\n";
} else {
echo "Restart did not confirm within " . $max_wait .
" seconds; check the server log.\n";
}
exit();
}
echo "No server appears to be running; starting a fresh one.\n";
$argv[1] = 'start';
}
if ($argv[1] == 'start' || $argv[1] == 'terminal') {
/* Yioop runs a single index web server. If one is already up, a
second start or terminal run (even on a different port) cannot
bind alongside it, so say so clearly and point at the right
next step rather than leaving the impression that two servers
on two ports are now running. A lock left behind by a server
that crashed or was stopped with Ctrl-C is recognized as stale
through its recorded process id and removed, so the next start
or terminal run is not blocked and needs no separate stop
first. */
$index_lock = L\CrawlDaemon::getLockFileName("index", '');
$index_lock_alt = L\CrawlDaemon::getLockFileName("index", '0');
foreach ([$index_lock, $index_lock_alt] as $lock) {
if (!file_exists($lock)) {
continue;
}
if (L\CrawlDaemon::lockHeldByLiveProcess($lock)) {
echo "An index web server is already running. " .
"Yioop runs one index web server " .
"at a time, so it cannot also start on " .
"another port.\n";
/* A reader who typed secure-start is told about
secure-stop and secure-restart, since the plain
commands serve a different pair of ports. */
$kind = (C\nsdefined("IS_SECURE_SERVER") &&
C\IS_SECURE_SERVER) ? "secure-" : "";
echo "To move it to a different port, stop it " .
"first (php index.php " . $kind . "stop), then " .
"start on the port you want. To restart it in " .
"place, use php index.php " . $kind .
"restart.\n";
exit();
}
unlink($lock);
}
}
if ($argv[1] == 'start') {
/* The address the started server answers on, which the start
connects to before it says whether the server came up. A
secure server answers on 443; a server bound to every
interface is reached on this machine's loopback address. */
$check_address = ((C\nsdefined("IS_SECURE_SERVER") &&
C\IS_SECURE_SERVER) || (C\nsdefined("SECURE_SINGLE_PORT") &&
C\SECURE_SINGLE_PORT)) ? "443" : (string)($argv[3] ?? "");
if (ctype_digit($check_address)) {
$check_address = "127.0.0.1:$check_address";
} else {
$address_parts = @parse_url($check_address) ?: [];
$check_host = trim($address_parts['host'] ?? "", "[]");
if (in_array($check_host, ["", "0.0.0.0", "::"])) {
$check_host = "127.0.0.1";
} else if (str_contains($check_host, ":")) {
$check_host = "[$check_host]";
}
$check_address = empty($address_parts['port']) ? "" :
$check_host . ":" . $address_parts['port'];
}
L\CrawlDaemon::$index_address = $check_address;
if ($check_address != "" &&
L\CrawlDaemon::indexServerUp($check_address, 0)) {
/* No index server runs, as its lock said above, so another
program holds the address and the server could not listen
on it. Nothing is started. */
echo "Another program already answers on $check_address, " .
"so the web server cannot listen there.\n";
exit();
}
}
if (in_array($argv[1], ['start', 'terminal']) && !empty(C\WWW_USER) &&
function_exists("posix_geteuid") && posix_geteuid() === 0) {
/* Start the one process that keeps root, before this one
spawns the server and exits. It is what starts a mail
server or a replacement web server later, since neither
can bind its ports once the server has become WWW_USER.
Where there is no user to become, or where this process is
not root, nothing here is needed and none is started. */
/* A terminal server runs in this process, so the supervisor is
given its id: it watches that process rather than starting a
web server of its own, and stops once that process ends. */
L\CrawlDaemon::execScriptInOwnProcess("'" . C\PARENT_DIR .
"/index.php'", "supervisor '" . $argv[3] . "'" .
(($argv[1] == 'terminal') ? " " . getmypid() : ""));
}
$launcher_lock = L\CrawlDaemon::getLockFileName("Launcher", "");
if (in_array($argv[1], ['start', 'terminal']) &&
!L\CrawlDaemon::lockHeldByLiveProcess($launcher_lock)) {
/* The Launcher starts the programs the web server is asked to
start, such as a fetcher that stopped. Started here, before
the server takes connections, it holds none of them, so
neither do the programs it starts. */
L\CrawlDaemon::start("Launcher", "none", "", -1);
}
if ($argv[1] == 'stop') {
(new M\MachineModel())->clearNameServerProcesses();
if (L\CrawlDaemon::lockHeldByLiveProcess($launcher_lock)) {
L\CrawlDaemon::stop("Launcher", "", false);
}
}
L\CrawlDaemon::init($argv, "index");
L\crawlLog("\n\nInitialize logger..", "index", true);
/* Write this server's lock now, at startup. The per-request
middleware also writes it, but a server that has just
(re)started in the background as a daemon may sit with no
inbound request for several seconds; the periodic crawlLog
in the stop-watch timer calls CrawlDaemon::processHandler,
which exits a daemon whose lock file is missing. Creating the
lock here keeps the freshly restarted server alive until the
first request arrives. */
if (file_exists(C\SCHEDULES_DIR)) {
$index_lock_file = L\CrawlDaemon::getLockFileName("index", '');
if (!file_exists($index_lock_file)) {
L\CrawlDaemon::writeLockFile($index_lock_file, time());
L\giveFileToWebUser($index_lock_file);
}
}
if ($argv[4] > 0) {
sleep($argv[4]);
}
if (empty($session_info)) {
$session_info = ['SESSIONS' => []];
}
if ($argv[1] == 'terminal') {
$session_info['SESSIONS']['TERMINAL'] = true;
}
$context = [];
/* Let the server accept a request body as large as PHP itself is
configured to accept, rather than capping it at the atto
default (10MB). Without this, a file upload bigger than 10MB —
well within a typical PHP post_max_size — is refused before it
reaches Yioop and the browser shows a bare 400 Bad Request.
PHP's post_max_size is read and converted to a byte count; a
value of 0 means "no limit" in PHP, and any value below the
atto default is raised to it so a small or unset ini setting
cannot shrink what the server will take. */
$php_post_max = L\metricToInt(ini_get('post_max_size'));
/* The floor on how big a request body may be comes from the web
server's own defaults rather than being repeated here, so the two
cannot drift apart. */
$atto_defaults = A\WebSite::defaultServerGlobals();
$atto_default_max_request = $atto_defaults['MAX_REQUEST_LEN'];
if ($php_post_max <= 0) {
$context['MAX_REQUEST_LEN'] = PHP_INT_MAX;
} else if ($php_post_max < $atto_default_max_request) {
$context['MAX_REQUEST_LEN'] = $atto_default_max_request;
} else {
$context['MAX_REQUEST_LEN'] = $php_post_max;
}
if (C\nsdefined("SERVER_CONTEXT")) {
$context['SERVER_CONTEXT'] = C\SERVER_CONTEXT;
}
/* The names this server answers to, so the WebSite can set
$_SERVER['SERVER_NAME'] per request to the domain a visitor
actually used (validated against this list) rather than the
single name it bound to. Outbound mail links read SERVER_NAME,
so a visitor on one of several served domains gets links back
to that same domain. Passed as a plain list to keep the atto
WebSite free of Yioop config; an empty list leaves the
per-request naming off. */
$configured_domains = trim((string)C\SECURE_DOMAINS);
if ($configured_domains !== "") {
$context['SECURE_DOMAINS'] = array_values(array_filter(
array_map('trim', preg_split('/[\s,]+/',
$configured_domains))));
}
/* How often (in whole seconds) the running web server should write
a memory-usage line to the error log, or 0 to leave it off. It
is a switchable diagnostic for tracking slow memory growth in the
always-on server. Passed as a plain server value so the atto
WebSite stays free of Yioop config; the WebSite takes it back out
of its globals so it does not leak into $_SERVER. */
$context['WEBSITE_MEMORY_SAMPLE_PERIOD'] =
(int)C\WEBSITE_MEMORY_SAMPLE_PERIOD;
$context['SESSION_INFO'] = $session_info;
if ((int)C\WEB_SERVER_WORKERS > 0) {
$context['WORKERS'] = (int)C\WEB_SERVER_WORKERS;
/* The crawl's own requests, from fetchers and queue servers, have
the last worker to themselves, so a crawl keeps moving while the
other workers answer searches. */
$context['WORKER_RESERVED'] = function () {
return ($_REQUEST['c'] ?? '') === 'fetch';
};
$context['WORKER_STOP'] = function ($process_id, $ask_first) {
L\CrawlDaemon::stopProcess($process_id, $ask_first);
};
$context['WORKER_START'] = function () {
L\CrawlDaemon::execScriptInOwnProcess("'" . C\PARENT_DIR .
"/index.php'", "worker");
};
$context['WORKER_LOG'] = C\LOG_DIR . "/index.log";
/* Streamed replies are sent from the web server as they are
made, so the requests that stream them stay there, as
WebSocket connections do: resources such as video, whether
asked for by c=resource or by a /wd/ path, anything asked for
by byte range, and mail attachment downloads. */
$context['FRONT_ONLY'] = function () {
/* A request from no session for a target already hot is
refused here, by bootstrap's refuseHotTarget, rather than
queued for a worker only to be refused there. */
$key = empty($_COOKIE[C\p('SESSION_NAME')]) ?
L\HotTargets::keyFromRequest($_SERVER['REQUEST_URI'] ?? "",
$_REQUEST) : "";
if ($key !== "" && L\HotTargets::isHot($key, time())) {
return true;
}
return ($_REQUEST['c'] ?? '') === 'resource' ||
str_contains($_SERVER['REQUEST_URI'] ?? '', '/wd/') ||
!empty($_SERVER['HTTP_RANGE']) ||
in_array($_REQUEST['arg'] ?? '', ['downloadAttachment',
'downloadAllAttachments']);
};
$context['FILE_JOURNAL'] = C\DATA_DIR . "/WebServerFileJournal.txt";
file_put_contents($context['FILE_JOURNAL'], "");
/* Made while the server is still root; the workers append to it
as the run-as user. */
L\giveFileToWebUser($context['FILE_JOURNAL']);
}
/* On a restart the WebSite re-spawns itself; for the secure
server it must come back binding 80 and 443, which the
launcher selects from this marker as argv[1] rather than the
single parsed address. Passing it as a server global lets the
atto WebSite reuse it without knowing anything Yioop-specific
about secure mode. */
if (C\IS_SECURE_SERVER) {
$context['RESTART_ADDRESS'] = C\SECURE_SERVER_MARKER;
}
$listen_address = $argv[3];
if (C\IS_SECURE_SERVER || C\SECURE_SINGLE_PORT) {
/*
Secure launcher: bind 80 and 443 from this one process
rather than the single port the plain modes use. Port
443 carries the managed certificate from SECURITY_DIR
(not SERVER_CONTEXT); port 80 is plain HTTP, used to
answer the ACME HTTP-01 challenge and otherwise redirect
to HTTPS. The privileged binds need root; WWW_USER (when
set) is the user the server drops to afterward, so the
long-running process does not hold root. Cert reload on
renewal updates the live listener context in place and
needs no rebind, so the drop does not block renewal.
*/
unset($context['SERVER_CONTEXT']);
$cert_file = C\SECURE_CERT_FILE;
$key_file = C\SECURE_KEY_FILE;
$has_cert = file_exists($cert_file) && file_exists($key_file);
if (!$has_cert) {
/* No managed certificate yet (for example a fresh
install). Generate a self-signed bootstrap certificate
so port 443 binds with a working certificate rather
than a missing one, which would accept connections and
then fail every TLS handshake. The server runs (with a
browser trust warning); when ACME is on, the HTTP-01
challenge on port 80 then replaces it with a real
certificate. Covers the configured secure domains so
the placeholder at least names the right hosts. */
$secure_domains = C\SECURE_DOMAINS;
$boot_domains = ($secure_domains === '') ? [] :
array_filter(array_map('trim',
explode(',', $secure_domains)));
$self_signed = LC\CsrBuilder::selfSigned($boot_domains);
if ($self_signed !== null) {
$installer = new LC\CertInstaller($cert_file,
$key_file);
$has_cert = $installer->writeFiles(
$self_signed['cert_pem'],
$self_signed['key_pem']);
}
if ($has_cert) {
L\crawlLog("Secure server: no managed certificate " .
"found; wrote a self-signed bootstrap " .
"certificate so 443 can bind. Browsers will warn " .
"until a real certificate is installed" .
(C\ACME_ON ? " (ACME will obtain one)." : "."));
} else {
L\crawlLog("Secure server: no managed certificate " .
"and could not write a self-signed bootstrap " .
"certificate; TLS on 443 will fail until a " .
"certificate is installed.");
}
}
$alpn = C\ALPN_PROTOCOLS;
$alpn_names = ($alpn === '') ? [] :
array_filter(array_map('trim', explode(',', $alpn)));
$ssl_context = [
'local_cert' => $cert_file,
'local_pk' => $key_file,
'allow_self_signed' => true,
'verify_peer' => false,
'alpn_protocols' => $alpn,
];
if (C\SECURE_SINGLE_PORT) {
/* single secure port: serve TLS on 443 only, with no
port-80 companion, since the user asked for one port. */
$listen_address = [
['address' => 'tcp://0.0.0.0:443',
'context' => ['ssl' => $ssl_context]],
];
} else {
$listen_address = [
['address' => 'tcp://0.0.0.0:443',
'context' => ['ssl' => $ssl_context]],
['address' => 'tcp://0.0.0.0:80'],
];
}
/* When h3 is among the selected protocols, add a QUIC
listener on UDP 443 carrying the same managed certificate.
It coexists with the TCP 443 listener: HTTP/2 and
HTTP/1.1 over TCP, HTTP/3 over QUIC, all advertised in the
TCP listener's ALPN list. The H3 listener opens only if
the optional QUIC pieces are present; otherwise it falls
through and the server keeps serving over TCP. */
$h3_on = in_array('h3', $alpn_names);
if ($h3_on) {
$listen_address[] = [
'address' => 'tcp://0.0.0.0:443',
'context' => ['ssl' => $ssl_context],
'protocol' => 'h3',
];
}
if (!empty(C\WWW_USER)) {
$context['USER'] = C\WWW_USER;
/* The server drops to this user after binding its ports.
PHP reads the certificate and key at the first handshake,
after the drop, and the server goes on writing its lock,
origin and messages files under schedules and its log
files under log, some of which this process has already
written as root. All of those are given to that user now,
while this process can still do it: the security folder
with its two files, and the schedules and log folders with
every plain file directly inside each. */
$www_user = function_exists("posix_getpwnam") ?
posix_getpwnam(C\WWW_USER) : false;
/* Only a process running as root can give a file away, and
only root can bind ports 80 and 443. A relaunch started by
a server that had already dropped to WWW_USER is neither,
and its chown calls raised warnings that Yioop's error
handler turned into output before the bind was even
tried. */
$as_root = function_exists("posix_geteuid") &&
posix_geteuid() === 0;
if ($as_root && !empty($www_user['uid'])) {
$owned = [C\SECURITY_DIR, C\SCHEDULES_DIR, C\LOG_DIR,
C\WORK_DIRECTORY . "/PreviousSession.txt",
C\DATA_DIR . "/RestartStatus.txt",
C\DATA_DIR . "/RestartDaemons.txt"];
foreach ([C\SCHEDULES_DIR, C\LOG_DIR] as $folder) {
foreach (glob("$folder/*") ?: [] as $entry) {
if (is_file($entry)) {
$owned[] = $entry;
}
}
}
if ($has_cert) {
$owned[] = $cert_file;
$owned[] = $key_file;
}
foreach ($owned as $entry) {
/* A path in this list need not be there yet: the
restart files are written only by a restart. The
@ does not keep Yioop's error handler quiet, and
what it prints lands in front of the page body
and stops the server answering at all. */
if (file_exists($entry)) {
chown($entry, $www_user['uid']);
}
}
clearstatcache();
foreach (($has_cert ? [$cert_file, $key_file] : []) as
$needed) {
if (fileowner($needed) !== $www_user['uid']) {
echo "Could not give $needed to " . C\WWW_USER .
"; the secure port will refuse every " .
"connection until that user can read it.\n";
}
}
}
}
if (!C\ACME_ON) {
L\crawlLog("Secure server: ACME is OFF (ACME_ON is " .
"false); certificates will not be obtained or renewed " .
"automatically.");
} else {
L\crawlLog("Secure server: ACME is ON. Certificate " .
"renewal runs as a media job, so MediaUpdater must be " .
"running for certificates to update.");
}
if ($h3_on) {
L\crawlLog("Secure server: HTTP/3 selected; opening a " .
"QUIC listener on UDP 443. If the optional QUIC " .
"extensions are missing the server keeps serving " .
"over TCP (HTTP/2 and HTTP/1.1).");
}
}
$gc_window = max(intval(C\AUTOLOGOUT),
intval(ini_get('session.gc_maxlifetime')));
ini_set('session.gc_maxlifetime', (string)$gc_window);
if (C\IS_SECURE_SERVER || C\SECURE_SINGLE_PORT) {
/*
Watch the managed certificate file and refresh the live
TLS listener context whenever it changes, so a renewal
written by the AcmeRenew media job (which runs in the
MediaUpdater process, not this one) is served without a
restart. The once-a-minute stat is cheap; new handshakes
pick up the refreshed certificate while in-flight
connections finish on the one they started with.
*/
$secure_cert_file = C\SECURE_CERT_FILE;
$secure_key_file = C\SECURE_KEY_FILE;
$last_cert_mtime = file_exists($secure_cert_file)
? filemtime($secure_cert_file) : 0;
$web_site->setTimer(C\ONE_MINUTE,
function () use ($web_site, $secure_cert_file,
$secure_key_file, &$last_cert_mtime) {
clearstatcache(true, $secure_cert_file);
if (!file_exists($secure_cert_file)) {
return;
}
$mtime = filemtime($secure_cert_file);
if ($mtime <= $last_cert_mtime) {
return;
}
$last_cert_mtime = $mtime;
$web_site->reloadSecureCertificate($secure_cert_file,
$secure_key_file);
L\crawlLog("Secure server: certificate file changed; " .
"refreshed the live TLS listener context. New " .
"connections use the renewed certificate.");
});
}
$restart_daemons_marker = C\DATA_DIR . "/RestartDaemons.txt";
if (file_exists($restart_daemons_marker)) {
/* a Restart Server action or profile change stopped the
background daemons (MediaUpdater, MailServer, TurnServer)
that had
been started through Manage Machines and recorded them
here; wait for each to finish exiting then relaunch it.
The waiting is done across repeating-timer ticks rather
than in one blocking loop so the freshly bound web server
keeps serving (and answering the restart-status poll)
while a daemon that is mid-download takes its time to
stop. The marker is removed up front so this fires once. */
$wanted = array_filter(array_map('trim',
explode("\n", (string) file_get_contents(
$restart_daemons_marker))));
unlink($restart_daemons_marker);
$relaunch_state = ["index" => 0, "waited" => 0, "id" => null];
$relaunch_labels = ["MediaUpdater" => "Media Updater",
"MailServer" => "Mail Server", "TurnServer" => "Turn Server"];
$relaunch_timer = $web_site->setTimer(1,
function () use ($web_site, $wanted, $relaunch_labels,
&$relaunch_state) {
while ($relaunch_state["index"] < count($wanted)) {
$entry_parts = explode(" ",
$wanted[$relaunch_state["index"]]);
$daemon = $entry_parts[0];
$old_process_id = isset($entry_parts[1]) ?
intval($entry_parts[1]) : 0;
if (!in_array($daemon,
L\CrawlDaemon::RESTART_DAEMONS, true)) {
$relaunch_state["index"]++;
$relaunch_state["waited"] = 0;
continue;
}
$daemon_label = $relaunch_labels[$daemon] ?? $daemon;
$step = L\CrawlDaemon::drainDaemonStep($daemon,
$old_process_id, $relaunch_state["waited"]);
if ($step == "waiting") {
L\CrawlDaemon::setRestartStatus(
"Waiting for " . $daemon_label .
" to stop...");
$relaunch_state["waited"]++;
return;
}
$statuses = L\CrawlDaemon::statuses();
if (empty($statuses[$daemon])) {
L\CrawlDaemon::setRestartStatus(
"Starting " . $daemon_label . "...");
/* -3 asks start not to look at the lock. This
runs inside the web server, and a start that
finds a lock still held ends the process that
called it, which here is the server itself:
that is what left a restart with no server and
no services. The daemon has been drained just
above, so the lock is stale whatever it says. */
L\CrawlDaemon::start($daemon, "none", "", -3);
}
$relaunch_state["index"]++;
$relaunch_state["waited"] = 0;
}
L\CrawlDaemon::setRestartStatus("");
if ($relaunch_state["id"] !== null) {
$web_site->clearTimer($relaunch_state["id"]);
}
});
$relaunch_state["id"] = $relaunch_timer;
}
$web_site->listen($listen_address, $context);
} else {
$web_site->process();
}
- Yioop Repo - Seekquarry