<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\wiki;
/**
* WikiPagesFile reads and writes a file of wiki pages in the form
* PublicHelpPages.php has. Such a file holds named arrays of pages, such
* as $public_pages and $help_pages, each keyed by locale tag and then by
* page name. Each page's text sits between a line ending in <<< 'EOD' and
* a line holding only EOD;. That form is PHP, but the file is only read
* as text here and never run, so a file made on one site and given to
* another runs no code. ConfigureTool uses it to make the databases and to
* export and import the public and help pages, and GroupWikiTool uses it
* to export and import the pages of a group.
*/
class WikiPagesFile
{
/**
* END_CHARACTERS matches what may come after a page's end mark on a
* line PHP 7.3 and later would take as the end of the page's text: no
* letter, digit or underscore. A page holding a line that starts, after
* spaces or tabs, with its end mark followed by none of these would be
* ended there, so write picks a mark no such line holds.
*/
const END_CHARACTERS = '(?![A-Za-z0-9_\x80-\xff])';
/**
* PUBDATES is put after the name of an array of pages to name the
* array that holds the time of each page's revision, such as
* $public_pages_pubdates beside $public_pages. The time is a Unix time,
* in seconds.
*/
const PUBDATES = "_pubdates";
/**
* COMMENTS is put after the name of an array of pages to name the
* array that holds the posts of each page's discussion thread, such as
* $public_pages_comments beside $public_pages. Each entry is one line
* of JSON (JavaScript Object Notation), a list of the thread's posts.
*/
const COMMENTS = "_comments";
/**
* HISTORY is put after the name of an array of pages to name the array
* that holds every revision of each page, such as
* $public_pages_history beside $public_pages. Each entry is one line of
* JSON, a list of the page's revisions, oldest first.
*/
const HISTORY = "_history";
/**
* PARTS is the name of the variable, near the top of a file, that says
* which kinds of data the file carries besides the pages, as words
* separated by spaces: issue_pages, comments and history. An import
* that deletes first deletes only the kinds the file carries.
*/
const PARTS = "exported_parts";
/**
* read gives the named arrays of pages that a file in the form of
* PublicHelpPages.php holds, without running the file. It finds each
* line of the form $name["locale"]["page"] = <<< 'EOD' and takes the
* lines after it, up to the line EOD;, as that page's text. A page
* written with another end mark, such as EOD_1, ends at that mark.
*
* The file is read a line at a time by eachPage, so the memory read
* needs is about the size of the pages themselves.
*
* @param string $file_name the file to read
* @return array the file's arrays: array name, then locale tag, then
* page name, to the page's text; empty if the file cannot be read
*/
public static function read($file_name)
{
$arrays = [];
foreach (self::eachPage($file_name) as [$array_name, $locale_tag,
$page_name, $page]) {
$arrays[$array_name][$locale_tag][$page_name] = $page;
}
return $arrays;
}
/**
* exportedParts gives the kinds of data a file in the form of
* PublicHelpPages.php says it carries besides the pages, from the
* variable PARTS names near its top. An import that deletes first
* calls it, so it deletes only the kinds the file brings back. A file
* written before such files said so, which carried git issue pages
* and neither comments nor history, gives just issue_pages.
*
* @param string $file_name the file to read
* @return array the kinds the file carries, among issue_pages,
* comments and history
*/
public static function exportedParts($file_name)
{
$handle = @fopen($file_name, "rb");
if ($handle === false) {
return [];
}
$parts = ["issue_pages"];
$pattern = '/^\$' . self::PARTS . ' = "([a-z_ ]*)";$/';
while (($line = fgets($handle)) !== false) {
$line = rtrim($line, "\r\n");
if (preg_match($pattern, $line, $match)) {
$parts = array_values(array_filter(explode(" ", $match[1])));
break;
}
if (str_contains($line, "<<< 'EOD")) {
break;
}
}
fclose($handle);
return $parts;
}
/**
* exportedGroups gives the arrays of pages a file in the form of
* PublicHelpPages.php declares, among public_pages and help_pages.
* write declares each array it is given, even one with no pages, so an
* export of just the Public group or just the Help group declares just
* that one. An import that deletes first deletes pages only in the
* groups the file declares, and createdb takes a group the file does
* not declare from the file that ships with Yioop.
*
* @param string $file_name the file to read
* @return array the names of the arrays the file declares, in the
* order it declares them; empty if the file cannot be read
*/
public static function exportedGroups($file_name)
{
$handle = @fopen($file_name, "rb");
if ($handle === false) {
return [];
}
$groups = [];
$pattern = '/^(?:\s*\*\/)?\$(public_pages|help_pages) = \[\];$/';
while (($line = fgets($handle)) !== false) {
if (preg_match($pattern, rtrim($line, "\r\n"), $match) &&
!in_array($match[1], $groups)) {
$groups[] = $match[1];
}
}
fclose($handle);
return $groups;
}
/**
* eachPage hands out the pages of a file in the form of
* PublicHelpPages.php one at a time, in the order the file lists them,
* reading the file a line at a time. upgradePublicHelpWiki writes each
* page to the database as it comes, so importing a site's export of
* many pages holds one page in memory rather than all of them. A page
* whose time is in the file, on the line just before the page, is
* handed out with that time, so the import dates it as the site that
* wrote the file had it dated. A page whose thread's posts are in the
* file, just before the page, is handed out with those posts.
*
* @param string $file_name the file to read
* @return \Generator yields [array name, locale tag, page name, the
* page's text, the Unix time of the page's revision or -1 where the
* file gives none, the list of the page's thread's posts or null
* where the file gives none, the list of the page's revisions or
* null where the file gives none] for each page; nothing if the
* file cannot be read
*/
public static function eachPage($file_name)
{
$handle = @fopen($file_name, "rb");
if ($handle === false) {
return;
}
$start_pattern = '/^\$(\w+)\["([^"]*)"\]\["([^"]*)"\] = ' .
"<<< '(EOD[A-Z0-9_]*)'$/u";
$pubdate_pattern = '/^\$(\w+)' . self::PUBDATES . '\["([^"]*)"\]' .
'\["([^"]*)"\] = (\d+);$/u';
$pubdate_key = "";
$pubdate = -1;
$side_keys = [self::COMMENTS => "", self::HISTORY => ""];
$side_values = [self::COMMENTS => null, self::HISTORY => null];
$page = null;
while (($line = fgets($handle)) !== false) {
$line = rtrim($line, "\r\n");
if ($page === null) {
if (preg_match($pubdate_pattern, $line, $match)) {
$pubdate_key = "$match[1]\n$match[2]\n$match[3]";
$pubdate = intval($match[4]);
} elseif (preg_match($start_pattern, $line, $match)) {
[, $array_name, $locale_tag, $page_name, $mark] =
$match;
$end_mark = "$mark;";
$page = "";
$first_line = true;
}
continue;
}
if ($line === $end_mark) {
$side = "";
foreach ([self::COMMENTS, self::HISTORY] as $suffix) {
$suffix_at = strlen($array_name) - strlen($suffix);
if ($suffix_at > 0 &&
substr($array_name, $suffix_at) == $suffix) {
$side = $suffix;
$side_keys[$suffix] = substr($array_name, 0,
$suffix_at) . "\n$locale_tag\n$page_name";
$value = json_decode($page, true);
$side_values[$suffix] = is_array($value) ? $value :
null;
}
}
if ($side == "") {
$key = "$array_name\n$locale_tag\n$page_name";
$found = [];
foreach ($side_keys as $suffix => $side_key) {
$found[$suffix] = ($side_key == $key) ?
$side_values[$suffix] : null;
$side_keys[$suffix] = "";
}
yield [$array_name, $locale_tag, $page_name, $page,
($pubdate_key == $key) ? $pubdate : -1,
$found[self::COMMENTS], $found[self::HISTORY]];
$pubdate_key = "";
}
$page = null;
continue;
}
$page .= ($first_line ? "" : "\n") . $line;
$first_line = false;
}
fclose($handle);
}
/**
* write saves named arrays of pages to a file in the form of
* PublicHelpPages.php. The file starts with Yioop's license, a title
* and a line naming the command that made it, and each array has a
* docblock with its description. Each page's text ends at the line
* EOD;. A page holding a line PHP would take as that end, such as a
* page showing this file's own form, is written with EOD_1, EOD_2 or
* the first such mark none of its lines starts with, so every page is
* written and the file stays valid PHP. Where the time of a page's
* revision is given, it is written on the line before the page, in an
* array named for the page's array with PUBDATES after it. Where the
* posts of a page's thread are given, they are written before the page
* as one line of JSON, in an array named with COMMENTS after it. A
* reader of the file from before these were written takes them as
* arrays of their own, which the import does not use. A page's
* revisions are written the same way, in an array named with HISTORY
* after it; they are asked for one page at a time, so a large export
* is never held whole, and the file is written as it is made. Where
* the kinds of data the file carries are given, they are written near
* the top in the variable PARTS names.
*
* @param string $file_name the file to write
* @param array $arrays array name to [description, pages, times,
* comments, history], where the last three may be left off. Pages
* are keyed by locale tag and then by page name. Times, keyed the
* same way, give the Unix time of each page's revision, and
* comments, keyed the same way, give the list of posts of each
* page's thread. History is a function taking a locale tag and a
* page name and giving the list of that page's revisions.
* @param string $title what the file holds, for its header
* @param string $made_by the command that made the file, for its
* header
* @param array $parts the kinds of data the file carries besides the
* pages, among issue_pages, comments and history; null writes no
* PARTS line, as for a file whose kinds are not known
*/
public static function write($file_name, $arrays, $title, $made_by,
$parts = null)
{
$handle = fopen($file_name, "wb");
if ($handle === false) {
return;
}
$out = "<" . "?php\n" . <<< EOD
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* $title
*
* This file should be generated using $made_by
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\configs;
EOD;
if ($parts !== null) {
$out .= "/**\n * The kinds of data this file carries besides " .
"the pages\n * @var string\n */\n\$" . self::PARTS .
" = \"" . implode(" ", $parts) . "\";\n";
}
fwrite($handle, $out);
$json_flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES |
JSON_INVALID_UTF8_SUBSTITUTE;
foreach ($arrays as $array_name => $array) {
[$description, $pages] = $array;
$pubdates = $array[2] ?? [];
$comments = $array[3] ?? [];
$history = $array[4] ?? null;
fwrite($handle, "/**\n * $description\n * @var array\n */\n" .
"\$$array_name = [];\n");
foreach (self::pageEntries($pages) as [$locale_tag, $page_name,
$page]) {
$page = str_replace("\r", "", $page);
/* A page holding a line PHP would read as its end,
such as a page showing this file's own form, gets
an end mark with a number added that no line of
it starts with. */
$mark = "EOD";
for ($number = 1; preg_match('/^[ \t]*' . $mark .
self::END_CHARACTERS . '/m', $page); $number++) {
$mark = "EOD_$number";
}
$out = "";
if (isset($pubdates[$locale_tag][$page_name])) {
$out .= "\$$array_name" . self::PUBDATES .
"[\"$locale_tag\"][\"$page_name\"] = " .
intval($pubdates[$locale_tag][$page_name]) .
";\n";
}
if (!empty($comments[$locale_tag][$page_name])) {
/* JSON written on one line starts with a bracket,
so it never holds a line read as its end. */
$out .= "\$$array_name" . self::COMMENTS .
"[\"$locale_tag\"][\"$page_name\"] = " .
"<<< 'EOD'\n" . json_encode(
$comments[$locale_tag][$page_name],
$json_flags) . "\nEOD;\n";
}
$revisions = ($history === null) ? [] :
$history($locale_tag, $page_name);
if (!empty($revisions)) {
$out .= "\$$array_name" . self::HISTORY .
"[\"$locale_tag\"][\"$page_name\"] = " .
"<<< 'EOD'\n" . json_encode($revisions,
$json_flags) . "\nEOD;\n";
}
fwrite($handle, $out . "\$$array_name" .
"[\"$locale_tag\"][\"$page_name\"] = " .
"<<< '$mark'\n$page\n$mark;\n");
}
}
fclose($handle);
}
/**
* pageEntries hands out the pages of one array given to write as
* [locale tag, page name, text] triples, in order. The pages may be a
* nested array, keyed by locale tag and then by page name, or a
* generator already handing out such triples, as for a group's
* discussions, which are read one thread at a time so a large group's
* discussions are never held whole.
*
* @param iterable $pages a nested array of pages, or a generator of
* [locale tag, page name, text] triples
* @return \Generator yields [locale tag, page name, text] for each page
*/
private static function pageEntries($pages)
{
if ($pages instanceof \Generator) {
yield from $pages;
return;
}
foreach ($pages as $locale_tag => $locale_pages) {
foreach ($locale_pages as $page_name => $page) {
yield [$locale_tag, $page_name, $page];
}
}
}
}
- Yioop Repo - Seekquarry