<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* GroupWikiTool is used to manage the integrity of resource folders for
* wiki pages of Yioop Groups.
*
* A description of its usage is given in the $usage global variable
*
*
* @author Chris Pollett
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\configs;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\version_control\VersionManager;
use seekquarry\yioop\models\Model;
use seekquarry\yioop\models\FeedModel;
use seekquarry\yioop\models\WikiModel;
use seekquarry\yioop\models\GroupModel;
use seekquarry\yioop\models\UserModel;
use seekquarry\yioop\controllers\GroupController;
use seekquarry\yioop\library\wiki\DiscussionImport;
use seekquarry\yioop\library\wiki as LW;
if (php_sapi_name() != 'cli' ||
defined("seekquarry\\yioop\\configs\\IS_OWN_WEB_SERVER")) {
echo "BAD REQUEST"; exit();
}
/** Loads common utility functions*/
require_once __DIR__."/../library/Utility.php";
require_once __DIR__."/../library/LocaleFunctions.php";
ini_set("memory_limit", C\ARC_TOOL_MEMORY_LIMIT);
if (!C\PROFILE) {
echo "Please configure the search engine instance by visiting" .
"its web interface on localhost.\n";
exit();
}
/**
* Used to print out a description of how to use GroupWikiTool.php
* @var string
*/
$usage = <<<EOD
GroupWikiTool.php
==============
GroupWikiTool is used to manage the integrity of resource folders for
wiki pages of Yioop Groups. The tool has a command to look up what is
the path for a wiki page of a given group for a locale. To maintain
previous version of wiki page resources, Yioop writes a .archive folder
in the wiki page's resource folder and uses it to maintain
previous versions of this folder. Before changes to the .archive
folder are made, a LOCK file is written. In the event of a crash before
completion of an operation, this LOCK file might be present and
prevent further changes to the resources in a wiki. This tool let's one
clear this lock. It also allows one to remove the existing .archive folder
and rebuild it from scratch. Finally, it allows a user to save a new version
snapshot of the current resource folder.
Usage
=====
php GroupWikiTool.php command folder
php GroupWikiTool.php clean
deletes from WORK_DIRECTORY/app/resources the group, page, post and
user folders that no group, wiki page, post or user of the site uses
any more, such as those of deleted groups, and the folders there
holding no file but .DS_Store, Thumbs.db or desktop.ini, and lists
what it deleted. Only folders named by a group, page, post or user
name are looked at, so the site, domains and needs_thumbs folders are
kept.
php GroupWikiTool.php clear-lock folder
if folder is the name of a Group Wiki page resource folder, then this
operation will remove any LOCK file on the .archive folder
php GroupWikiTool.php export group_name file_name [pages|discussions|all]
[locale_tag]
saves the Yioop group group_name to file_name, so that import with type
yioop, on this or another Yioop site, can make the same group content.
pages saves the group's wiki pages: each page's text and date, its
earlier versions with who made them, its comments, and the issues of a
git repository page. discussions saves the group's discussion threads
with all their posts. all, the default, saves both. With locale_tag,
only the pages in that locale are saved, such as en-US.
php GroupWikiTool.php import type group_name locale_tag wiki_path from_base_url
imports pages from the wiki system of type, dokuwiki or yioop, to the
Yioop group given by group_name and to the local locale_tag.
For dokuwiki, wiki_path is the file system path to the wiki system to be
imported and from_base_url is the url that the wiki_system was
originally at.
For yioop, wiki_path is a file saved by export or by ConfigureTool.php
export-public-help, and from_base_url is not used. Only the pages of
locale_tag are imported, or those of every locale if locale_tag is all.
A page whose text differs from the group's is written as a new version
with the date it had, and its earlier versions and comments are made the
same as the file's where the file has them; the rest are left as they
are. Each discussion thread in the file is added unless the group
already has it, a thread whose first post has the same time, title and
text; then only the replies it lacks are added, so importing a file twice
adds each thread once.
php GroupWikiTool.php import-discussions type ...
imports a discussion board into Yioop; type names the board. With type
mantis dump_file group_name page_name locale_tag, imports a MantisBT bug
tracker (a mysqldump --no-create-info) into the git repository wiki page
page_name's issue tracker: each bug becomes an issue, each note a comment,
and its users are matched or created. With type phpbb, googlegroup, or
phorum feed_file group_name, imports a forum or mailing-list feed saved as
RSS or Atom into group_name's discussion; the board is read from the feed.
php GroupWikiTool.php path group_name page_name locale_tag
returns the resource and thumb folders for the given group, page,
and locale.
php GroupWikiTool.php reset folder
if folder is the name of a Group Wiki page
resource folder, then this will delete the current .archive folder and replace
it with a freshly computed one
php GroupWikiTool.php version folder
if folder is the name of a Group Wiki page resource folder, then this
will save a save a new version snapshot to the .archive subfolder
EOD;
if (empty($argv[2])) {
$argv[2] = getcwd();
}
$num_args = count($argv);
if ( $num_args < 3 ) {
echo $usage;
exit();
}
switch ($argv[1]) {
case "clean":
$removed = (new GroupModel())->removeUnusedResourceFolders();
foreach ($removed as $path) {
echo "$path\n";
}
echo count($removed) . " folders deleted from " . C\APP_DIR .
"/resources\n";
break;
case "clear-lock":
$lock_file = $argv[2] . "/.archive/LOCK";
if (file_exists($argv[2])) {
unlink($lock_file);
echo "Group Wiki Page Resource Lock file removed!";
}
break;
case "export":
if (empty($argv[3])) {
echo $usage;
exit();
}
$what = "all";
$export_locale = $argv[4] ?? "";
if (in_array($export_locale, ["pages", "discussions", "all"])) {
$what = $export_locale;
$export_locale = $argv[5] ?? "";
}
exportYioopPages($argv[2], $argv[3], $what, $export_locale);
break;
case "import":
if (empty($argv[5])) {
echo $usage;
exit();
}
if (empty($argv[6])) {
$argv[6] = "";
}
list(,, $type, $group_name, $locale_tag, $wiki_path,
$from_base_url) = $argv;
if ($type == "yioop") {
importYioopPages($group_name, $locale_tag, $wiki_path);
break;
}
if ($type != "dokuwiki") {
echo "Only importing from Dokuwiki or Yioop is supported!\n";
}
importWiki($type, $group_name, $locale_tag, $wiki_path, $from_base_url);
break;
case "import-discussions":
$import_type = $argv[2] ?? "";
if ($import_type == "mantis") {
if (empty($argv[6])) {
echo $usage;
exit();
}
list(, , , $sql_path, $group_name, $page_name, $locale_tag) =
$argv;
importMantisIssues($sql_path, $group_name, $page_name,
$locale_tag);
} else if (in_array($import_type,
["phpbb", "googlegroup", "phorum"])) {
if (empty($argv[4])) {
echo $usage;
exit();
}
list(, , , $feed_file, $group_name) = $argv;
importDiscussionFeed($feed_file, $group_name);
} else {
echo $usage;
exit();
}
break;
case "path":
if (empty($argv[4])) {
$argv[4] = C\DEFAULT_LOCALE;
}
if (empty($argv[3])) {
$argv[3] = "Main";
}
if (empty($argv[2])) {
$argv[2] = "Public";
}
$group_model = new GroupModel();
$wiki_model = new WikiModel();
$group_id = $group_model->getGroupId($argv[2]);
if (!$group_id) {
echo "Could not locate that group name!!\n\n";
exit();
}
$page_id = $wiki_model->getPageId($group_id, $argv[3], $argv[4]);
if (!$page_id) {
echo "Could not locate that page name in {$argv[2]} wiki!!\n\n";
exit();
}
$folders =
$wiki_model->getGroupPageResourcesFolders($group_id, $page_id);
if (empty($folders[1])) {
echo "{$argv[3]} page folders not yet created!!\n\n";
echo $usage;
exit();
}
echo "Resource folder: {$folders[0]}\nThumb folder: {$folders[1]}\n";
break;
case "reset":
if (file_exists($argv[2] . "/.archive")) {
$model = new Model();
$db = $model->db;
$db->unlinkRecursive($argv[2] . "/.archive");
}
$vcs = new VersionManager($argv[2]);
break;
case "version":
$vcs = new VersionManager($argv[2]);
$vcs->createVersion();
break;
default:
echo $usage;
exit();
}
/**
* exportYioopPages writes a Yioop group's wiki pages, its discussions, or
* both to a file that importYioopPages, on this or another site, reads
* back to make the same pages and threads. The export command calls it.
* A group's wiki pages are written with each page's date, every earlier
* revision with who saved it, the posts of the page's own discussion
* thread, and the pages a git repository page keeps its issues in. A
* group's discussions are the threads of its discussion that are not a
* wiki page's thread, each with all its posts. Writers are named by user
* name, since a user's id differs from one site to another. The file is
* written as it is made, a page or a thread at a time.
*
* @param string $group_name the group whose pages or discussions to write
* @param string $file_name the file to write them to
* @param string $what pages for the wiki pages, discussions for the
* discussions, or all for both
* @param string $locale_tag only pages in this locale; empty for all
*/
function exportYioopPages($group_name, $file_name, $what = "all",
$locale_tag = "")
{
$group_model = new GroupModel();
$group_id = $group_model->getGroupId($group_name);
if ($group_id < 0) {
echo "Group $group_name does not exist in this Yioop instance!\n";
return;
}
$wiki_model = new WikiModel();
$feed_model = new FeedModel();
$arrays = [];
$parts = [];
$page_count = 0;
if ($what != "discussions") {
$pubdates = [];
$pages = $wiki_model->latestGroupPages($group_id, $locale_tag,
$pubdates);
$page_count = array_sum(array_map("count", $pages));
$history = function ($page_locale, $page_name) use ($wiki_model,
$group_id) {
$page_id = $wiki_model->getPageId($group_id, $page_name,
$page_locale);
return ($page_id) ? $wiki_model->pageHistory($page_id) : [];
};
$arrays["pages"] = ["Wiki pages of the group $group_name", $pages,
$pubdates, $wiki_model->groupPageComments($group_id), $history];
$parts = ["issue_pages", "comments", "history"];
}
$thread_count = 0;
if ($what != "pages") {
$threads = (function () use ($feed_model, $group_id,
&$thread_count) {
foreach ($feed_model->groupDiscussions($group_id) as
[$thread, $posts]) {
$thread_count++;
yield ["thread", $thread, json_encode($posts,
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES |
JSON_INVALID_UTF8_SUBSTITUTE)];
}
})();
$arrays["discussions"] = ["Discussion threads of the group " .
"$group_name, each a list of its posts", $threads];
$parts[] = "discussions";
}
LW\WikiPagesFile::write($file_name, $arrays,
"Wiki Pages and Discussions of the Group $group_name",
"GroupWikiTool.php export", $parts);
echo "Wrote to $file_name from the group $group_name: wiki pages " .
"$page_count, discussion threads $thread_count\n";
}
/**
* importYioopPages writes the wiki pages and discussion threads of a file
* that exportYioopPages or ConfigureTool's export-public-help wrote into a
* Yioop group. The file's pages array is used, as exportYioopPages writes
* it; a file with none, as export-public-help writes, gives its help_pages
* to the Help group and its public_pages to any other group. A page that
* is not in the group is made, and a page whose text differs is given the
* file's text as a new revision, with the date the file gives it. Where
* the file carries a page's thread posts or its earlier revisions, the
* page's thread or history is made the same as the file's. Each thread of
* the file's discussions is added to the group's discussion, unless the
* group already has it, in which case only the replies it lacks are
* added, as FeedModel::addDiscussionThread says; so importing the same
* file twice adds each thread once. The
* file is read as text, a page at a time, and never run. The import
* command calls it with type yioop.
*
* @param string $group_name the group to write the pages into
* @param string $locale_tag only pages of this locale; all for every
* locale in the file
* @param string $file_name the file to read the pages from
*/
function importYioopPages($group_name, $locale_tag, $file_name)
{
$group_model = new GroupModel();
$wiki_model = new WikiModel();
$feed_model = new FeedModel();
$group_id = $group_model->getGroupId($group_name);
if ($group_id < 0) {
echo "Group $group_name does not exist in this Yioop instance!\n";
return;
}
$page_arrays = ["pages", ($group_id == C\HELP_GROUP_ID) ?
"help_pages" : "public_pages"];
$default_locale = L\getLocaleTag();
$current_locale = "";
$written = 0;
$same = 0;
$threads = 0;
$threads_there = 0;
$posts_added = 0;
foreach (LW\WikiPagesFile::eachPage($file_name) as [$array_name,
$page_locale, $page_name, $page, $pubdate, $comments, $history]) {
if ($array_name == "discussions") {
$posts = json_decode($page, true);
if (is_array($posts)) {
[$thread, $is_new, $added] =
$feed_model->addDiscussionThread($group_id, $posts);
if ($is_new && $thread > 0) {
$threads++;
} elseif ($thread > 0) {
$threads_there++;
$posts_added += $added;
}
}
continue;
}
if (!in_array($array_name, $page_arrays) ||
($locale_tag != "all" && $page_locale != $locale_tag)) {
continue;
}
if ($page_locale !== $current_locale) {
L\setLocaleObject($page_locale);
$current_locale = $page_locale;
}
$page_info = $wiki_model->getPageInfoByName($group_id,
$page_name, $page_locale, "edit");
if (!empty($page_info) && $page_info['PAGE'] == $page) {
$same++;
} else {
$wiki_model->setPageName(C\ROOT_ID, $group_id, $page_name,
$page, $page_locale, "Yioop import",
L\tl('social_component_page_created', $page_name),
L\tl('social_component_page_discuss_here'),
WikiModel::WIKI_LINK_BASE, true, $pubdate);
$written++;
if ($history !== null) {
$wiki_model->setPageHistory($group_id, $page_name,
$page_locale, $history);
}
}
if ($comments !== null) {
$wiki_model->setPageComments($group_id, $page_name,
$page_locale, $comments);
}
}
L\setLocaleObject($default_locale);
echo "Into the group $group_name: wiki pages written $written, wiki " .
"pages already the same $same, discussion threads added $threads, " .
"discussion threads already there $threads_there, with replies " .
"added to them $posts_added\n";
}
/**
* Imports an external wiki tree into the named Yioop group.
*
* @param string $type external-wiki format; currently only "dokuwiki"
* is supported
* @param string $group_name target Yioop group the imported pages
* will live under
* @param string $locale_tag IANA locale tag the imported pages are
* tagged with (e.g. "en-US")
* @param string $wiki_path filesystem path to the root of the source
* wiki's pages folder
* @param string $from_base_url base URL of the source wiki used to
* rewrite inline links; defaults to empty (no rewriting)
*/
function importWiki($type, $group_name, $locale_tag, $wiki_path,
$from_base_url = "")
{
if ($type != 'dokuwiki') {
echo "$type is an unknown wiki type\n";
return;
}
$group_model = new GroupModel();
$wiki_model = new WikiModel();
$group_id = $group_model->getGroupId($group_name);
if ($group_id == -1) {
echo "Group $group_name does not exist in this Yioop instance!\n";
return;
}
$class_or_id = '0-9a-zA-Z\_\-\s\.\,\:\;\"\[\/\%\?\^' .
"\]\{\}\(\)\!\|।\&\`\’\‘©®™℠…\/\>，。）：、" .
"”“《》（「」★【】·；！—―？！،؛؞؟٪٬٭\‚\‘";
$doku_to_internals = [
["/^\s*======([$class_or_id]+)======/su", 'ZZH1ZZ$1ZZH1ZZ'],
["/(\A|\n)\s*======([$class_or_id]+)======/su", '\nZZH1ZZ$2ZZH1ZZ'],
["/(\A|\n)\s*=====([$class_or_id]+)=====/su", '\nZZH2ZZ$2ZZH2ZZ'],
["/(\A|\n)\s*====([$class_or_id]+)====/su", '\nZZH3ZZ$2ZZH3ZZ'],
["/(\A|\n)\s*===([$class_or_id]+)===/su", '\nZZH4ZZ$2ZZH4ZZ'],
["/(\A|\n)\s*==([$class_or_id]+)==/su", '\nZZH5ZZ$2ZZH5ZZ'],
["/(\A|\s|\n)\s*\*\*\s*([$class_or_id]+)\s*\*\*/su",
'$1\'\'\'$2\'\'\''],
["@(\A|\s)//([$class_or_id]+)//@su", '$1\'\'$2\'\''],
["/(\A|\s)\_\_([$class_or_id]+)\_\_/su", '$1<u>$2<u>'],
["/(\A|\n)\s*\-/su", "$1#"],
["/(\A|\n)\s*\*/u", "$1*"],
["/\[(\s*[^\|]+)\s+\|([^\]]+)\]/su", "[$1|$2]"],
["/\[([^\|]+)\|\s+([^\]]+)\]/su", "[$1|$2]"],
["/\{\{\s*\:?([a-zA-z]*)(\:)([^\|]+)\|([^\}\{]+)\}\}/su",
'((resource-link:media:$3|$1|$4))'],
['/\/\/$/', "<br>\n"],
['/(\{|\[)\s+/', "$1"],
['/\s+(\}|\])/', "$1"],
['/(\A|\n)\s(\{|\[)/s', "$1$2"],
['/(\A|\n)(\*+)\s+/s', "$1$2"],
['/(\A|\n)(\#+)\s+/s', "$1$2"],
];
$internal_to_yioops = [
["/ZZH1ZZ\s*([$class_or_id]+)\s*ZZH1ZZ\s*/su", '\n=$1=\n'],
["/ZZH2ZZ\s*([$class_or_id]+)\s*ZZH2ZZ\s*/su", '\n==$1==\n'],
["/ZZH3ZZ\s*([$class_or_id]+)\s*ZZH3ZZ\s*/su", '\n===$1===\n'],
["/ZZH4ZZ\s*([$class_or_id]+)\s*ZZH4ZZ\s*/su", '\n====$1====\n'],
["/ZZH5ZZ\s*([$class_or_id]+)\s*ZZH5ZZ\s*/su", '\n=====$1=====\n'],
];
$doku_matches = [];
$doku_replaces = [];
foreach ($doku_to_internals as $doku_to_internal) {
list($doku_matches[], $doku_replaces[]) = $doku_to_internal;
}
$internal_matches = [];
$internal_replaces = [];
foreach ($internal_to_yioops as $internal_to_yioop) {
list($internal_matches[], $internal_replaces[]) = $internal_to_yioop;
}
$documents = glob("$wiki_path/attic/*.txt.gz");
importWikiMedia(C\ROOT_ID, $group_id, $locale_tag, $wiki_path);
$first_start = true;
$old_original_page_name = "";
$meta_info = [];
foreach ($documents as $pre_doc) {
$document = gzdecode(file_get_contents($pre_doc));
$document = preg_replace($doku_matches, $doku_replaces, $document);
$document = preg_replace($internal_matches, $internal_replaces,
$document);
if (!empty($from_base_url)) {
$quoted_url = preg_quote($from_base_url);
$document = preg_replace(
"@\[\[$quoted_url([^\:\|\]]+)id=([^\|\]\s]+)@", "[[$2",
$document);
$document = preg_replace("@\[\[$quoted_url([^\:\|\]]+)fetch\.php\?".
"media\=([a-zA-Z]+)\:([^\:\|\]]+)\:([^\|\]]+)\|([^\]]+)\]\]@",
'((resource-link:media:$4|$2/$3|$5))',
$document);
}
$document = str_replace('\\n',"\n", $document);
$document = str_replace("'''","'''", $document);
$document = str_replace("''","''", $document);
if (preg_match('/\/([^\/]+)\.([^\/\.]+)\.txt\.gz$/', $pre_doc,
$matches)) {
list(, $original_page_name, $timestamp) = $matches;
if ($original_page_name != $old_original_page_name) {
$meta_path = "$wiki_path/meta/$original_page_name.changes";
if (file_exists($meta_path)) {
$pre_meta_info = file($meta_path);
foreach ($pre_meta_info as $pre_meta_item) {
list($meta_timestamp, $meta_edit_reason) =
explode("\t", $pre_meta_item, 2);
$meta_edit_reason = " Dokuwiki import " .
preg_replace("/\s+/", " ", trim($meta_edit_reason));
$meta_info[trim($meta_timestamp)] = $meta_edit_reason;
}
}
$old_original_page_name = $original_page_name;
}
$page_name = preg_replace('/^Main(\d*)$/', "DokuWikiMain$1",
$original_page_name);
$page_name = str_replace(" ", "_", $page_name);
if ($page_name == 'start') {
$page_name = 'Main';
if ($first_start) {
$first_start = false;
/* Main page is created by createdb with a too new timestamp
as compared to some doku wiki pages might be improting
*/
$page_id = $wiki_model->getPageId($group_id, $page_name,
$locale_tag);
if (intval($page_id) > 0) {
$db = $group_model->db;
$db->execute("DELETE FROM GROUP_PAGE
WHERE ID=$page_id");
$db->execute("DELETE FROM GROUP_PAGE_HISTORY
WHERE PAGE_ID=$page_id");
}
}
}
$edit_reason = (empty($meta_info[$timestamp])) ?
"[Wiki Import on " . time(). " ]" : $meta_info[$timestamp];
$timestamp = intval($timestamp);
if (!empty($timestamp) && !empty($page_name)) {
echo "Inserting $page_name revision from ". date("Y-m-d H:i:s",
$timestamp) ."\n!";
$wiki_model->setPageName(C\ROOT_ID,
$group_id, $page_name, $document,
$locale_tag, $edit_reason,
L\tl('social_component_page_created', $page_name),
L\tl('social_component_page_discuss_here'),
WikiModel::WIKI_LINK_BASE, pubdate: $timestamp);
} else {
echo "Skipping a revision of $page_name" .
" with timestamp $timestamp\n";
}
}
}
}
/**
* Imports wiki media (images, attachments) into the target group's
* resource folder, walking the source wiki's media tree.
*
* @param int $user_id user-id recorded as the uploader of each file
* @param int $group_id Yioop group whose wiki page is being
* populated with the imported media
* @param string $locale_tag IANA locale tag the destination wiki
* page is tagged with
* @param string $wiki_path filesystem path to the source wiki's
* root folder; media lives in "$wiki_path/media"
* @param string $sub_path sub-folder beneath
* "$wiki_path/media" to import from; empty imports the whole
* media tree
*/
function importWikiMedia($user_id, $group_id, $locale_tag, $wiki_path,
$sub_path = "")
{
$base_prefix = "$wiki_path/media/";
$media_prefix = "$base_prefix$sub_path/";
$media_paths = glob("$media_prefix*");
$len_prefix = strlen($media_prefix);
$group_model = new GroupModel();
$wiki_model = new WikiModel();
if (!($page_id = $wiki_model->getPageId($group_id, "media",
$locale_tag))) {
$media_page = LW\WikiParser::makeWikiPageHead(
['page_type' => 'media_list']) . LW\WikiParser::END_HEAD_VARS .
"media";
$page_id = $wiki_model->setPageName($user_id, $group_id, "media",
$media_page, $locale_tag, "create",
L\tl('social_component_page_created', $media_page),
L\tl('social_component_page_discuss_here'),
WikiModel::WIKI_LINK_BASE);
}
foreach ($media_paths as $media_path) {
if (is_dir($media_path)) {
$sub_subpath = substr($media_path, strlen($base_prefix));
$folders = $wiki_model->getGroupPageResourcesFolders($group_id,
$page_id, $sub_subpath, true);
echo "Importing $sub_subpath\n";
importWikiMedia($user_id, $group_id, $locale_tag, $wiki_path,
$sub_subpath);
} else {
$file_name = substr($media_path, $len_prefix);
$mime_type = L\mimeType($file_name);
$data = file_get_contents($media_path);
$wiki_model->copyFileToGroupPageResource($media_path, $file_name,
$mime_type, $group_id, $page_id, $sub_path, $data);
}
}
}
/**
* Finds the Yioop user that a MantisBT user should map to, making one when
* needed. An existing Yioop user with the same username is reused. Failing
* that, a compatible username becomes a new Yioop user carrying the
* MantisBT real name and email and a random password, so the person can
* take the account over through the usual password-reset. A name Yioop
* cannot take falls back to the shared bot-sender account. A user resolved
* to a real account is made a member of the group the repository is on, so
* they can reach the issues, unless they already belong to it. Answers are
* remembered in the cache so each MantisBT user is resolved once.
*
* @param object $user_model model used to look up and add users
* @param object $group_model model used to check and add group membership
* @param int $group_id group the repository wiki page belongs to
* @param array $mantis the MantisBT user's id, username, real name, email
* @param array &$cache map from MantisBT user id to Yioop user id so far
* @param int $bot_id Yioop user id used when a name cannot be taken
* @return array the Yioop user id the MantisBT user maps to and whether
* that meant falling back to the shared bot-sender account
*/
function resolveMantisUser($user_model, $group_model, $group_id, $mantis,
&$cache, $bot_id)
{
$mantis_id = (int)($mantis["id"] ?? 0);
if (isset($cache[$mantis_id])) {
return $cache[$mantis_id];
}
$yioop_id = $bot_id;
$fell_back = true;
$username = trim($mantis["username"] ?? "");
$existing = ($username !== "") ? $user_model->getUser($username) : false;
if ($existing) {
$yioop_id = (int)$existing["USER_ID"];
$fell_back = false;
} else if (DiscussionImport::usernameCompatible($username)) {
$realname = trim($mantis["realname"] ?? "");
$space = strpos($realname, " ");
$first = ($space === false) ? $realname :
substr($realname, 0, $space);
$last = ($space === false) ? "" : substr($realname, $space + 1);
$password = L\base64Hash(random_bytes(C\AUTH_KEY_NUM_BYTES));
$new_id = $user_model->addUser($username, $password, $first, $last,
$mantis["email"] ?? "");
if ($new_id) {
$yioop_id = (int)$new_id;
$fell_back = false;
}
}
if (!$fell_back &&
!$group_model->checkUserGroup($yioop_id, $group_id)) {
$group_model->addUserGroup($yioop_id, $group_id);
}
$cache[$mantis_id] = [$yioop_id, $fell_back];
return $cache[$mantis_id];
}
/**
* Copies each imported MantisBT file's bytes into a discussion post's own
* resource folder, the same "post" folder the issue tracker uses for a
* comment's own attachments, so a wiki resource reference to the file
* resolves when the post is read. Files with no bytes in the dump are
* skipped.
*
* @param object $group_model model used to store the resources
* @param int $group_id group the issue's page belongs to
* @param int $post_id discussion post the files belong to
* @param array $files the file rows, each with a filename, type, and bytes
*/
function storeMantisFiles($group_model, $group_id, $post_id, $files)
{
$wiki_model = $this->parent->model("wiki");
foreach ($files as $file) {
if ($file["content"] === "") {
continue;
}
$wiki_model->copyFileToGroupPageResource("", $file["filename"],
$file["file_type"], $group_id, "post" . $post_id, "",
$file["content"]);
}
}
/**
* Imports a group's discussion threads from a forum or mailing-list feed
* exported as RSS or Atom, for example from phpBB, Google Groups, or
* Phorum, into the group's own discussion. The board the feed came from is
* recognised from the feed's contents, so only the group needs naming. The
* work is handed to the social component's importDiscussions, the same code
* the Manage Groups page uses, so importing from the command line behaves
* just like importing through the web interface.
*
* @param string $feed_file file system path to the RSS or Atom feed
* @param string $group_name group to import the discussion threads into
*/
function importDiscussionFeed($feed_file, $group_name)
{
if (!file_exists($feed_file)) {
echo "Could not find the feed file $feed_file!\n";
return;
}
$group_model = new GroupModel();
$wiki_model = new WikiModel();
$group_id = $group_model->getGroupId($group_name);
if (!$group_id || $group_id == -1) {
echo "Group $group_name does not exist in this Yioop instance!\n";
return;
}
$group = $group_model->getGroupById($group_id, C\ROOT_ID);
$owner_id = $group["OWNER_ID"] ?? C\ROOT_ID;
$feed_data = file_get_contents($feed_file);
$controller = new GroupController();
$social = $controller->component("social");
$social->importDiscussions($group_id, $owner_id, $feed_data);
echo "Imported discussions into $group_name.\n";
}
/**
* Imports the issues of a MantisBT bug tracker from a mysqldump into the
* issue tracker of a git repository wiki page. The dump is expected to come
* from mysqldump --no-create-info, so it holds INSERT rows only. Each bug
* becomes an issue reported by the Yioop user its MantisBT reporter maps to
* and dated when the bug was submitted; its notes become comments dated and
* owned by the users their authors map to. MantisBT users are matched to
* Yioop users by username, created when their name allows, or folded onto
* the bot-sender account when it does not. When the tracker starts empty
* the MantisBT issue numbers are kept as the Yioop issue numbers; otherwise
* issues take the next free numbers.
*
* @param string $sql_path file system path to the mysqldump
* @param string $group_name group holding the git repository wiki page
* @param string $page_name the git repository wiki page to import into
* @param string $locale_tag language the page is written for
*/
function importMantisIssues($sql_path, $group_name, $page_name, $locale_tag)
{
if (!file_exists($sql_path)) {
echo "Could not find the dump file $sql_path!\n";
return;
}
$group_model = new GroupModel();
$wiki_model = new WikiModel();
$user_model = new UserModel();
$group_id = $group_model->getGroupId($group_name);
if (!$group_id || $group_id == -1) {
echo "Group $group_name does not exist in this Yioop instance!\n";
return;
}
if (!$wiki_model->getPageId($group_id, $page_name, $locale_tag)) {
echo "Page $page_name was not found in group $group_name!\n";
return;
}
$sql = file_get_contents($sql_path);
$mantis_users = [];
foreach (DiscussionImport::parseInserts($sql,
"mantis_user_table") as $row) {
$mantis_users[(int)$row[0]] = ["id" => (int)$row[0],
"username" => $row[1] ?? "", "realname" => $row[2] ?? "",
"email" => $row[3] ?? ""];
}
$bot = $user_model->getUser(C\MAIL_SENDER);
$bot_id = $bot ? (int)$bot["USER_ID"] : (int)$user_model->addUser(
C\MAIL_SENDER, L\base64Hash(random_bytes(C\AUTH_KEY_NUM_BYTES)));
$bug_text = [];
foreach (DiscussionImport::parseInserts($sql,
"mantis_bug_text_table") as $row) {
$bug_text[(int)$row[0]] = [$row[1] ?? "", $row[2] ?? "",
$row[3] ?? ""];
}
$note_text = [];
foreach (DiscussionImport::parseInserts($sql,
"mantis_bugnote_text_table") as $row) {
$note_text[(int)$row[0]] = $row[1] ?? "";
}
$notes_by_bug = [];
foreach (DiscussionImport::parseInserts($sql,
"mantis_bugnote_table") as $row) {
$notes_by_bug[(int)$row[1]][] = ["id" => (int)$row[0],
"reporter" => (int)$row[2],
"text" => $note_text[(int)$row[3]] ?? "", "time" => (int)$row[9]];
}
/* attachments, grouped by the bug they belong to and then by the note
they hang off, with note 0 standing for the issue itself */
$files_by_bug = [];
foreach (DiscussionImport::parseInserts($sql,
"mantis_bug_file_table") as $row) {
$files_by_bug[(int)$row[1]][(int)($row[12] ?? 0)][] = [
"filename" => $row[5] ?? "", "file_type" => $row[8] ?? "",
"content" => DiscussionImport::fileContent($row[9] ?? "")];
}
$bugs = DiscussionImport::parseInserts($sql, "mantis_bug_table");
usort($bugs, fn($first, $second) =>
(int)$first[0] - (int)$second[0]);
$empty_tracker = ($wiki_model->nextGitIssueNumber($group_id, $page_name,
$locale_tag) == 1);
$count = importMantisBugList($group_model, $user_model,
[$group_id, $page_name, $locale_tag, $bot_id, $empty_tracker],
$bugs, [$bug_text, $notes_by_bug, $mantis_users, $files_by_bug]);
echo "\nImported $count issue(s) into $page_name.\n";
}
/**
* Creates one Yioop issue for each MantisBT bug and adds its notes as
* comments, mapping every MantisBT user to a Yioop user as it goes. Split
* out from importMantisIssues so the field mapping reads on its own. The
* target argument holds, in order, the group id, page name, locale tag, the
* fallback bot user id, and whether the tracker was empty (so numbers may
* be kept). The lookups argument holds the bug-text rows, the notes grouped
* by bug, the MantisBT users, and the attachments grouped by bug and note.
*
* @param object $group_model model used to create issues and comments
* @param object $user_model model used to map and add users
* @param array $target group id, page name, locale, bot id, empty flag
* @param array $bugs the MantisBT bug rows, oldest first
* @param array $lookups bug text, notes, users, and attachments to draw on
* @return int how many issues were created
*/
function importMantisBugList($group_model, $user_model, $target, $bugs,
$lookups)
{
$wiki_model = new WikiModel();
$feed_model = new FeedModel();
list($group_id, $page_name, $locale_tag, $bot_id, $empty_tracker) =
$target;
list($bug_text, $notes_by_bug, $mantis_users, $files_by_bug) = $lookups;
/* MantisBT priority codes 10 none, 20 low, 30 normal, 40 high,
50 urgent, 60 immediate, mapped onto Yioop's low, medium, high */
$priority_to_yioop = [10 => LW\WikiIssue::PRIORITY_LOW,
20 => LW\WikiIssue::PRIORITY_LOW,
30 => LW\WikiIssue::PRIORITY_MEDIUM,
40 => LW\WikiIssue::PRIORITY_HIGH,
50 => LW\WikiIssue::PRIORITY_HIGH,
60 => LW\WikiIssue::PRIORITY_HIGH];
/* a MantisBT status at or past 80 (resolved) counts as closed */
$closed_status = 80;
/* resolution codes meaning closed unfixed: 50 not fixable,
80 suspended, 90 won't fix */
$wont_fix = [50, 80, 90];
$user_cache = [];
$count = 0;
foreach ($bugs as $bug) {
$mantis_id = (int)($bug[0] ?? 0);
$summary = $bug[21] ?? "";
$status_code = (int)($bug[8] ?? 0);
$submitted = (int)($bug[26] ?? 0);
$updated = (int)($bug[28] ?? 0);
$reporter = $mantis_users[(int)($bug[2] ?? 0)] ?? ["id" => 0];
list($reporter_id, $reporter_fell_back) = resolveMantisUser(
$user_model, $group_model, $group_id, $reporter, $user_cache,
$bot_id);
$reporter_note = $reporter_fell_back ?
DiscussionImport::fallbackByline($reporter) : "";
$body = DiscussionImport::issueBody($bug_text[(int)($bug[12] ?? 0)] ??
["", "", ""], $reporter_note);
$record = LW\WikiIssue::open($reporter_id, $submitted, $summary,
"", $bug[16] ?? "");
$record["priority"] = $priority_to_yioop[(int)($bug[5] ?? 0)] ??
LW\WikiIssue::PRIORITY_MEDIUM;
$record["last_modified"] = $updated;
$record["description"] = $body;
if ($status_code >= $closed_status) {
$resolution = in_array((int)($bug[9] ?? 0), $wont_fix) ?
LW\WikiIssue::RESOLUTION_WONT_FIX :
LW\WikiIssue::RESOLUTION_FIXED;
$record = LW\WikiIssue::close(
$record, $resolution, $reporter_id,
$updated);
} else if ((int)($bug[3] ?? 0) > 0) {
/* an open MantisBT bug that has a handler is one assigned to
that person, so map the handler to a Yioop user and assign
the issue to them; it then shows as assigned rather than as
only reported */
$handler = $mantis_users[(int)$bug[3]] ?? ["id" => 0];
list($handler_id, ) = resolveMantisUser($user_model,
$group_model, $group_id, $handler, $user_cache, $bot_id);
$record = LW\WikiIssue::assign($record, $handler_id,
$reporter_id, $updated);
}
$number = $empty_tracker ? $mantis_id : 0;
$number = $wiki_model->createGitIssue($reporter_id, $group_id,
$page_name, $record, $locale_tag, $summary, $body, $submitted,
$number);
list($thread_id, ) = $feed_model->getGitIssueThread($group_id,
$page_name, $number, $locale_tag);
$bug_files = $files_by_bug[$mantis_id] ?? [];
if ($thread_id > 0) {
foreach ($notes_by_bug[$mantis_id] ?? [] as $note) {
$author = $mantis_users[$note["reporter"]] ?? ["id" => 0];
list($author_id, $author_fell_back) = resolveMantisUser(
$user_model, $group_model, $group_id, $author,
$user_cache, $bot_id);
$text = $note["text"];
if ($author_fell_back) {
$byline = DiscussionImport::fallbackByline($author);
if ($byline !== "") {
$text = "(originally by " . $byline . ")\n\n" .
$text;
}
}
$note_files = $bug_files[$note["id"]] ?? [];
$text .= DiscussionImport::fileReferences($note_files);
$post_id = $feed_model->addGroupItem($thread_id, $group_id,
$author_id, "-- " . $summary, $text,
C\STANDARD_GROUP_ITEM, $note["time"]);
storeMantisFiles($group_model, $group_id, $post_id,
$note_files);
}
/* files hung off the bug itself rather than a note become a
closing comment, since the issue's own description does not
resolve resource references but a comment does */
$issue_files = $bug_files[0] ?? [];
if (DiscussionImport::fileReferences($issue_files) !== "") {
$post_id = $feed_model->addGroupItem($thread_id, $group_id,
$reporter_id, "-- " . $summary,
"Attachments:" .
DiscussionImport::fileReferences($issue_files),
C\STANDARD_GROUP_ITEM, $updated);
storeMantisFiles($group_model, $group_id, $post_id,
$issue_files);
}
}
$count++;
echo " issue $number <- MantisBT #$mantis_id $summary\n";
}
return $count;
}
- Yioop Repo - Seekquarry