<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\locale\zh_CN\resources\Tokenizer;
use seekquarry\yioop\library\UnitTest;
use seekquarry\yioop\library\language_processing as LLP;
/**
* ZhTokenizerTest checks the Chinese language tools: named entity tagging,
* which finds names of people and places in text, part of speech tagging,
* and turning traditional characters into simplified ones. Word
* segmentation is tested in PhraseParserTest.
*/
class ZhTokenizerTest extends UnitTest
{
/**
* Most times as long as splitting words into characters that
* normalizing them one at a time may take; it takes 8 to 16 times as
* long
*/
const WORD_NORMALIZE_RATIO = 50;
/**
* Largest share of the first segmenting's time that segmenting the same
* text again may take
*/
const SEGMENT_AGAIN_SHARE = 0.25;
/**
* setUp does nothing, since each case makes what it uses.
*/
public function setUp()
{
}
/**
* tearDown does nothing, since a case leaves nothing behind.
*/
public function tearDown()
{
}
/**
* namedEntityTestCase checks that the named entity tagger marks, in a
* Chinese sentence, a person's name with nr and a place name with ns.
*/
public function namedEntityTestCase()
{
$source = "孙向宏喜欢去洛杉矶旅游";
$expected_tagging = "孙向宏_nr 洛杉矶_ns";
$ne_tagger = new LLP\NamedEntityContextTagger(
'zh-CN');
$output_tagging = $ne_tagger->tag($source);
$this->assertEqual($output_tagging, $expected_tagging,
"Named Entities Correctly Found in Chinese Source String");
}
/**
* partOfSpeechTestCase checks that the part of speech tagger gives
* each word of a segmented Chinese sentence its expected tag, such as
* NR for a proper noun and NN for a common noun.
*/
public function partOfSpeechTestCase()
{
$source = "印度 总统 是 印度 国家元首 和 " .
"武装部队 总司令 有 该国 第一 公民 之 称";
$expected_tagging = "印度_NR 总统_NN 是_VC 印度_NR 国家元首_NN ".
"和_CC 武装部队_NN 总司令_NN 有_VE 该国_NN 第一_VV 公民_NN 之_DEG 称_NN";
$pos_tagger = new LLP\PartOfSpeechContextTagger(
'zh-CN');
$output_tagging = $pos_tagger->tag($source);
$this->assertEqual($output_tagging, $expected_tagging,
"Parts of Speech Correctly Tagged in Chinese Source String");
}
/**
* traditionalSimplifiedTestCase checks that Tokenizer::normalize turns
* traditional Chinese characters into their simplified forms. The map
* it uses ships in src/locale, so a site with no locale folder of its
* own must still find it.
*/
public function traditionalSimplifiedTestCase()
{
$traditional = "那是一個黑暗而暴風雨的夜晚。";
$simplified = "那是一个黑暗而暴风雨的夜晚。";
$this->assertEqual(Tokenizer::normalize($traditional), $simplified,
"Traditional characters correctly mapped to simplied ones");
$long = str_repeat($traditional, 100);
$this->assertEqual(Tokenizer::normalize($long),
str_repeat($simplified, 100),
"A text too long to map a character at a time is mapped too");
}
/**
* normalizeWordTimeTestCase normalizes 20,000 two character words one
* at a time, as the named entity tagger does once a word, and splits
* the same words into characters, and checks normalizing takes less
* than WORD_NORMALIZE_RATIO times as long. The faster of three rounds
* is taken, so a busy moment does not decide it. Mapping each word with
* strtr built the table of the whole map, 3,964 characters, every
* call, and took 160 to 300 times as long as the split; it was 40
* percent of the time of reading the words of a Chinese description.
*/
public function normalizeWordTimeTestCase()
{
$characters = preg_split('//u', "這是一個測試句子用來看看轉換的" .
"速度中國人民", -1, PREG_SPLIT_NO_EMPTY);
$words = [];
for ($i = 0; $i < 20000; $i++) {
$words[] = $characters[$i % 20] . $characters[($i * 7) % 20];
}
Tokenizer::normalize($words[0]);
$ratio = PHP_FLOAT_MAX;
for ($round = 0; $round < 3; $round++) {
$start = microtime(true);
foreach ($words as $word) {
mb_str_split($word);
}
$split_time = microtime(true) - $start;
$start = microtime(true);
foreach ($words as $word) {
Tokenizer::normalize($word);
}
$ratio = min($ratio, (microtime(true) - $start) /
max($split_time, 1e-6));
}
$this->assertTrue($ratio < self::WORD_NORMALIZE_RATIO,
"Normalizing words one at a time took " . round($ratio) .
" times as long as splitting them");
}
/**
* segmentAgainTimeTestCase segments a Chinese text, then the same runs
* of text in another order, as a fetcher segments a page's description
* to index it and then sentence by sentence to find its questions and
* answers. It checks the second gives the same words for each run in
* less than SEGMENT_AGAIN_SHARE of the time taken to segment new runs
* made of the same characters, so the dictionary is read as much. The
* runs took as long the second time before the segmenter kept the
* words of the runs it segmented.
*/
public function segmentAgainTimeTestCase()
{
$segmenter = Tokenizer::getStochasticTermSegmenter();
$runs = ["印度总统是印度国家元首和武装部队最高统帅",
"孙向宏喜欢去洛杉矶旅游", "那是一个黑暗而暴风雨的夜晚",
"中华人民共和国成立于一九四九年", "北京是中国的首都"];
$words = $segmenter->segmentSentence(implode(" ", $runs));
$joined_runs = [$runs[0] . $runs[1], $runs[2] . $runs[3],
$runs[4] . $runs[0]];
$start = microtime(true);
$segmenter->segmentSentence(implode(" ", $joined_runs));
$new_time = microtime(true) - $start;
$start = microtime(true);
$words_again = $segmenter->segmentSentence(implode(" ",
array_reverse($runs)));
$again_time = microtime(true) - $start;
$words_of_runs = [];
foreach (array_reverse($runs) as $run) {
$words_of_runs = array_merge($words_of_runs,
$segmenter->segmentSentence($run));
}
$this->assertEqual($words_again, $words_of_runs,
"Segmenting runs again gives the same words");
$this->assertTrue($again_time < self::SEGMENT_AGAIN_SHARE *
$new_time, "Segmenting runs again took " . round(100 *
$again_time / max($new_time, 1e-6)) . " percent of the time " .
"to segment new runs");
}
}
- Yioop Repo - Seekquarry