<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\UnitTest;
/**
* PhraseParserTest tests the PhraseParser class: the words and phrases it
* extracts from text, its safe search score, its character n-grams and
* its Chinese segmenting.
*
* @author Chris Pollett
*/
class PhraseParserTest extends UnitTest
{
/**
* setUp does nothing, since PhraseParser's methods are static and
* keep no state between test cases.
*/
public function setUp()
{
}
/**
* tearDown does nothing, since setUp makes nothing to remove.
*/
public function tearDown()
{
}
/**
* extractPhrasesTestCase checks that extractPhrasesInLists finds the
* words of hard cases: acronyms, phrases with an ampersand, and urls.
*/
public function extractPhrasesTestCase()
{
$phrase_string = <<< EOD
Dr. T.Y Lin's home page. J. R. R. Tolkien
EOD;
$extracted_data = PhraseParser::extractPhrasesInLists($phrase_string,
"en-US");
$word_lists = $extracted_data['WORD_LIST'];
$words = array_keys($word_lists);
$this->assertTrue(in_array("dr", $words), "Abbreviation 1");
$this->assertTrue(in_array("_ty", $words), "Initials 1");
$this->assertTrue(in_array("_jrr", $words), "Initials 2");
$phrase_string = <<< EOD
THE THE
‘Deep Space nine’ ‘Deep Space’ version of GIANT the the
©2012
reddit: the front page of the internet prime minister
EOD;
$extracted_data = PhraseParser::extractPhrasesInLists($phrase_string,
"en-US");
$word_lists = $extracted_data['WORD_LIST'];
$words = array_keys($word_lists);
$this->assertTrue(in_array("prime-minist", $words),
"Extract Entity 1");
$this->assertTrue(in_array("prime", $words),
"Extract Entity PART 1");
$this->assertTrue(in_array("minist", $words),
"Extract Entity PART 2");
$this->assertTrue(in_array("deep", $words), "Unigrams still present 1");
$this->assertTrue(in_array("space", $words),
"Unigrams still present 2");
$this->assertTrue(in_array("2012", $words), "Punctuation removal 1");
$phrase_string = <<< EOD
百度一下，你就知道
.. 知 道 MP3 图 片 视 频 地 图 输入法 手写
拼音 关闭 空间 百科 hao123 | 更多>>
About Baidu
EOD;
$extracted_data = PhraseParser::extractPhrasesInLists($phrase_string,
"zh-CN");
$word_lists = $extracted_data['WORD_LIST'];
$words = array_keys($word_lists);
$this->assertTrue(in_array("百科", $words), "Chinese test 1");
$this->assertTrue(in_array("baidu", $words), "Chinese test 2");
$this->assertTrue(in_array("about", $words), "Chinese test 3");
$this->assertFalse(in_array("", $words), "Chinese test 4");
$this->assertFalse(in_array("下，", $words), "Chinese test 5");
$phrase_string = <<< EOD
P.O. Box 765, http://somewhere.edu.au
negative) results. bigram/trigram
Simon & Somebody (1880b) analysed roughly
the U.K. based newspaper,
15, 2006, from http://www.yo.org/index.pl?a=b&c=d
http://yo.lo.edu/faculty_pages/zebra/
A&W a&TT chris@pollett.org PG & E
Fish 'n chips
Hawaii's national parks service
EOD;
$extracted_data = PhraseParser::extractPhrasesInLists($phrase_string,
"en-US");
$word_lists = $extracted_data['WORD_LIST'];
$words = array_keys($word_lists);
$this->assertTrue(in_array("_po", $words), "Acronym Test 1");
$this->assertTrue(in_array("_uk", $words), "Acronym Test 2");
$this->assertTrue(in_array("a_and_w", $words), "Ampersand Test 1");
$this->assertTrue(in_array("a_and_tt", $words), "Ampersand Test 2");
$this->assertTrue(in_array("fish_and_chips", $words), "n for and test");
$this->assertTrue(in_array("chris_at_pollett_d_org", $words),
"Email Check 1");
$this->assertTrue(in_array(
"http_c__s__s_www_d_yo_d_org_s_index_d_pl_q_a_e_b_and_c_e_d",
$words), "URL Check 1");
$this->assertTrue(in_array(
"http_c__s__s_yo_d_lo_d_edu_s_faculty_pages_s_zebra_s_",
$words), "URL Check 2");
}
/**
* computeSafeSearchScoreTestCase checks that one score threshold
* separates adult sites from other sites. Samples were taken from a
* couple of adult sites, sorted alphabetically by word, and then some
* of the non sensitive words
* were substituted so as to avoid copyright issues. For the safe tests
* a similar process was done with the Wizard of Oz (now public domain)
* and with some sexually related Wikipedia articles (Creative Commons SA).
*/
public function computeSafeSearchScoreTestCase()
{
$phrase_string = <<< EOD
a a a a a a a a a a a a all and and
and and and and and another any arose at aunt aunt be bed bed beds big
build building by by called carried case cellar cellar chairs contained
cookstove corner corner could crush cupboard cyclone dark dishes door
dorothy dorothy down dug em em enough except family farmer farmer's floor
floor for for four four from garret go great great ground had had henry
henry hole hole house in in in in in in in into it it its kansas ladder led
little lived looking lumber made many middle midst mighty miles no no of of
of one one one or path prairies reached roof room room rusty small small
small table the the the the the the the the the the the their there there
this those three to to to trap uncle uncle wagon walls was was was was was
were where which which whirlwinds who who wife with
EOD;
$score = PhraseParser::computeSafeSearchScore($phrase_string);
$this->assertTrue(($score < PhraseParser::SAFE_PHRASE_THRESHOLD),
"Easy Safe Test 1");
$phrase_string = <<< EOD
a afraid all and anon baby big boobs but cock crave dicking does
for from grown has how in is isnt knot lolita matts monster pussies ready
she she shew slut teens their thom them thought they're tight to to to total
up use whether
EOD;
$score = PhraseParser::computeSafeSearchScore($phrase_string);
$this->assertTrue(($score > PhraseParser::SAFE_PHRASE_THRESHOLD),
"Easy Unsafe Test 1");
$phrase_string = <<< EOD
a a a a a adventure after all alotta amazing and and and and and
and and and and and around as ball ball big body boobies bounce boy
brunhilda came check check chilled cirque do enjoy ensued exercises
flap friends fucking fucking give going gorge got got grabbing had
had had has he hell her her horny i if in it it it it it jog junk
just know little little loved me mean melons melons my my of on out out
ploy precious kitties see she she she sought sizzle so so spent spicy
started stretch sucking swinging that that that the the the then things
those those those tit titties titty to to to togo today tramp truly
us was we we we what what when what wild with with with workout wrap yes
you
EOD;
$score = PhraseParser::computeSafeSearchScore($phrase_string);
$this->assertTrue(($score > PhraseParser::SAFE_PHRASE_THRESHOLD),
"Harder Unsafe Test 1");
$phrase_string = <<< EOD
amino hog known a a a a an and and
and and are are as as asymmetry be biology both but can cases cells
combining combining contain deem distance each early evolved exist
female female for firm firm from function gametes gametes gametes gametes
genetic genentech has ideal in in in in information disinherit into intone
is isopod known large mole mole many mixing motile motile necessary
non nutrients of of of of offspring often optimized or organism organisms
over parents process reproduce reproduce result sex sex sexual
sexual small specialist specialized specific such that that the the the
the their to to traits traits transport two types variety while young
EOD;
$score = PhraseParser::computeSafeSearchScore($phrase_string);
$this->assertTrue(($score < PhraseParser::SAFE_PHRASE_THRESHOLD),
"Harder Safe Test 1");
$phrase_string = <<< EOD
a a active adverb an an and are as as as attribute be
between by caught characterized daft describe describe desire desire deft
french female female females having homosexuality identify in is language
lesbian may moist verb object of of or or or others secondary refer relay
romantic same sex sexual trim the the the them to to to to to used
used who who wide women ward
EOD;
$score = PhraseParser::computeSafeSearchScore($phrase_string);
$this->assertTrue(($score < PhraseParser::SAFE_PHRASE_THRESHOLD),
"Harder Safe Test 2");
}
/**
* computeCharGramTestCase checks the character n-grams of a word, and
* that an English word, which is stemmed instead, gets none.
*/
public function computeCharGramTestCase()
{
$n_grams = PhraseParser::getNGramsTerm(["orange"], 6);
$this->assertTrue((count($n_grams) == 1), "NGram Test 1");
$three_grams = ['ora', 'ran', 'ang', 'nge'];
$n_grams = PhraseParser::getNGramsTerm(["orange"], 3);
$this->assertTrue((count($three_grams) == 4), "NGram Test 2");
$this->assertTrue(array_diff($three_grams, $n_grams) == [],
"NGram Test 3");
$n_grams = PhraseParser::getCharGramsTerm(["orange"], 'en-US');
$this->assertTrue($n_grams == [],
"NGram Test 4");
}
/**
* computeSegmentTestCase checks that a Chinese string is split into
* the right words, with and without a question mark at its end.
*/
public function computeSegmentTestCase()
{
$segments = PhraseParser::segmentSegment("你们好吗", 'zh-CN');
$correct_segments = ["你们", "好", "吗"];
$this->assertTrue((count($segments) == 3), "Segmenter Test 1");
$this->assertTrue(array_diff($segments, $correct_segments) == [],
"Segmenter Test 2");
$segments = PhraseParser::segmentSegment("你们好吗?", 'zh-CN');
$correct_segments = ["你们", "好", "吗"];
$this->assertTrue((count($segments) == 3), "Segmenter Test 3");
$this->assertTrue(array_diff($segments, $correct_segments) == [],
"Segmenter Test 4");
}
/**
* wordsFiledWithoutDiacriticsTestCase checks that a word with a
* diacritic mark, an accent or other sign on a letter, is filed both
* as written and without the mark, at the same positions. It also
* checks that marks on Hindi letters are kept, and that every letter
* of LATIN_DIACRITIC_LETTERS folds to its base letter.
*/
public function wordsFiledWithoutDiacriticsTestCase()
{
$lists = PhraseParser::extractPhrasesInLists(
"San José State University", "en-US");
$words = $lists["WORD_LIST"];
$this->assertTrue(isset($words["josé"]) && isset($words["jose"]),
"josé is filed both with and without its accent");
$this->assertEqual($words["jose"] ?? [], $words["josé"] ?? [-1],
"both forms have the same positions");
$this->assertEqual(PhraseParser::foldDiacritics("नमस्ते"), "नमस्ते",
"marks on Hindi letters are kept");
$same = 0;
foreach (PhraseParser::LATIN_DIACRITIC_LETTERS as $letter => $base) {
if (PhraseParser::foldDiacritics($letter) == $base) {
$same++;
}
}
$this->assertEqual($same,
count(PhraseParser::LATIN_DIACRITIC_LETTERS),
"every letter of the table folds to its base letter");
}
/**
* hyphensAndOwnershipSignsTestCase checks the terms made from the
* title of www.coca-cola.com, "Explore the Coca‑Cola® Official Site",
* whose hyphen is the non-breaking hyphen U+2011 and whose name
* carries the registered sign. Its name must be filed as coca-cola,
* coca and cola, the terms a search for coca-cola looks up, and no
* term may keep the registered sign or the other hyphen.
*/
public function hyphensAndOwnershipSignsTestCase()
{
$words = PhraseParser::extractPhrasesInLists(
"Explore the Coca\u{2011}Cola\u{00AE} Official Site Pepsi\u{2122}",
"en-US")["WORD_LIST"];
foreach (["coca-cola", "coca", "cola", "pepsi"] as $term) {
$this->assertTrue(isset($words[$term]), "$term is a term");
}
$kept_signs = 0;
foreach (array_keys($words) as $term) {
if (preg_match("/[\u{2011}" . PhraseParser::OWNERSHIP_SIGNS .
"]/u", $term)) {
$kept_signs++;
}
}
$this->assertEqual($kept_signs, 0,
"no term keeps a sign or the non-breaking hyphen");
}
/**
* entityWordsStemmedOnceTestCase checks that the words of an entity
* are stemmed once. A page with "San Jose State University" has the
* entity san-jose-state-univers. Its last word must be filed as
* "univers", the stem a query for "university" looks up.
*/
public function entityWordsStemmedOnceTestCase()
{
$lists = PhraseParser::extractPhrasesInLists(
"San Jose State University", "en-US");
$words = array_keys($lists["WORD_LIST"]);
$this->assertTrue(in_array("san-jose-state-univers", $words),
"the title is filed as an entity");
$this->assertTrue(in_array("univers", $words),
"the entity's last word is filed with the query's stem");
$this->assertFalse(in_array("univ", $words),
"the entity's last word is not stemmed twice");
}
/**
* accentedNameFiledAsEntityTestCase checks that San José State
* University, whose name the list of names holds only without the
* accent, is still filed as an entity, both as written and without
* the accent, as San Jose State University is.
*/
public function accentedNameFiledAsEntityTestCase()
{
$lists = PhraseParser::extractPhrasesInLists(
"San José State University", "en-US");
$words = array_keys($lists["WORD_LIST"]);
$this->assertTrue(in_array("san-josé-state-univers", $words),
"the name is filed as an entity as written");
$this->assertTrue(in_array("san-jose-state-univers", $words),
"the name is filed as an entity without the accent");
}
/**
* pageTaggedFrenchGetsFrenchEntitiesTestCase checks that a page whose
* language tag is fr has its French names filed as entities. The
* list of French names is in the fr_FR locale folder, and a tag of fr
* looked only for a folder named fr, so union européenne, which the
* list holds, was never filed as one.
*/
public function pageTaggedFrenchGetsFrenchEntitiesTestCase()
{
$lists = PhraseParser::extractPhrasesInLists(
"Le traité de l union européenne", "fr");
$entities = array_filter(array_keys($lists["WORD_LIST"]),
fn($word) => str_starts_with($word, "union-"));
$this->assertTrue(count($entities) > 0,
"union européenne is filed as an entity");
}
/**
* questionAnswersAreThoseOfAllTheListsTestCase checks that
* extractPhrasesInLists asked for question answers only gives the same
* question answer triplets it gives with all its lists, for English
* and Chinese text, for text with no words and for empty text, and
* none for French, whose tokenizer cannot tag parts of speech.
*/
public function questionAnswersAreThoseOfAllTheListsTestCase()
{
$texts = [["Who is Bob? Bob is a cat. The cat sat on the mat " .
"near San Jose State University.", "en"],
["Yioop is a search engine. Chris Pollett wrote Yioop in " .
"PHP. It crawls the web and indexes pages.", "en"],
["韩国政府宣布了新的文化交流计划。首尔是韩国的首都。", "zh-CN"],
["!!! ?? ...", "en"], ["", "en"],
["Le chat est sur la table. Qui est Bob?", "fr"]];
foreach ($texts as list($text, $lang)) {
$lists = PhraseParser::extractPhrasesInLists($text, $lang);
$this->assertEqual(PhraseParser::extractPhrasesInLists($text,
$lang, true)['QUESTION_ANSWER_LIST'] ?? [],
$lists['QUESTION_ANSWER_LIST'] ?? [],
"the same triplets for " . json_encode($text,
JSON_UNESCAPED_UNICODE));
}
$this->assertTrue(count(PhraseParser::extractPhrasesInLists(
$texts[0][0], "en", true)['QUESTION_ANSWER_LIST'] ?? []) > 0,
"English text has triplets");
}
/**
* sentencePositionsSplitAtSentenceEndsTestCase checks that
* sentencePositions splits text at a full stop before a letter, a
* question mark, an exclamation mark and a blank line, lower cases
* each sentence and gives the offsets where it starts, after the
* dummy sentence it puts first. A full stop between digits, as in
* 3.5, does not end a sentence.
*/
public function sentencePositionsSplitAtSentenceEndsTestCase()
{
$this->assertEqual(PhraseParser::sentencePositions(
"Bob is 3.5 feet tall. Who is Bob? Bob!\n\nBob again"),
["bob is 3.5 feet tall" => [8], "who is bob" => [30],
"bob" => [42], "bob again" => [48]],
"each sentence with where it starts");
}
/**
* redirectGetsLocationWordsOnlyTestCase checks the meta words
* calculateMetas gives the record of a redirect, whose LOCATION field
* lists the page's url and the url it redirects to. The target gets a
* location: word, so a search can find what redirects to it, but no
* info: word. An info: word names the url whose copy a record is, and
* the newest info: row is taken as that url's copy, so the redirect
* would stand in for the target's page.
*/
public function redirectGetsLocationWordsOnlyTestCase()
{
$url = "https://coke.com/";
$target = "https://coke.com/us/en";
$site = [CrawlConstants::URL => $url,
CrawlConstants::LOCATION => [$url, $target],
CrawlConstants::HTTP_CODE => 301];
$metas = PhraseParser::calculateMetas($site);
$this->assertEqual([in_array("location:$target", $metas),
in_array("info:$target", $metas),
in_array("info:" . L\crawlHash($target), $metas),
in_array("info:$url", $metas)], [true, false, false, true],
"the target gets a location: word and no info: word");
}
}
- Yioop Repo - Seekquarry