- Yioop Repo - Seekquarry

<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\tests;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\UnitTest;

/**
 * EnTokenizerTest checks the English tokenizer: its stemmer, which cuts
 * words to their stems by the original Porter algorithm (not Porter 2),
 * and its question answering. The stemmer's words and expected stems are
 * the lists at http://snowball.tartarus.org/algorithms/porter/voc.txt and
 * http://snowball.tartarus.org/algorithms/porter/output.txt.
 *
 * @author Chris Pollett
 */
class EnTokenizerTest extends UnitTest
{
    /**
     * setUp gets the English tokenizer the cases share.
     */
    public function setUp()
    {
        $this->test_objects['FILE1'] = PhraseParser::getTokenizer("en-US");
    }
    /**
     * tearDown does nothing, since a case leaves nothing behind.
     */
    public function tearDown()
    {
    }
    /**
     * stemmerTestCase checks that the tokenizer's stem method gives, for
     * each word of tests/test_files/english_stemmer's word list, the stem
     * on the same line of its list of stems. It also checks that a
     * hyphenated word has only its last part stemmed.
     */
    public function stemmerTestCase()
    {
        $stem_dir = C\PARENT_DIR.'/tests/test_files/english_stemmer';
        $test_words = file("$stem_dir/input_vocabulary.txt");
        /* the stems the snowball page gives for $test_words, line for
           line, which stem must match */
        $stem_words = file("$stem_dir/stemmed_result.txt");
        $tokenizer = $this->test_objects['FILE1'];
        $unstemmed_words = $tokenizer::$no_stem_list ?? [];
        for ($i = 0; $i < count($test_words); $i++) {
            $word = trim($test_words[$i]);
            if (in_array($word, $unstemmed_words) ||
                strlen($word) < 3) {
                continue;
            }
            $stem = trim($stem_words[$i]);
            $word_stem = $tokenizer->stem($word);
            $this->assertEqual($word_stem, $stem, "function stem correctly " .
                "stems $word to $stem, stems to $word_stem instead");
        }
        $this->assertEqual($tokenizer->stem("well-running"), "well-run",
            "a hyphenated word has only its last part stemmed");
    }
    /**
     * questionAnswerTestCase checks the question answering of the English
     * tokenizer: that it takes [subject verb object] triplets, stemmed,
     * from sentences, and uses them to answer questions. The tagging it
     * rests on reads the shipped lexicon, so a site with no locale folder
     * of its own must still find it. A missing answer is checked as null,
     * so a failure prints no warning of its own.
     */
    public function questionAnswerTestCase()
    {
        $tokenizer = $this->test_objects['FILE1'];
        $phrases["seekquarry is the parent site for yioop."] = 0;
        $phrases["yioop is a gplv3, open source, php search engine."] = 7;
        $triplets = $tokenizer->extractTripletsPhrases($phrases);
        $this->assertTrue(isset($triplets['QUESTION_LIST'][
            'seekquarri qqq site']), "Triplet questions contains:".
            " [seekquarri qqq site]");
        $this->assertEqual($triplets['QUESTION_ANSWER_LIST'][
            'seekquarri is qqq'] ?? null, "the parent site for yioop",
            "Triplet question answers show the answer to: ".
            " [seekquarri is qqq] is: the parent site for yioop");
        $this->assertEqual($triplets['QUESTION_ANSWER_LIST'][
            'the parent site for yioop is qqq'] ?? null, "seekquarry",
            "Triplet question answers show the answer to: ".
            " [the parent site for yioop is qqq] is: seekquarry");
    }
}