- Yioop Repo - Seekquarry

<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\tests;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\UnitTest;

/**
 * HiTokenizerTest checks the Hindi tokenizer: its stemmer, which cuts
 * words to their stems, and its part of speech tagging. The expected stems
 * come from the Java program the PHP stemmer is based on, at
 * http://members.unine.ch/jacques.savoy/clef/HindiStemmerLight.java.txt,
 * which has since been changed to improve its accuracy.
 *
 * @author Chris Pollett
 */
class HiTokenizerTest extends UnitTest
{
    /**
     * setUp gets the Hindi tokenizer the cases share.
     */
    public function setUp()
    {
        $this->test_objects['FILE1'] = PhraseParser::getTokenizer("hi");
    }
    /**
     * tearDown does nothing, since a case leaves nothing behind.
     */
    public function tearDown()
    {
    }
    /**
     * stemmerTestCase checks that the tokenizer's stem method gives, for
     * each word of tests/test_files/hindi_stemmer/input_vocabulary.txt,
     * the stem on the same line of stemmed_result.txt. Words on the
     * tokenizer's list of words it does not stem, and words shorter than
     * three bytes, are skipped.
     */
    public function stemmerTestCase()
    {
        $stem_dir = C\PARENT_DIR.'/tests/test_files/hindi_stemmer';
        //Test word set from snowball
        $test_words = file("$stem_dir/input_vocabulary.txt");
        //Stem word set from snowball for comparing results
        $stem_words = file("$stem_dir/stemmed_result.txt");
        $tokenizer = $this->test_objects['FILE1'];
        $unstemmed_words = $tokenizer::$no_stem_list ?? [];
        for ($i = 0; $i < count($test_words); $i++) {
            $word = trim($test_words[$i]);
            if (in_array($word, $unstemmed_words) ||
                strlen($word) < 3) {
                continue;
            }
            $stem = trim($stem_words[$i]);
            $word_stem = $tokenizer->stem($word);
            $this->assertEqual($word_stem, $stem, "function stem correctly ".
                "stems $word to $stem, stems to $word_stem instead");
        }
    }
    /**
     * lexiconTagsWordsTestCase checks that tagTokenizePartOfSpeech gives a
     * word the part of speech the shipped lexicon lists for it: CC, a
     * conjunction, for और (and), and JJ, an adjective, for सुंदर
     * (beautiful). The lexicon ships in src/locale, so a site with no
     * locale folder of its own must still find it.
     */
    public function lexiconTagsWordsTestCase()
    {
        $tokenizer = $this->test_objects['FILE1'];
        $tags = array_column($tokenizer::tagTokenizePartOfSpeech(
            "और सुंदर"), "tag");
        $this->assertEqual($tags, ["CC", "JJ"],
            "each word gets the lexicon's part of speech");
    }
}