<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\locale\nl\resources;

/**
 * This class has a collection of methods for Dutch locale specific
 * tokenization. In particular, it has a stemmer, .
 *
 * @author Charles Bocage
 */
class Tokenizer
{
    /**
     * A list of frequently occurring terms for this locale which should
     * be excluded from certain kinds of queries. This is also used
     * for language detection
     * @var array
     */
    public static $stop_words = ['als', 'I', 'zijn', 'dat', 'hij', 'was',
        'voor', 'op', 'zijn', 'met', 'ze', 'zijn', 'bij', 'een', 'hebben',
        'deze', 'van', 'door', 'heet', 'woord', 'maar', 'wat', 'sommige',
        'is', 'het', 'u', 'of', 'had', 'de', 'van', 'aan', 'en', 'een',
        'in', 'we', 'kan', 'uit', 'andere', 'waren', 'die', 'doen', 'hun',
        'tijd', 'indien', 'zal', 'hoe', 'zei', 'een', 'elk', 'vertellen',
        'doet', 'set', 'drie', 'willen', 'lucht', 'goed', 'ook', 'spelen',
        'klein', 'end', 'zetten', 'thuis', 'lezen', 'de hand', 'poort',
        'grote', 'spell', 'toevoegen', 'zelfs', 'land', 'hier', 'moet',
        'grote', 'hoog', 'dergelijke', 'volgen', 'act', 'waarom', 'vragen',
        'mannen', 'verandering', 'ging', 'licht', 'soort', 'uitgeschakeld',
        'nodig', 'huis', 'afbeelding', 'proberen', 'ons', 'weer', 'dier',
        'punt', 'moeder', 'wereld', 'dichtbij', 'bouwen', 'zelf', 'aarde',
        'vader'];
    /**
     * Words we don't want to be stemmed
     * @var array
     */
    public static $no_stem_list = [
        "abs", "ahs", "aken", "àlle", "als", "are",
        "allèen", "ate", "aten", "azen", "bse", "cfce", "curaçao",
        "dègelijk", "dme", "ede", "eden", "eds", "ehs", "ems", "ene", "epe",
        "eps", "ers", "eten", "ets", "even", "fme", "gedaçht", "ghe", "gve",
        "hdpe", "hôte", "hpe", "hse", "ibs", "ics", "ile", "ims", "jònge",
        "kwe", "ldpe", "lldpe", "lme", "lze", "maitres", "mwe", "nme", "ode",
        "ogen", "oke", "ole", "ons", "ònze", "open", "ops", "oren", "ors",
        "oss", "oven", "ows", "pre", "pve", "rhône", "ròme", "rwe", "ske",
        "sme", "spe", "ste", "the", "tje", "uce", "uden", "uien", "uren",
        "use", "uwe", "vse", "ype"
    ];
    /**
     * Removes the stop words from the page (used for Word Cloud generation
     * and language detection)
     *
     * @param mixed $data either a string or an array of string to remove
     *      stop words from
     * @return mixed $data with no stop words
     */
    public static function stopwordsRemover($data)
    {
        static $pattern = "";
        if (empty($pattern)) {
            $pattern = '/\b(' . implode('|', self::$stop_words) . ')\b/u';
        }
        $data = preg_replace($pattern, '', $data);
        return $data;
    }
    /**
     * Stub function which could be used for a word segmenter.
     * Such a segmenter on input thisisabunchofwords would output
     * this is a bunch of words
     *
     * @param string $pre_segment  before segmentation
     * @return string should return string with words separated by space
     *     in this case does nothing
     */
    public static function segment($pre_segment)
    {
        return $pre_segment;
    }
    /**
     * boolean that tells the code if the e suffix was removed in step2 or not
     * @var array
     */
    static $removed_e_suffix = false;
    /**
     * Computes the stem of a Dutch word
     *
     * For example, lichamelijk, lichamelijke, lichamelijkheden and lichamen,
     * all have licham as a stem
     *
     * @param string $word the string to stem
     * @return string the stem of $words
     */
    public static function stem($word)
    {
        self::$removed_e_suffix = false;
        $result = $word;
        if (isset($word) && !empty($word)) {
            $result = self::removeAllUmlautAndAcuteAccents($word);
            $result = trim(mb_strtolower($result));
            $result = self::substituteIAndY($result);
            $R1 = self::getRIndex($result, 1);
            $R2 = self::getRIndex($result, $R1);
            $result = self::step1($result, $R1);
            $result = self::step2($result);
            $result = self::step3a($result, $R2);
            $result = self::step3b($result, $R2);
            $result = self::step4($result);
            $result = mb_strtolower($result);
        }
        return $result;
    }
    /**
     * Remove all umlaut and acute accents that need to be removed.
     *
     * @param string $word the string to remove the umlauts and accents from
     * @return string the string with the umlauts and accents removed
     */
    private static function removeAllUmlautAndAcuteAccents($word)
    {
        $result = preg_replace("/\é|\ë/", "e", $word);
        $result = preg_replace("/\á|ä/", "a", $result);
        $result = preg_replace("/\ó|ö/", "o", $result);
        $result = preg_replace("/\ï/", "i", $result);
        $result = preg_replace("/\ü|\ú/", "u", $result);
        return $result;
    }
    /**
     * Put initial y, y after a vowel, and i between vowels into upper case.
     *
     * @param string $word the string to put initial y, y after a vowel, and
     * i between vowels into upper case.
     * @return string the string with an initial y, y after a vowel, and i
     * between vowels into upper case.
     */
    private static function substituteIAndY($word)
    {
        $word_split = preg_split('/(?!^)(?=.)/u', $word);
        for ($i = 0; $i < count($word_split); $i++) {
            if ($i == 0) {
                if ($word_split[$i] == 'y') {
                    $word_split[$i] = 'Y';
                }
            } else {
                if ($word_split[$i] == 'i') {
                    if (count($word_split) > ($i + 1)) {
                        if (self::isVowel($word_split[$i - 1]) &&
                            self::isVowel($word_split[$i + 1])) {
                            $word_split[$i] = 'I';
                        }
                    }
                } elseif ($word_split[$i] == 'y') {
                    if (self::isVowel($word_split[$i - 1])) {
                        $word_split[$i] = 'Y';
                    }
                }
            }
        }
        return implode('', $word_split);
    }
    /**
     * Check that the letter is a vowel
     *
     * @param string $letter the character to check
     * @return boolean true if it is a vowel, otherwise false
     */
    private static function isVowel($letter)
    {
        return match ($letter) {
            'a', 'e', 'i', 'o', 'u', 'y', 'è' => true,
            default => false,
        };
    }
    /**
     * Get the R index.  The R index is the first consonent that follows a
     * vowel after the $start index
     *
     * @param string $word the string to search for the R index
     * @param int $start the index to start searching for the R index in the
     * string
     * @return int the R index if found, otherwise -1
     */
    private static function getRIndex($word, $start)
    {
        $result = -1;
        $word_split = preg_split('/(?!^)(?=.)/u', $word);
        for ($i = $start <= 0 ? 1 : $start; $i < count($word_split); $i++) {
            if (!self::isVowel($word_split[$i]) &&
                self::isVowel($word_split[$i - 1])) {
                $result = ($i + 1);
                break;
            }
        }
        return $result;
    }
    /**
     * Define a valid en-ending as a non-vowel, and not gem and remove it
     *
     * @param string $word the string to stem
     * @param int $R1 the int that represents the R index
     * @return string the string with the valid en-ending as a non-vowel, and
     * not gem ending removed
     */
    private static function step1($word, $R1)
    {
        $result = $word;
        if ($R1 > -1) {
            $wordLength = strlen($word);
            if ($wordLength > 2 && $R1 < $wordLength) {
                if (self::endsWith($word, "heden")) {
                    $result = self::replace($word, '/heden$/', 'heid', $R1);
                } else {
                    if (preg_match("/(?<![aeioèuy]|gem)(ene?)$/", $word,
                        $matches, 0, $R1) && $word != "eten" &&
                        $word != "even" && $word != "opene") {
                        $result = self::undouble(self::replace($word,
                            '/ene$/', '', $R1));
                        $result = self::undouble(self::replace($result,
                            '/en$/', '', $R1));
                    } else {
                        if (preg_match("/(?<![aeiouyèj])se$/", $word,
                            $matches, 0, $R1) && $word != "osse") {
                            $result = self::replace($word,
                                '/(?<![aeiouyèj])se$/', '', $R1);
                        } elseif (preg_match("/(?<![aeiouyèj])s$/", $word,
                            $matches, 0, $R1)) {
                            $result = self::replace($word,
                                '/(?<![aeiouyèj])s$/', '', $R1);
                        }
                    }
                }
            }
        }
        return $result;
    }
    /**
     * Delete the suffix e if in R1 and preceded by a non-vowel, and then
     * undouble the ending
     *
     * @param string $word the string to delete the suffix e if in R1 and
     * preceded by a non-vowel, and then undouble the ending
     * @return string the string with the suffix e if in R1 and preceded by
     * a non-vowel deleted, and then undouble the ending
     */
    private static function step2($word)
    {
        $result = $word;
        if (self::endsWith($word, "e")
            && $word != "ene" && $word != "oce" && $word != "ohe" &&
                $word != "pâte") {
            $word_split = preg_split('/(?!^)(?=.)/u', $word);
            $word_split_length = count($word_split);
            if ($word_split_length > 2 &&
                !self::isVowel($word_split[$word_split_length - 2])) {
                $word_split = array_slice($word_split, 0,
                    $word_split_length - 1);
                $result = implode('', $word_split);
                $result = self::undouble($result);
                self::$removed_e_suffix = true;
            }
        }
        return $result;
    }
    /**
     * Delete the letters heid if in R2 and not preceded by a c, and treat an a
     * preceding en like in step 1
     *
     * @param string $word the string to delete the letters heid if in R2 and
     * not preceded by a c, and treat an a preceding en like in step 1
     * @param int $R2 the R index
     * @return string the string with the letters heid if in R2 and not
     * preceded by a c deleted, and treated an a preceding en like in step 1
     */
    private static function step3a($word, $R2)
    {
        $result = $word;
        if ($R2 > -1) {
            if (preg_match("/(?<![c])(heid)$/", $word, $matches, 0, $R2)) {
                $result = self::replace($word, '/(?<![c])(heid)$/', '', $R2);
                if (preg_match("/(?<![aeiouyè]|gem)(ene?)$/", $result,
                    $matches, 0, $R2)) {
                    $result = self::undouble(self::replace($result,
                        '/(?<![aeiouyè]|gem)(ene?)$/', '', $R2));
                }
            }
        }
        return $result;
    }
    /**
     * Search for the longest among the following suffixes, and perform the
     * action indicated.
     * If in R2 and ends with eigend, eigingm igend or iging remove it
     * If in R2 and ends with ig preceded by an e remove it
     * If in R2 and ends with lijk, baar or bar then remove it
     *
     * @param string $word the string to stem
     * @param int $R2 the R index
     * @return string the string with the various endings removed if they exist
     */
    public static function step3b($word, $R2)
    {
        $result = $word;
        $word_split = preg_split('/(?!^)(?=.)/u', $word);
        if (count($word_split) > 2) {
            if ($R2 > -1) {
                if (preg_match('/(end|ing)$/', $word, $matches, 0, $R2)) {
                    if (preg_match('/eig(end|ing)$/', $word, $matches, 0,
                        $R2)) {
                        $result = self::replace($word, '/(end|ing)$/', '',
                            $R2);
                    }
                    elseif (preg_match('/ig(end|ing)$/', $word, $matches, 0,
                        $R2)) {
                        $result = self::replace($word, '/(igend|iging)$/', '',
                            $R2);
                        $result = self::undouble($result);
                    }
                    else {
                        $result = self::replace($word, '/(end|ing)$/', '',
                            $R2);
                        $result = self::undouble($result);
                    }
                }
                elseif (preg_match("/(?<![e])ig$/", $word, $matches, 0, $R2)) {
                    $result = self::replace($word, '/(?<![e])ig$/', '', $R2);
                }
                elseif (preg_match("/lijk$/", $word, $matches, 0, $R2)) {
                    $result = self::replace($word, '/lijk$/', '', $R2);
                    $result = self::step2($result);
                }
                elseif (preg_match("/baar$/", $word, $matches, 0, $R2)) {
                    $result = self::replace($word, '/baar$/', '', $R2);
                }
                elseif (preg_match("/bar$/", $word, $matches, 0, $R2) &&
                    self::$removed_e_suffix) {
                    $result = self::replace($word, '/bar$/', '', $R2);
                }
            }
        }
        return $result;
    }
    /**
     * If the words ends CVD, where C is a non-vowel, D is a non-vowel
     * other than I, and V is double a, e, o or u, remove one of the
     * vowels from V (for example, moom -> mon, weed -> wed).
     *
     * @param string $word the string to check for the CVD combination
     * @return string the string with the CVD combination removed otherwise
     * the original string
     */
    private static function step4($word)
    {
        $result = $word;
        $word_split = preg_split('/(?!^)(?=.)/u', $word);
        $numberOfLetters = count($word_split);
        if ($numberOfLetters > 3) {
            $c = $word_split[$numberOfLetters - 4];
            $v1 = $word_split[$numberOfLetters - 3];
            $v2 = $word_split[$numberOfLetters - 2];
            $d = $word_split[$numberOfLetters - 1];
            if (!self::isVowel($c) &&
                self::isVowel($v1) &&
                self::isVowel($v2) &&
                !self::isVowel($d) &&
                $v1 == $v2 &&
                $d != 'I' &&
                $v1 != 'i') {
                unset($word_split[$numberOfLetters - 2]);
                $result = implode('', $word_split);
            }
        }
        return $result;
    }
    /**
     * Replace a string based on a regex expression
     *
     * @param string $word the string to search for regex replacement
     * @param string $regex the regex to use to find and replacement
     * @param string $replace the string to replace if the pattern is matched
     * @param int $offset the int to start to look for the regex replacement
     * @return string the string with the characters replaced if the regex
     * matches, otherwise the original string
     */
    private static function replace($word, $regex, $replace, $offset)
    {
        $result = "";
        if ($offset > 0) {
            $part1 = substr($word, 0, $offset);
            $part2 = substr($word, $offset, strlen($word));
            $part2 = preg_replace($regex, $replace, $part2);
            $result = $part1 . "" . $part2;
        } else {
            $result = preg_replace($regex, $replace, $word);
        }
        return $result;
    }
    /**
     * Checks to see if a string ends with a certain string
     *
     * @param string $haystack the string to check
     * @param string $needle the string to match at the end
     * @param bool $case whether the check should be case insensitive or not
     * @return boolean true if it ends with $needle, otherwise false
     */
    private static function endsWith($haystack, $needle, $case = true)
    {
        if ($case) {
            return (strcmp(substr($haystack,
                strlen($haystack) - strlen($needle)), $needle) === 0);
        }
        return (strcasecmp(substr($haystack,
            strlen($haystack) - strlen($needle)), $needle) === 0);
    }
    /**
     * undoubles the end of a string.  If the string ends in kk, tt, dd remove
     * one of the characters
     *
     * @param string $word the string to undouble
     * @return string the undoubled string, otherwise the original string
     */
    private static function undouble($word)
    {
        $result = $word;
        if (self::endsWith($word, "kk") ||
            self::endsWith($word, "tt") ||
            self::endsWith($word, "dd")) {
            $result = substr($word, 0, strlen($word) - 1);
        }
        return $result;
    }
}
