<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * @author Niravkumar Patel niravkumar.patel1989@gmail.com
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\locale\pt\resources;

/**
 * This class has a collection of methods for Portuguese locale specific
 * tokenization. In particular, it has a stemmer implementing the
 * Snowball Stemming algorithm presented in
 * http://snowball.tartarus.org/algorithms/portuguese/stemmer.html
 *
 * @author Niravkumar Patel
 */
class Tokenizer
{
    /**
     * A list of frequently occurring terms for this locale which should
     * be excluded from certain kinds of queries. This is also used
     * for language detection
     * @var array
     */
    public static $stop_words = ['como', 'I', 'seu', 'ele', 'foi', 'para', 'em',
        'são', 'com', 'eles', 'ser', 'em', 'uma', 'tem', 'este',
        'partir', 'de', 'por', 'quente', 'palavra', 'mas',
        'que', 'alguns', 'é', 'ele', 'você', 'ou', 'teve',
        'o', 'a', 'e', 'uma', 'em', 'nós', 'lata', 'fora',
        'outro', 'foram', 'que', 'fazer', 'seu', 'tempo', 'se',
        'vontade', 'como', 'disse', 'uma', 'cada', 'dizer', 'faz',
        'conjunto', 'três', 'quer', 'ar', 'bem', 'também', 'jogar',
        'pequeno', 'fim', 'colocar', 'casa', 'ler', 'mão', 'port',
        'grande', 'soletrar', 'adicionar', 'mesmo', 'terra', 'aqui',
        'necessário', 'grande', 'alto', 'tais', 'siga', 'ato',
        'perguntar', 'homens', 'mudança', 'fui', 'luz',
        'tipo', 'off', 'precisa', 'casa', 'imagem', 'tentar', 'nós',
        'novamente', 'animais', 'ponto', 'mãe', 'mundo', 'perto',
        'construir', 'auto', 'terra', 'pai'];
    /**
     * Phrases we would like yioop to rewrite before performing a query
     * @var array
     */
    public static $semantic_rewrites = [];
    /**
     * storage used in computing the stem
     * @var string
     */
    private static $buffer;
    /**
     * Index of the current end of the word at the current state of computing
     * its stem
     * @var int
     */
    private static $k;
    /**
     * R1 is the region in the word after the first non-vowel following
     * a vowel, or is the null region at the end of the word if there
     * is no such non-vowel
     * @var string
     */
    private static $r1 = "";
    /**
     * R2 is the region in the R1 after the first non-vowel following a vowel,
     * or is the null region at the end of the word if there is no non-vowel
     * @var string
     */
    private static $r2 = "";
    /**
     * If the second letter is a consonant, RV is the region after the next
     * following vowel, or if the first two letters are vowels, RV is the
     * region after the next consonant, and otherwise (consonant-vowel case)
     * RV is the region after the third letter.
     * @var string
     */
    private static $rv = "";
    /**
     * Removes the stop words from the page (used for Word Cloud generation
     * and language detection)
     *
     * @param mixed $data either a string or an array of string to remove
     *      stop words from
     * @return mixed $data with no stop words
     */
    public static function stopwordsRemover($data)
    {
        static $pattern = "";
        if (empty($pattern)) {
            $pattern = '/\b(' . implode('|', self::$stop_words) . ')\b/u';
        }
        $data = preg_replace($pattern, '', $data);
        return $data;
    }
    /**
     * Stub function which could be used for a word segmenter.
     * Such a segmenter on input thisisabunchofwords would output
     * this is a bunch of words
     *
     * @param string $pre_segment  before segmentation
     * @return string should return string with words separated by space
     *     in this case does nothing
     */
    public static function segment($pre_segment)
    {
        return $pre_segment;
    }
    /**
     * Computes the stem of an Portuguese word
     * For example, química, químicas, químico, químicos
     * all have químic as a stem
     *
     * @param string $word the string to stem
     * @return string the stem of $words
     */
    public static function stem($word)
    {
        self::$buffer = $word;
        self::$k = strlen($word) - 1;
        if (self::$k <= 1) { return $word; }
        $word = str_ireplace("ã", "a~", $word);
        $word = str_ireplace("õ", "o~", $word);
        self::$r1 = self::findR1($word);
        self::$r2 = self::findR1(self::$r1);
        self::findRV($word);
        $temp_word = self::step1($word);
        if ($temp_word == $word){
            $temp_word = self::step2($word);
        }
        if ($temp_word != $word){
            self::findRV($temp_word);
            $temp_word = self::step3($temp_word);
        }else{
            $temp_word = self::step4($temp_word);
            self::findRV($temp_word);
        }
        $temp_word = self::step5($temp_word);
        $temp_word = str_ireplace("a~", "ã", $temp_word);
        $temp_word = str_ireplace("o~", "õ", $temp_word);
        self::$rv = "";
        $word = $temp_word;
        return $word;
    }
    /**
     * Standard Suffix Removal Step
     * It search for longest suffix from given set and remove
     * if found
     *
     * @param string $word the string to suffix removal
     * @return processed string
     */
    private static function step1($word)
    {
        $suffix_set = ['eza','ezas','ico','ica','icos','icas','ismo',
            'ismos','ável','ível','ista','istas','oso','osa',
            'osos','osas','amento','amentos','imento','imentos',
            'adora','ador','aça~o','adoras','adores','aço~es',
            'ante','antes','ância'];
        $regex = "/[a-zA-Z~]*(eza|ezas|ico|ica|icos|icas|ismo|ismos|ável|ível
            |ista|istas|oso|osa|osos|osas|amento|amentos|imento
            |imentos|adora|ador|aça~o|adoras|adores|aço~es|ante
            |antes|ância)";
        foreach ($suffix_set as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r2)) {
                $word = substr($word,0,-strlen($value));
                break;
            }
        }
        $special_suffix = ['logía' , 'logías'];
        foreach ($special_suffix as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r2)){
                $word = substr($word,0,-strlen($value))."log";
                break;
            }
        }
        $special_suffix = ['ución','uciones'];
        foreach ($special_suffix as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r2)) {
                $word = substr($word,0,-strlen($value))."u";
                break;
            }
        }
        $special_suffix = ['ência','ências'];
        foreach ($special_suffix as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r2)){
                $word = substr($word,0,-strlen($value))."ente";
                break;
            }
        }
        $special_suffix = ['ativamente','ivamente','osamente','icamente',
            'adamente','antemente','avelmente','ívelmente'];
        $is_removed = false;
        foreach ($special_suffix as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r2)) {
                $word = substr($word,0,-strlen($value));
                $is_removed = true;
                break;
            }
        }
        if (!$is_removed) {
            $is_removed = false;
            $value = "amente";
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r1)) {
                $word = substr($word,0,-strlen($value));
                $is_removed = true;
            }
            if (!$is_removed) {
                $value = "mente";
                $regex = "/[a-zA-Z~]*($value)$/";
                if (preg_match($regex, self::$r2)){
                    $word = substr($word,0,-strlen($value));
                    $is_removed = true;
                }
            }
        }

        $special_suffix = ['abilidades','icidades','ividades','abilidade',
            'icidade','ividade','idades','idade'];
        foreach ($special_suffix as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r2)) {
                $word = substr($word,0,-strlen($value));
                break;
            }
        }
        $special_suffix = ['ativa','ativo','ativas','ativos','iva','ivo',
            'ivas','ivos'];
        foreach ($special_suffix as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$r2)){
                $word = substr($word,0,-strlen($value));
                break;
            }
        }
        $special_suffix = ['iras','ira'];
        foreach ($special_suffix as $value) {
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$rv)
                && preg_match("/[a-zA-Z~]*(eiras|eira)$/",$word)) {
                $word = substr($word,0,-strlen($value))."ir";
                break;
            }
        }
        return $word;
    }
    /**
     * Verb Suffix Removal Step
     * If step 1 does not change anything than this function
     * will be called
     *
     * It will also check for longest suffix from the suffix set
     * Remove if found
     *
     * @param string $word the string to suffix removal
     * @return processed string
     */
    private static function step2($word)
    {
        $suffix_set = ["aríamos","eríamos","iríamos","ássemos",
            "êssemos","íssemos","aríeis","eríeis","iríeis","ásseis",
            "ésseis","ísseis","áramos","éramos","íramos","ávamos",
            "aremos","eremos","iremos","ariam","eriam","iriam",
            "assem","essem","issem","ara~o","era~o","ira~o","arias",
            "erias","irias","ardes","erdes","irdes","asses","esses",
            "isses","astes","estes","istes","áreis","areis","éreis",
            "ereis","íreis","ireis","áveis","íamos","armos","ermos","irmos",
            "aria","eria","iria","asse","esse","isse","aste","este","iste",
            "arei","erei","irei","aram","eram","iram","avam","arem","erem",
            "irem","ando","endo","indo","adas","idas","arás","aras","erás",
            "eras","irás","avas","ares","eres","ires","íeis","ados","idos",
            "ámos","amos","emos","imos","iras","ada","ida","ará","ara","erá",
            "era","irá","ava","iam","ado","ido","ias","ais","eis","ira","ia",
            "ei","am","em","ar","er","ir","as","es","is","eu","iu","ou"];
        foreach ($suffix_set as $value){
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, self::$rv)) {
                $word = substr($word,0,-strlen($value));
                break;
            }
        }
        return $word;
    }
    /**
     * Delete suffix i if in RV and preceded by c
     *
     * @param string $word the string to suffix removal
     * @return processed string
     */
    private static function step3($word)
    {
        $regex = "/[a-zA-Z~]*(ci)$/";
        if (preg_match($regex, $word)) {
            $regex = "/(i)[a-zA-Z~]*$/";
            if (preg_match($regex, self::$rv)) {
                $word = substr($word,0,-1);
            }
        }
        return $word;
    }
    /**
     * Residual suffix
     * If the word ends with one of [os a i o á í ó]
     * in RV
     *
     * @param string $word the string to suffix removal
     * @return processed string
     */
    private static function step4($word)
    {
        $regex = "/[a-zA-Z~]*(os)$/";
        if (preg_match($regex, self::$rv)){
            $word = substr($word,0,-2);
        }else{
            $regex = "/[a-zA-Z~]*(a|i|o)$/";
            if (preg_match($regex, self::$rv)){
                $word = substr($word,0,-1);
            }else{
                $regex = "/[a-zA-Z~]*(á|í|ó)$/";
                if (preg_match($regex, self::$rv)){
                    $word = substr($word,0,-2);
                }
            }
        }
        return $word;
    }
    /**
     * Residual suffix
     * If the word ends with one of [e é ê]
     * in RV
     *
     * @param string $word the string to suffix removal
     * @return processed string
     */
    private static function step5($word)
    {
        if (preg_match("/[a-zA-Z~]*(ç)$/", $word)){
            $word = substr($word,0,-2)."c";
            return $word;
        }
        $special_suffix = ["gue","cie"];
        foreach ($special_suffix as $value){
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, $word)) {
                $value = substr($value,1,strlen($value));
                $regex = "/[a-zA-Z~]*($value)[a-zA-Z~]*$/";
                if (preg_match($regex, self::$rv)){
                    $word = substr($word, 0, -2);
                    return $word;
                }
            }
        }
        $special_suffix = ["gué","guê","cié","ciê"];
        foreach ($special_suffix as $value){
            $regex = "/[a-zA-Z~]*($value)$/";
            if (preg_match($regex, $word)) {
                $value = substr($value, 1, strlen($value));
                $regex = "/[a-zA-Z~]*($value)$/";
                if (preg_match($regex, self::$rv)){
                    $word = substr($word, 0, -3);
                    return $word;
                }
            }
        }
        if (preg_match("/[a-zA-Z~]*(e)$/", $word)
            && preg_match("/[a-zA-Z~]*(e)[a-zA-Z~]*$/", self::$rv)){
            $word = substr($word, 0, -1);
            return $word;
        }
        if (preg_match("/[a-zA-Z~]*(é|ê)$/", $word)
            && preg_match("/[a-zA-Z~]*(é|ê)[a-zA-Z~]*$/", self::$rv)){
            $word = substr($word, 0, -2);
            return $word;
        }
        return $word;
    }
    /**
     * This method will find R1 region in the $word
     * R1 is the region after the first non-vowel following a vowel, or is
     * the null region at the end of the word if there is no such non-vowel
     *
     * @param string $word the Portuguese word to find R1 within
     * @return string $r1 region
     */
    private static function findR1($word)
    {
        $word_length = mb_strlen($word);
        $vowel = self::mbStringToArray("aeiouáéíóúâêô");
        $first_vowel = false;
        $second_vowel = false;
        $first_consonent = false;
        $second_consonent = false;
        $r1 = "";
        $word_array = self::mbStringToArray($word);
        $i=0;
        for ($i=0; $i<$word_length; $i++) {
            if (in_array($word_array[$i],$vowel)) {
                if ($first_vowel) {
                    if ($first_consonent) {
                        break;
                    }
                }
                $first_vowel = true;

            } else {
                if ($first_vowel) {
                    if ($first_consonent ){
                        break;
                    }
                    $first_consonent = true;
                }
            }
        }
        return mb_substr($word, $i, mb_strlen($word));
    }
    /**
     * This method will find RV region in the $word
     * If the second letter is a consonant,
     * RV is the region after the next following vowel, or
     * if the first two letters are vowels,
     * RV is the region after the next consonant,
     * and otherwise (consonant-vowel case)
     * RV is the region after the third letter.
     *
     * @param string $word the Portuguese word to find RV within
     * @return string $rv region
     */
    private static function findRV($word)
    {
        $vowel = self::mbStringToArray("aeiouáéíóúâêô");
        $word_array = self::mbStringToArray($word);
        if (!isset($word_array[1])  || !in_array($word_array[1], $vowel)){
            $word_length = mb_strlen($word);
            $i = 2;
            for ($i = 2; $i < $word_length; $i++) {
                if (in_array($word_array[$i],$vowel)){
                    $i++;
                    break;
                }
            }
            self::$rv = mb_substr($word, $i, mb_strlen($word));
        } else if (in_array($word_array[0], $vowel)
            && in_array($word_array[1],$vowel)) {
            $word_length = mb_strlen($word);
            for ($i = 2; $i < $word_length; $i++){
                if (!in_array($word_array[$i],$vowel)){
                    self::$rv = mb_substr($word, $i + 1, mb_strlen($word));
                    break;
                }
            }
        } else if (!in_array($word_array[0], $vowel)
            && in_array($word_array[1],$vowel)){
            self::$rv = mb_substr($word, 3, mb_strlen($word));
        }
    }
    /**
     * This method will break-up a multibyte string into its individual
     * characters and generate an array of characters
     *
     * @param string $string of multibyte characters to break-up
     * @return array of multibyte characters
     */
    private static function mbStringToArray($string)
    {
        $len = mb_strlen($string);
        $array = [];
        while($len) {
            $array[] = mb_substr($string,0, 1, "UTF-8");
            $string = mb_substr($string, 1, $len,"UTF-8");
            $len = mb_strlen($string);
        }
        return $array;
    }
}
