_binary From 224353b8e9ea816311887e653bb04e4e43049adf Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Fri, 6 Dec 2019 21:43:32 -0800
Subject: [PATCH 1/3] Added more features in Chinese Segmenter;Moved partial
 code from StochasticTermSegmenter class to zh_CN\Tokenizer class for
 readability;Changed the default segmenter for Chinese from
 reverseMaximalMatch to StochasticTermSegmenter; r=chris

---
 src/library/StochasticTermSegmenter.php  | 58 +++++++-------------
 src/locale/zh_CN/resources/Tokenizer.php | 68 ++++++++++++++++++++++--
 2 files changed, 84 insertions(+), 42 deletions(-)

diff --git a/src/library/StochasticTermSegmenter.php b/src/library/StochasticTermSegmenter.php
index bd2a9d4f..ba6f6bdb 100644
--- a/src/library/StochasticTermSegmenter.php
+++ b/src/library/StochasticTermSegmenter.php
@@ -29,6 +29,7 @@
 namespace seekquarry\yioop\library;
 
 use seekquarry\yioop\library\Trie;
+use seekquarry\yioop\locale\zh_CN\resources as ZH;
 use seekquarry\yioop\configs as C;
 /**
  * A Stochastic Finite-State Word-Segmenter.
@@ -92,53 +93,21 @@ class StochasticTermSegmenter
         {
             case "zh_CN":
                 $this->non_char_preg = "/^[^\p{Han}]+$/u";
-                $this->num_dict =
-                   "1234567890○零一二三四五六七八九十百千万亿".
-                   "０１２３４５６７８９壹贰叁肆伍陆柒捌玖拾廿卅卌佰仟萬億";
-                $this->dot = "\.．";
-                $this->num_end = "％%";
-                $this->punctuation =
-                "\x{3000}-\x{303F}\x{FF00}-\x{FF0F}\x{FF1A}-\x{FF20}" .
-                "\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}\x{FFE0}-\x{FFEE}" .
-                "\x{21}-\x{2F}\x{21}-\x{2F}\x{3A}-\x{40}\x{5B}-\x{60}";
-                /**
-                 * Check if the term passed in is a Cardinal Number
-                 */
-                $this->isCardinalNumber = function($term) {
-                    return preg_match("/^[" . $this->num_dict .
-                        $this->dot . "]+[" . $this->num_end .
-                        "]?[余餘]?[百千万亿佰仟萬億]?$/u", $term);
-                };
-                /*
-                 * Check if the term passed in is a Ordinal Number
-                 */
-                $this->isOrdinalNumber = function($term) {
-                    return preg_match("/^第[" . $this->num_dict .
-                    "]*$/u", $term);
-                };
-                /*
-                 * Check if the term passed in is a date
-                 */
-                $this->isDate = function($term) {
-                    return preg_match("/^[" . $this->num_dict .
-                    "]+(年|年代|月|日|时|小时|時|小時|" .
-                    "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
-                };
                 /*
                  * Check if the term passed in is an exception term
                  */
                 $this->isExceptionImpl = function($term) {
-                    return $this->isCardinalNumber($term)
-                    || $this->isOrdinalNumber($term)
-                    || $this->isDate($term);
+                    return ZH\Tokenizer::isCardinalNumber($term)
+                    || ZH\Tokenizer::isOrdinalNumber($term)
+                    || ZH\Tokenizer::isDate($term);
                 };
                 /*
                  * Check if the term passed in is a punctuation
                  */
                 $this->isPunctuation = function($term)
                 {
-                    return preg_match("/^[" . $this->punctuation .
-                        "]$/u", $term);
+                    return preg_match("/^[" . ZH\Tokenizer::$punctuation .
+                        "]+$/u", $term);
                 };
                 break;
             case "ja":
@@ -331,6 +300,15 @@ class StochasticTermSegmenter
      */
     public function segmentSentence($sentence)
     {
+        $t=explode(" ",$sentence);
+        if(count($t) > 1) {
+            $ret = [];
+            foreach($t as $s) {
+                $ret=array_merge($ret,$this->segmentSentence($s));
+            }
+            return $ret;
+        }
+        
         if (!$this->dictionary_file) {
             $dic_file = C\LOCALE_DIR .
                 "/{$this->lang}/resources/term_weight.txt.gz";
@@ -353,10 +331,12 @@ class StochasticTermSegmenter
         $score[-1] = 0;
         for($index = 0; $index < count($characters); $index++) {
             //if not current language
-            if ($this->notCurrentLang($characters[$index]) ) {
+            if ($this->notCurrentLang($characters[$index])
+                && !$this->isPunctuation($characters[$index])) {
                 $current_char = $characters[$index];
                 for($j = $index + 1; $j < count($characters); $j++) {
-                    if ($this->notCurrentLang($current_char.$characters[$j])) {
+                    if ($this->notCurrentLang($current_char.$characters[$j])
+                        && !$this->isPunctuation($characters[$j])) {
                         $current_char .= $characters[$j];
                     } else {
                         break;
diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index bb46318c..70c409a7 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -29,6 +29,7 @@
 namespace seekquarry\yioop\locale\zh_CN\resources;
 
 use seekquarry\yioop\library\PhraseParser;
+use seekquarry\yioop\library\StochasticTermSegmenter;
 
 /**
  * Chinese specific tokenization code. Typically, tokenizer.php
@@ -56,6 +57,32 @@ class Tokenizer
         '与', '次', '狗', '决', '金', '史', '姆', '部', '正在', '活', '刚',
         '回家', '贝', '如何', '须', '战', '不會', '夫', '喂', '父', '亚', '肯定',
         '女孩', '世界'];
+    /**
+     * The dictionary of characters can be used as Chinese Numbers
+     * @string
+     */
+    public static $num_dict =
+       "1234567890○零一二三四五六七八九十百千万亿".
+       "０１２３４５６７８９壹贰叁肆伍陆柒捌玖拾廿卅卌佰仟萬億";
+    /**
+     * Dots used in Chinese Numbers
+     * @string
+     */
+    public static $dot = "\.．";
+    /**
+     * A list of characters can be used at the end of numbers
+     * @string
+     */
+    public static $num_end = "％%";
+    /**
+     * A list of characters can be used as Chinese punctuations
+     * @string
+     */
+    public static $punctuation =
+    "\x{2000}-\x{206F}\x{3000}-\x{303F}\x{FF00}-\x{FF0F}" .
+    "\x{FF1A}-\x{FF20}\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}" .
+    "\x{FFE0}-\x{FFEE}\x{21}-\x{2F}\x{21}-\x{2F}" .
+    "\x{3A}-\x{40}\x{5B}-\x{60}";
     /**
      * Removes the stop words from the page (used for Word Cloud generation
      * and language detection)
@@ -79,11 +106,46 @@ class Tokenizer
      * this is a bunch of words
      *
      * @param string $pre_segment  before segmentation
+     * @param string $method  indicates which method to use
      * @return string with words separated by space
      */
-    public static function segment($pre_segment)
+    public static function segment($pre_segment, $method="STS")
+    {
+        switch($method) {
+            case("RMM"):
+                return PhraseParser::reverseMaximalMatch($pre_segment, "zh-CN",
+                ['/\d+/', '/[a-zA-Z]+/']);
+                break;
+            case("STS"):
+                $segmenter = new StochasticTermSegmenter("zh_CN");
+                return $segmenter->segmentText($pre_segment,true);
+                break;
+        }
+    }
+    /**
+     * Check if the term passed in is a Cardinal Number
+     */
+    public static function isCardinalNumber($term)
+    {
+        return preg_match("/^[" . self::$num_dict .
+            self::$dot . "]+[" . self::$num_end .
+            "]?[余餘]?[百千万亿佰仟萬億]?$/u", $term);
+    }
+    /*
+     * Check if the term passed in is a Ordinal Number
+     */
+    public static function isOrdinalNumber($term)
+    {
+        return preg_match("/^第[" . self::$num_dict .
+        "]*$/u", $term);
+    }
+    /*
+     * Check if the term passed in is a date
+     */
+    public static function isDate($term)
     {
-        return PhraseParser::reverseMaximalMatch($pre_segment, "zh-CN",
-            ['/\d+/', '/[a-zA-Z]+/']);
+        return preg_match("/^[" . self::$num_dict .
+        "]+(年|年代|月|日|时|小时|時|小時|" .
+        "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
     }
 }
-- 
2.21.0.windows.1


From 84ec6e0c0b612be69b7c3357ed8275b195f0b569 Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Sat, 14 Dec 2019 16:01:17 -0800
Subject: [PATCH 2/3] fix RMM bugs; r=chris

---
 src/library/NWordGrams.php               |  2 +-
 src/library/PhraseParser.php             | 99 +++++++++++++++++-------
 src/locale/zh_CN/resources/Tokenizer.php |  2 +-
 3 files changed, 71 insertions(+), 32 deletions(-)

diff --git a/src/library/NWordGrams.php b/src/library/NWordGrams.php
index 818278d8..d1d12a8c 100644
--- a/src/library/NWordGrams.php
+++ b/src/library/NWordGrams.php
@@ -172,7 +172,7 @@ class NWordGrams
         foreach ($words as $word) {
             $tmp = trim($word);
             $len = mb_strlen($tmp);
-            $filter->add(mb_strtolower($tmp));;
+            $filter->add(mb_strtolower($tmp));
             for ($i = 1; $i < $len; $i++) {
                 $tmp2 = "*" . mb_substr($tmp, $i, $len, "UTF-8");
                 if ($tmp2 == "*") {continue;}
diff --git a/src/library/PhraseParser.php b/src/library/PhraseParser.php
index 1d479ab9..9536593a 100755
--- a/src/library/PhraseParser.php
+++ b/src/library/PhraseParser.php
@@ -1291,7 +1291,6 @@ class PhraseParser
     public static function reverseMaximalMatch($segment, $locale,
         $additional_regexes =[])
     {
-        $segment = " " . $segment;
         $len = mb_strlen($segment);
         $cur_pos = $len;
         if ($cur_pos < 1) {
@@ -1299,53 +1298,93 @@ class PhraseParser
         }
         $out_segment = "";
         $char_added = "";
-        $word_guess = "";
+        $word_guess_buffer = "";
+        $word_guess_buffer_index = 0;
         $was_space = true;
         while($cur_pos > 0) {
             $cur_pos--;
             $char_added =  mb_substr($segment, $cur_pos, 1);
             $is_space = trim($char_added) == "";
             if ($is_space && $was_space) {
-                continue;
+                //Do nothing
             } else if ($is_space) {
                 $was_space = true;
-                $one_word = self::oneWord($word_guess, $locale,
+                $one_word = self::oneWord($word_guess_buffer, $locale,
                     $additional_regexes);
                 if ($one_word) {
-                    $out_segment .= " " . strrev($word_guess);
-                    $word_guess = "";
+                    $out_segment .= " ";
+                    $word_guess_buffer = "";
+                    $word_guess_buffer_index=0;
                 } else {
-                    $out_segment .= " " . strrev(mb_substr($word_guess, 1));
-                    $out_segment .= " " . $char_added;
-                    $word_guess = "";
+                    if (mb_strlen($word_guess_buffer) ==
+                        $word_guess_buffer_index) {
+                        $out_segment .= " " . strrev(mb_substr($word_guess_buffer,
+                            $word_guess_buffer_index-1));
+                        if ($word_guess_buffer_index == 1) {
+                            $out_segment .= " " . $char_added;
+                        } else {
+                            $cur_pos += $word_guess_buffer_index;
+                        }
+                        $word_guess_buffer = "";
+                        $word_guess_buffer_index = 0;
+                    } else {
+                        $cur_pos += $word_guess_buffer_index;
+                        $word_guess_buffer = "";
+                        $word_guess_buffer_index = 0;
+                    }
                 }
-                continue;
-            } else {
-                $word_guess = $char_added . $word_guess;
-                $was_space = false;
             }
-            $is_suffix = NWordGrams::ngramsContains("*" . $word_guess,
-                $locale, "segment");
-            if (!$is_suffix) {
-                foreach ($additional_regexes as $regex) {
-                    if (preg_match($regex, $word_guess)) {
-                        $is_suffix = true;
-                        break;
+            //not space
+            else {
+                $word_guess_buffer = $char_added . $word_guess_buffer;
+                $word_guess_buffer_index++;
+                $was_space = false;
+            
+                $is_suffix = NWordGrams::ngramsContains("*" . $word_guess_buffer,
+                    $locale, "segment");
+                $is_one_word = self::oneWord($word_guess_buffer,
+                    $locale, $additional_regexes);
+                if (!$is_suffix) {
+                    foreach ($additional_regexes as $regex) {
+                        if (preg_match($regex, $word_guess_buffer)) {
+                            $is_suffix = true;
+                            break;
+                        }
                     }
                 }
-            }
-            if (!$is_suffix) {
-                if (mb_strlen($word_guess) > 1 &&
-                    !self::oneWord($word_guess, $locale, $additional_regexes)) {
-                    $out_segment .= " " . strrev(mb_substr($word_guess, 1));
-                    $word_guess = $char_added;
-                } else {
-                    $out_segment .= " " . strrev($word_guess);
-                    $word_guess = "";
+                // If this is a word, add to out_segment
+                // will add more if there is a longer match
+                if ($is_one_word) {
+                    $out_segment .= strrev(mb_substr($word_guess_buffer, 0,
+                        $word_guess_buffer_index));
+                    $word_guess_buffer_index=0;
                 }
-                $was_space = false;
+                if (!$is_suffix) {
+                    //no previous word found, add last character
+                    if (mb_strlen($word_guess_buffer) == $word_guess_buffer_index) {
+                        $out_segment .= " " . strrev(mb_substr($word_guess_buffer,
+                            $word_guess_buffer_index-1));
+                        $cur_pos+=$word_guess_buffer_index-1;
+                        $word_guess_buffer_index=0;
+                        $word_guess_buffer = "";
+                    }
+                    //has previous word found
+                    else {
+                        $cur_pos+=$word_guess_buffer_index;
+                        if ($cur_pos > 0) {
+                            $out_segment .= " ";
+                        }
+                        $word_guess_buffer_index=0;
+                        $word_guess_buffer = "";
+                    }
+                    $was_space = false;
+                }
+            
             }
         }
+        if ($word_guess_buffer_index > 0) {
+            $out_segment .= " " . strrev($word_guess_buffer);
+        }
         $out_segment = strrev($out_segment);
         return $out_segment;
     }
diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index 70c409a7..a6e674fa 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -114,7 +114,7 @@ class Tokenizer
         switch($method) {
             case("RMM"):
                 return PhraseParser::reverseMaximalMatch($pre_segment, "zh-CN",
-                ['/\d+/', '/[a-zA-Z]+/']);
+                ['/\d+/', '/^[a-zA-Z]+$/']);
                 break;
             case("STS"):
                 $segmenter = new StochasticTermSegmenter("zh_CN");
-- 
2.21.0.windows.1


From 4072f99db9e795a624faed51bab3900a06054716 Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Sat, 14 Dec 2019 16:33:26 -0800
Subject: [PATCH 3/3] fix a minor bug

---
 src/locale/zh_CN/resources/Tokenizer.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index a6e674fa..b0e66e98 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -114,7 +114,7 @@ class Tokenizer
         switch($method) {
             case("RMM"):
                 return PhraseParser::reverseMaximalMatch($pre_segment, "zh-CN",
-                ['/\d+/', '/^[a-zA-Z]+$/']);
+                ['/^\d+$/', '/^[a-zA-Z]+$/']);
                 break;
             case("STS"):
                 $segmenter = new StochasticTermSegmenter("zh_CN");
-- 
2.21.0.windows.1

