_binary From d62541736eec88f048d5c7b742d3b260fe8bb74d Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Sun, 23 Feb 2020 23:09:43 -0800
Subject: [PATCH] ContextWeightedPosTagger.php version 1.0; some updates on
 Segmenter and ZH\Tokenizer r=chris

---
 src/library/ContextWeightedPosTagger.php | 613 +++++++++++++++++++++++
 src/library/StochasticTermSegmenter.php  |  28 +-
 src/locale/zh_CN/resources/Tokenizer.php |  95 +++-
 3 files changed, 713 insertions(+), 23 deletions(-)
 create mode 100644 src/library/ContextWeightedPosTagger.php

diff --git a/src/library/ContextWeightedPosTagger.php b/src/library/ContextWeightedPosTagger.php
new file mode 100644
index 00000000..aadcd510
--- /dev/null
+++ b/src/library/ContextWeightedPosTagger.php
@@ -0,0 +1,613 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * @author Xianghong Sun sxh19911230@gmail.com
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+ 
+namespace seekquarry\yioop\library;
+
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\locale\zh_CN\resources as ZH;
+
+/**
+ * Machine learning based POS tagger. Typically, ContextWeightedPosTagger.php
+ * can train the language with some dataset and predict
+ * the tag given a list of word.
+ *
+ * @author Xianghong Sun
+ */
+ 
+class ContextWeightedPosTagger {
+    /**
+     * Current Language, only tested on Simplified Chinese
+     * Might be extensable for other languages in the furture
+     * @var string
+     */
+    public $lang;
+    
+    /**
+     * The weight for predicting the pos tag
+     * y = wx + b
+     * Generized by training method
+     * @var array
+     */
+    public $w;
+    
+    /**
+     * The bias for predicting the pos tag
+     * y = wx + b
+     * Generized by training method
+     * @var array
+     */
+    public $b;
+    
+    /**
+     * Should weight and bias packed?
+     * Packed variables are much smaller than unpacked
+     * In Chinese, the amount difference is 884.05 MB vs 64.72MB
+     * The trade off is some speed in training. About 3 : 2
+     * Don't find much difference in prediction
+     * So can just set packed true in prediction
+     * @var bool
+     */
+    private $packed;
+    
+    /**
+     * All Possiable tag set
+     * Generized by training method
+     * @var associative array [tag => tag index]
+     */
+    private $tag_set;
+    /**
+     * Check if all the chars in the term is not current language
+     * @param $term is a string that to be checked
+     * @return true if all the chars in $term is not current language
+     *         false otherwise
+     */
+    public function notCurrentLang($term)
+    {
+        return preg_match("/^[^\p{Han}]+$/u", $term);
+    }
+    
+    /**
+     * The constructer of the pos tagger
+     * To extend to other languages, some work are needed:
+     * Define $this->getKeyImpl, $this->rule_defined_key
+     * See Chinese example.
+     * @param @string $lang describes current langauge
+     * @param @book $packed describes how weight and bias would look like
+     */
+    public function __construct($lang, $packed = true)
+    {
+        $this->packed = $packed;
+        switch($lang) {
+            case("zh_CN"):
+            case("zh-CH"):
+                $this->lang = "zh_CN";
+                
+                /*
+                 * Some Exception of Tags. Some tags are detemined by ruls.
+                 * e.x. There are infinity amount of number combination in Chinese.
+                 */
+                $this->getKeyImpl = function($term) {
+                    if ($term == 'x') {
+                        return $this->tag_set['X'];
+                    } else if (ZH\Tokenizer::isPunctuation($term)) {
+                        return $this->tag_set['PU'];
+                    } else if (ZH\Tokenizer::isCardinalNumber($term)) {
+                        return $this->tag_set['CD'];
+                    } else if (ZH\Tokenizer::isOrdinalNumber($term)) {
+                        return $this->tag_set['OD'];
+                    } else if (ZH\Tokenizer::isDate($term)) {
+                        return $this->tag_set['NT'];
+                    } else if (preg_match("/https?:\/\/|www\.|\.com|\.cn|\.org/u",
+                        mb_convert_kana($term, 'a', 'UTF-8'))) {
+                        return $this->tag_set['URL'];
+                    } else if ($this->notCurrentLang($term)) {
+                        return $this->tag_set['FW'];
+                    }
+                    return $term;
+                };
+                //Tags from above
+                $this->rule_defined_key = ['X','PU','CD','OD','NT','URL','FW'];
+                break;
+            default:
+                $this->lang = $lang;
+        }
+    }
+    /**
+     * __call  for calling dynamic methods
+     * @param string $method method of this class to call
+     * @param array $args arguments to pass to method
+     * @return mixed result of method calculation
+     */
+    public function __call($method, $args)
+    {
+        return call_user_func_array($this->$method, $args);
+    }
+    /**
+     *  __get  for getting dynamic variables
+     * @param string $var_name variable to retrieve
+     * @return mixed result of retrieval
+     */
+    public function __get($var_name)
+    {
+        return $this->$var_name;
+    }
+    /**
+     *  __set  for assigning dynamic variables
+     * @param string $var_name variable to assign
+     * @param  mixed $value value to assign to it
+     */
+    public function __set($var_name, $value)
+    {
+        $this->$var_name = $value;
+    }
+    /**
+     * check if the term can be determined by algorithm
+     * then use the determined tag to be the weight key
+     * @param @string $term is the term to be checked
+     * @return right key in feature matrix
+     */
+    
+    public function getKey($term)
+    {
+        if (isset($this->getKeyImpl)) {
+            return $this->getKeyImpl($term);
+        }
+        return $term;
+    }
+    
+    /**
+     * A function that process the trainning data
+     * @param @mixed $text_files can be a file or an array of file names
+     * @return @array of seperated sentences, each sentenfce have the format of
+     *               [[words...],[tags...]]
+     * Currently, the trainning data needs to fit CTB format:
+     * term followed by a underscore and followed by the tag
+     * e.g. "新_VA 的_DEC 南斯拉夫_NR 会国_NN"
+     * To adapt to other language, some modifications are needed
+     */
+    public static function processTexts($text_files)
+    {
+        $ret=[];
+        foreach($text_files as $text_file) {
+            if (file_exists($text_file)) {
+                $fn = fopen($text_file,"r");
+                while(! feof($fn))  {
+                    $line = fgets($fn);
+                    if(strpos($line, '<') !== false) {
+                        continue;
+                    }
+                    $word_tag_pairs = preg_split("/[\s　]+/u", $line);
+                    if (!count($word_tag_pairs)) {
+                        continue;
+                    }
+                    $ret[]=[];
+                    $ret[count($ret)-1][0]=[];
+                    $ret[count($ret)-1][1]=[];
+                    foreach ($word_tag_pairs as $word_tag_pair) {
+                        $t = explode("_", $word_tag_pair);
+                        if (count($t) == 2) {
+                            $ret[count($ret)-1][0][] = $t[0];
+                            $ret[count($ret)-1][1][] = $t[1];
+                        }
+                    }
+                }
+                fclose($fn);
+            }
+        }
+        return $ret;
+    }
+    
+    /**
+    * Function to train a data
+    * Notice: This function might run very long time, depending on training set
+    * @param @mixed $text_files are training data
+    *               can be a file or an array of file names
+    * @param @float $learning_rate 
+    * @param @int  $max_epoch 1200 might be a good one,
+                   the weight will overfit if it's greater than this number
+    * @parama @bool $resume if true, read the weight file and continue training
+                            if false, start from beginning
+    */
+    public function train($text_files, $learning_rate=0.1,
+                        $max_epoch = 1200, $resume=false)
+    {
+        if (is_string($text_files)) {
+            $text_files = [$text_files];
+        }
+        echo "Reading files\n";
+        // term_tag_sentences[sentence#]=[[words...],[tags...]]
+        $term_tag_sentences = self::processTexts($text_files);
+        if ($resume) {
+            echo "Loading weights... ";
+            $this->load_weight();
+            echo "ok\n";
+        } else {
+            $this->w=[];
+            $this->tag_set=[];
+            $tag_index = 0;
+            if (isset($this->rule_defined_key)) {
+                foreach($this->rule_defined_key as $k) {
+                    $this->tag_set[$k] = $tag_index++;
+                }
+            }
+            for ($i = -4; $i <= -1; $i++) {
+                $this->w[$i] = [];
+            }
+            foreach ($term_tag_sentences as $term_tag_pairs) {
+                $terms=$term_tag_pairs[0];
+                $tags=$term_tag_pairs[1];
+                for ($i = 0; $i < count($terms); $i++) {
+                    if (!isset($this->tag_set[$tags[$i]])) {
+                        $this->tag_set[$tags[$i]] = $tag_index++;
+                    }
+                    $k = $this->getIndex($i,$terms);
+                    
+                    if (!isset($this->w[$k])) {
+                        $this->w[$k] = [];
+                    }
+                    
+                }
+            }
+            foreach (array_keys($this->w) as $key) {
+                for ($i=-2; $i<=2;$i++) {
+                    $this->w[$key][$i] = [];
+                    foreach($this->tag_set as $possiable_tag => $tag_index) {
+                        $this->w[$key][$i][$tag_index] = 0;
+                    }
+                }
+                if ($this->packed) {
+                    $this->w[$key] = $this->pack_w($key);
+                }
+            }
+            foreach($this->tag_set as $possiable_tag => $tag_index) {
+                $this->b[$tag_index] = 0;
+            }
+            if ($this->packed) {
+                $this->b = $this->pack_b();
+            }
+        }
+        echo "Training\n";
+        //train the weight
+        $cross_entropy_loss = 1;
+        for ($epoch = 0; $epoch < $max_epoch; $epoch++) {
+            $time = time();
+            $dy_dw = [];
+            $dy_dw_n = [];
+            $pre_cross_entropy_loss = $cross_entropy_loss;
+            $cross_entropy_loss = 0;
+            $cross_entropy_loss_n = 0;
+            
+            $dy_db=[];
+            $dy_db_n=[];
+            for($i = 0; $i < count($this->tag_set); $i++) {
+                $dy_db[$i] = 0;
+                $dy_db_n[$i] = 0;
+            }
+            //for each sentence
+            foreach ($term_tag_sentences as $term_tag_pairs) {
+                $terms=$term_tag_pairs[0];
+                $tags=$term_tag_pairs[1];
+                for ($i = 0; $i < count($terms); $i++) {
+                    $k=[];
+                    
+                    for ($j=-2; $j<=2;$j++) {
+                        $k[$j]= $this->getIndex($i+$j,$terms);
+                    }
+                    foreach ($this->tag_set as $possiable_tag => $tag_index) {
+                        $equality = $possiable_tag == $tags[$i] ? 1 : 0;
+                        $sum=0;
+                        for ($j=-2; $j<=2;$j++) {
+                            $sum += $this->getW($k[$j],$j,$tag_index);
+                        }
+                        $sum += $this->getB($tag_index);
+                        $sigmoid = 1 / (1 + exp(-1 * $sum));
+                        for ($j=-2; $j<=2;$j++) {
+                            if (!isset($dy_dw[$k[$j]])) {
+                                $dy_dw[$k[$j]] = [];
+                                $dy_dw_n[$k[$j]] = [];
+                            }
+                            if (!isset($dy_dw[$k[$j]][$j])) {
+                                $dy_dw[$k[$j]][$j] = [];
+                                $dy_dw_n[$k[$j]][$j] = [];
+                            }
+                            if (!isset($dy_dw[$k[$j]][$j][$tag_index])) {
+                                $dy_dw[$k[$j]][$j][$tag_index] = 0;
+                                $dy_dw_n[$k[$j]][$j][$tag_index] = 0;
+                            }
+                            
+                            $dy_dw[$k[$j]][$j][$tag_index] +=
+                                ($sigmoid - $equality);
+                            $dy_dw_n[$k[$j]][$j][$tag_index] += 1;
+                            
+                        }
+                        //dy_db
+                        $dy_db[$tag_index] += ($sigmoid - $equality);
+                        $dy_db_n[$tag_index] += 1;
+                        $cross_entropy_loss+=
+                            - $equality*log($sigmoid)
+                            - (1-$equality)*log(1-$sigmoid);
+                        $cross_entropy_loss_n++;
+                    }
+                }
+            }
+            $cross_entropy_loss /= $cross_entropy_loss_n;
+            $duration = time() - $time;
+            echo "epoch {$epoch} cross_entropy {$cross_entropy_loss}".
+                " Takes {$duration} seconds\n";
+            if ($pre_cross_entropy_loss - $cross_entropy_loss < 0.00001) {
+                break;
+            }
+            
+            
+            foreach ($dy_dw as $i =>$v1) {
+                foreach ($v1 as $j =>$v2) {
+                    foreach ($v2 as $k =>$v3) {
+                        $this->setW($i,$j,$k,
+                        $this->getW($i,$j,$k) -
+                        $dy_dw[$i][$j][$k] /
+                        $dy_dw_n[$i][$j][$k] *
+                        $learning_rate);
+                    }
+                }
+            }
+            foreach ($dy_db as $k =>$v) {
+                $this->setB($k,$this->getB($k)
+                    - $dy_db[$k] / $dy_db_n[$k] * $learning_rate);
+            }
+            
+            if ($epoch % 10 == 9) {
+                $this->save_weight();
+            }
+        }
+
+        $this->save_weight();
+        return true;
+    }
+    
+    /**
+     * The primary function to predit the tag
+     * @param mixed $sentence is an array of segmented words/terms
+     *     or a string with words/terms seperated by space
+     * @return @array of tags 
+     */
+    public function predict($sentence)
+    {
+        if (!is_array($sentence)) {
+            if ($sentence == "") {
+                $terms=[];
+            } else {
+                $terms=preg_split("/[\s]+/",$sentence);
+            }
+        } else {
+            $terms=$sentence;
+        }
+        if (!count($terms)) {
+            return [];
+        }
+        if (!$this->w) {
+            $this->load_weight();
+        }
+        $ret = [];
+        for($i = 0; $i < count($terms); $i++) {
+            $term = $terms[$i];
+            $score =[];
+            $key=$this->getKey($term);
+            foreach($this->tag_set as $possiable_tag => $tag_index) {
+                $score[$possiable_tag]=0;
+                for ($j=-2; $j <=2; $j++) {
+                    $k=$this->getIndex($i+$j, $terms);
+                    if (isset($this->w[$k])) {
+                        $score[$possiable_tag] += 
+                                $this->getW($k,$j,$tag_index);
+                    }
+                }
+                $score[$possiable_tag] += $this->getB($tag_index);
+                $score[$possiable_tag]
+                    += 1 / (1 + exp(-1 * $score[$possiable_tag]));
+            }
+            $ret[]=array_keys($score, max($score))[0];
+        }
+        return $ret;
+    }
+    
+    /**
+     * Wrap function for predict
+     * @param $texts to be a @string of texts
+     * @param $return_string is a boolean to determing if the user
+     *        want it to out put to stdout or a return value
+     * @return @string if $return_string is true;
+               @boolean true otherwise
+     * e.g. 中国_NR 人民_NN 将_AD 满怀信心_VV
+            地_DEV 开创_VV 新_VA 的_DEC 业绩_NN 。_PU
+     */
+    public function tag($texts, $return_string=false) {
+        if ($return_string) {
+            $ret = "";
+        }
+        $sentences = preg_split('/\r\n|\r|\n/', $texts);
+        foreach($sentences as $sentence) {
+            $sentence=explode(" ",trim($sentence));
+            $term_pos = $this->predict($sentence);
+            for($i = 0; $i < count($term_pos); $i++) {
+                $term_pos[$i]=$sentence[$i]."_".$term_pos[$i];
+            }
+            $t = join(" ", $term_pos);
+            if ($return_string) {
+                $ret .= $t;
+            } else {
+                echo $t, "\n";
+            }
+        }
+        if ($return_string) {
+            return $ret;
+        } else {
+            return true;
+        }
+    }
+    
+    /**
+     * A list of private helper functions
+     * Given a setence ($term), find the key at position $index
+     */
+    private function getIndex($index, $terms) {
+        if ($index < 0) $k = $index - 2;
+        else if ($index >= count($terms)) {
+            $k = $index - count($terms) - 2;
+        }
+        else {
+            $k = $this->getKey($terms[$index]);
+        }
+        return $k;
+    }
+    
+    /**
+     * Get the bias value for tag
+     */
+    private function getB($tag_index) {
+        if ($this->packed) {
+            return unpack("f",$this->b,$tag_index*4)[1];
+        } else {
+            return $this->b[$tag_index];
+        }
+    }
+    /**
+     * Set the bias value for tag
+     */
+    private function setB($tag_index, $value) {
+        if ($this->packed) {
+            $this->b = substr_replace($this->b,pack("f",$value),$tag_index*4,4);
+        } else {
+            $this->b[$tag_index]=$value;
+        }
+    }
+    
+    /**
+     * Get the weight value for term at postion for tag
+     */
+    private function getW($term, $position, $tag_index) {
+        if ($this->packed) {
+            return unpack("f",$this->w[$term],
+                4*($position+2)*count($this->tag_set)+$tag_index*4)[1];
+        } else {
+            return $this->w[$term][$position][$tag_index];
+        }
+    }
+    /**
+     * Set the weight value for term at postion for tag
+     */
+    private function setW($term, $position, $tag_index, $value) {
+        if ($this->packed) {
+            $this->w[$term] = substr_replace($this->w[$term],
+                pack("f",$value),
+                4*($position+2)*count($this->tag_set)+$tag_index*4,
+                4);
+        } else {
+            $this->w[$term][$position][$tag_index] = $value;
+        }
+    }
+    /**
+     * save the trained weight to disk
+     */
+    private function save_weight()
+    {
+        $out_file = C\LOCALE_DIR . "/{$this->lang}/resources/pos_weight.txt.gz";
+        $out = [];
+        if ($this->packed) {
+            $out["w"] = $this->w;
+            $out["b"] = $this->b;
+        } else {
+            $out["w"]=[];
+            foreach(array_keys($this->w) as $key) {
+                $out["w"][$key] = $this->pack_w($key);
+            }
+            $out["b"] = $this->pack_b();
+        }
+        $out["tag_set"] = $this->tag_set;
+        echo "Saving...";
+        file_put_contents($out_file,
+            gzencode(serialize($out),9));
+        echo " ok\n";
+    }
+    /**
+     * load the trained weight from disk
+     */
+    private function load_weight() {
+        $dic_file
+            = C\LOCALE_DIR . "/{$this->lang}/resources/pos_weight.txt.gz";
+        if (!file_exists($dic_file)) {
+            echo "$dic_file does not exist!";
+            exit();
+        }
+        $f = unserialize(gzdecode(file_get_contents($dic_file))
+            ,['allowed_classes' => false]);
+        $this->w=$f["w"];
+        $this->b=$f["b"];
+        $this->tag_set=$f["tag_set"];
+        if (!$this->packed) {
+            foreach(array_keys($this->w) as $key) {
+                $this->w[$key] = $this->unpack_w($key);
+            }
+            $this->b = $this->unpack_b($this->b);
+        }
+    }
+    
+    /**
+     * Pack the bias
+     */
+    private function pack_b() {
+        return pack("f*", ...$this->b);
+    }
+    /**
+     * Unpack the bias
+     */
+    private function unpack_b() {
+        return array_merge(unpack("f".strval(count($this->tag_set)),$this->b));
+    }
+    /**
+     * Pack the weight
+     */
+    private function pack_w($key) {
+        $bin_str = "";
+        foreach($this->w[$key] as $i => $t) {
+            $bin_str .= pack("f*", ...$t);
+        }
+        return $bin_str;
+    }
+    /**
+     * Unpack the weight
+     */
+    private function unpack_w($key) {
+        $tmp = [];
+        for ($i = 0; $i < 5; $i++) {
+            $tmp[$i-2] = array_merge(unpack("f".strval(count($this->tag_set)),
+                $this->w[$key], 4*$i*count($this->tag_set)));
+        }
+        return $tmp;
+    }
+}
diff --git a/src/library/StochasticTermSegmenter.php b/src/library/StochasticTermSegmenter.php
index 474be126..93167b08 100644
--- a/src/library/StochasticTermSegmenter.php
+++ b/src/library/StochasticTermSegmenter.php
@@ -28,7 +28,6 @@
  */
 namespace seekquarry\yioop\library;
 
-use seekquarry\yioop\library\Trie;
 use seekquarry\yioop\locale\zh_CN\resources as ZH;
 use seekquarry\yioop\configs as C;
 /**
@@ -66,7 +65,7 @@ class StochasticTermSegmenter
      */
     private $cache_pct;
     /**
-     * Used in memory saving mode
+     * Cache. Will have runtime data for the segmentation
      * @var array
      */
     private $cache=[];
@@ -111,9 +110,9 @@ class StochasticTermSegmenter
         switch($lang)
         {
             case "zh_CN":
-                case "zh-CN":
+            case "zh-CN":
                 $this->lang = "zh_CN";
-                $this->non_char_preg = "/^[^\p{Han}]+$/u";
+                //$this->non_char_preg = "/^[^\p{Han}]+$/u";
                 /*
                  * Check if the term passed in is an exception term
                  */
@@ -127,7 +126,14 @@ class StochasticTermSegmenter
                  */
                 $this->isPunctuationImpl = function($term)
                 {
-                    return preg_match(ZH\Tokenizer::$punctuation_preg, $term);
+                    return ZH\Tokenizer::isPunctuation($term);
+                };
+                /*
+                 * Check if all the chars in the term is NOT current language
+                 */
+                $this->isNotCurrentLangImpl = function($term)
+                {
+                    return ZH\Tokenizer::isNotCurrentLang($term);
                 };
                 break;
             case "ja":
@@ -208,7 +214,9 @@ class StochasticTermSegmenter
      */
     public function notCurrentLang($term)
     {
-        return preg_match($this->non_char_preg, $term);
+        if (isset($this->isNotCurrentLangImpl))
+            return $this->isNotCurrentLangImpl($term);
+        return false;
     }
     /**
      * Generate a term dictionary file for later segmentation
@@ -219,7 +227,7 @@ class StochasticTermSegmenter
      * @return bool true if success
      */
     public function train($text_files, $format = "default",
-        $max_num_term_weights = 40000)
+        $max_num_term_weights = 40000000)
     {
         $ctb_fmt=false;
         switch ($format) {
@@ -254,9 +262,7 @@ class StochasticTermSegmenter
                             && !$this->notCurrentLang($word)) {
                             if (!empty($dictionary[$word])) {
                                 $dictionary[$word]++;
-                            } else if (count($dictionary) <
-                                $max_num_term_weights
-                                && mb_strlen($word) < 7) {
+                            } else if (mb_strlen($word) < 7) {
                                 $dictionary[$word] = 1;
                             }
                         }
@@ -319,6 +325,8 @@ class StochasticTermSegmenter
                     }
                 }
                 fclose($fh);
+            } else {
+                echo "cannot open $text_file\n";
             }
         }
         if ($return_string) {
diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index 02113c79..9e789661 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -57,23 +57,39 @@ class Tokenizer
         '与', '次', '狗', '决', '金', '史', '姆', '部', '正在', '活', '刚',
         '回家', '贝', '如何', '须', '战', '不會', '夫', '喂', '父', '亚', '肯定',
         '女孩', '世界'];
+    /**
+     * regular expression to determine if the None of the char in this
+     * term is in current language.
+     * @var string
+     */
+    public static $non_char_preg = "/^[^\p{Han}]+$/u";
     /**
      * The dictionary of characters can be used as Chinese Numbers
      * @string
      */
     public static $num_dict =
-       "1234567890○零一二三四五六七八九十百千万亿".
+       "1234567890○〇零一二两三四五六七八九十百千万亿".
        "０１２３４５６７８９壹贰叁肆伍陆柒捌玖拾廿卅卌佰仟萬億";
     /**
      * Dots used in Chinese Numbers
      * @string
      */
-    public static $dot = "\.．";
+    public static $dot = "\.．点";
     /**
      * A list of characters can be used at the end of numbers
      * @string
      */
     public static $num_end = "％%";
+    /**
+     * Exception words of the regex found by functions:
+     * isCardinalNumber, isOrdinalNumber, isDate
+     * ex. "十分" in most of time means "very", but it will
+     * be determined to be "10 minutes" by the function so we
+     * need to remove it
+     * @array of string
+     */
+    public static $exception_list= ["十分","一","一点","千万",
+    "万一", "一一", "拾", "一时", "千千", "万万", "陆"];
     /**
      * A list of characters can be used as Chinese punctuations
      * @string
@@ -82,7 +98,12 @@ class Tokenizer
     "/^([\x{2000}-\x{206F}\x{3000}-\x{303F}\x{FF00}-\x{FF0F}" .
     "\x{FF1A}-\x{FF20}\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}" .
     "\x{FFE0}-\x{FFEE}\x{21}-\x{2F}\x{21}-\x{2F}" .
-    "\x{3A}-\x{40}\x{5B}-\x{60}]|[\—]+|[\.]+)$/u";
+    "\x{3A}-\x{40}\x{5B}-\x{60}\x{25cf}])\\1*$/u";
+    /**
+     * Stochastic Term Segmenter instance
+     * @object
+     */
+    private static $stochasticTermSegmenter;
     /**
      * Removes the stop words from the page (used for Word Cloud generation
      * and language detection)
@@ -117,8 +138,8 @@ class Tokenizer
                 ['/^\d+$/', '/^[a-zA-Z]+$/']);
                 break;
             case("STS"):
-                $segmenter = new StochasticTermSegmenter("zh_CN");
-                return $segmenter->segmentText($pre_segment,true);
+                return self::getStochasticTermSegmenter()
+                        ->segmentText($pre_segment,true);
                 break;
         }
     }
@@ -127,25 +148,73 @@ class Tokenizer
      */
     public static function isCardinalNumber($term)
     {
-        return preg_match("/^[" . self::$num_dict .
-            self::$dot . "]+[" . self::$num_end .
-            "]?[余餘]?[百千万亿佰仟萬億]?$/u", $term);
+        return !in_array($term,self::$exception_list)
+            && preg_match("/^([" . self::$num_dict .
+            "]+([" . self::$dot . "][" .self::$num_dict . 
+            "]+)?[" . self::$num_end .
+            "]?[余餘多]?[百千万亿佰仟萬億]*)".
+            "$|^([".self::$num_dict."]+分之[" . self::$num_dict .
+            "]+([" . self::$dot . "][" .self::$num_dict . 
+            "]+)?)$/u", $term);
     }
     /*
      * Check if the term passed in is a Ordinal Number
      */
     public static function isOrdinalNumber($term)
     {
-        return preg_match("/^第[" . self::$num_dict .
-        "]*$/u", $term);
+        return !in_array($term,self::$exception_list)
+            && preg_match("/^第[" . self::$num_dict .
+            "]*$/u", $term);
     }
     /*
      * Check if the term passed in is a date
      */
     public static function isDate($term)
     {
-        return preg_match("/^[" . self::$num_dict .
-        "]+(年|年代|月|日|时|小时|時|小時|" .
-        "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
+        return !in_array($term,self::$exception_list)
+            && preg_match("/^[" . self::$num_dict .
+            "]+(年|年代|月|日|时|小时|時|小時|" .
+            "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
+    }
+    /*
+     * Check if the term is a punctuation
+     */
+    public static function isPunctuation($term)
+    {
+        return preg_match(self::$punctuation_preg, $term);
+    }
+    /**
+     * Check if all the chars in the term is NOT current language
+     * @param $term is a string that to be checked
+     * @return bool true if all the chars in $term is NOT current language
+     *         false otherwise
+     */
+    public static function isNotCurrentLang($term)
+    {
+        return preg_match(self::$non_char_preg, $term);
+    }
+    /*
+     * Create stochastic term segmenter
+     */
+    public static function createStochasticTermSegmenter($cache_pct=0.06)
+    {
+        self::$stochasticTermSegmenter
+            = new StochasticTermSegmenter("zh_CN", $cache_pct);
+    }
+    /*
+     * Destory stochastic term segmenter
+     */
+    public static function destoryStochasticTermSegmenter()
+    {
+        self::$stochasticTermSegmenter = null;
+    }
+    /*
+     * Get the segmenter instance
+     */
+    public static function getStochasticTermSegmenter() {
+        if (!self::$stochasticTermSegmenter) {
+            self::createStochasticTermSegmenter();
+        }
+        return self::$stochasticTermSegmenter;
     }
 }
-- 
2.21.0.windows.1

