_binary From 4fefbb0dd4dc004ef56792c36919d348df285708 Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Mon, 30 Mar 2020 12:23:48 -0700
Subject: [PATCH] POS tagger and NER. r=chris

---
 .../ContextWeightedNamedEntityRecognizer.php  | 608 ++++++++++++++++++
 src/library/ContextWeightedPosTagger.php      | 599 +++++++++++++++++
 src/library/StochasticTermSegmenter.php       |  75 ++-
 src/locale/zh_CN/resources/Tokenizer.php      | 139 +++-
 4 files changed, 1381 insertions(+), 40 deletions(-)
 create mode 100644 src/library/ContextWeightedNamedEntityRecognizer.php
 create mode 100644 src/library/ContextWeightedPosTagger.php

diff --git a/src/library/ContextWeightedNamedEntityRecognizer.php b/src/library/ContextWeightedNamedEntityRecognizer.php
new file mode 100644
index 00000000..0790fdab
--- /dev/null
+++ b/src/library/ContextWeightedNamedEntityRecognizer.php
@@ -0,0 +1,608 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * @author Xianghong Sun sxh19911230@gmail.com
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+ 
+namespace seekquarry\yioop\library;
+
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\locale\zh_CN\resources as ZH;
+
+/**
+ * Machine learning based NER tagger. Typically, ContextWeightedNERTagger.php
+ * can train the language with some dataset and predict
+ * the tag given a list of word.
+ *
+ * @author Xianghong Sun
+ */
+
+class ContextWeightedNamedEntityRecognizer
+{
+    /**
+     * Current Language, only tested on Simplified Chinese
+     * Might be extensable for other languages in the furture
+     * @var string
+     */
+    public $lang;
+    /**
+     * The word weight feature
+     * y = wx + b
+     * Generized by training method
+     * @var array
+     */
+    public $word_feature;
+    /**
+     * The tag weight feature
+     * y = wx + b
+     * Generized by training method
+     * @var array
+     */
+    public $tag_feature;
+    /**
+     * The bias
+     * y = wx + b
+     * Generized by training method
+     * @var array
+     */
+    public $bias;
+     /**
+     * All Possiable tag set
+     * Generized by training method
+     * @var associative array [tag => tag index]
+     */
+    private $tag_set;
+     /**
+     * The constructer of the pos tagger
+     * To extend to other languages, some work are needed:
+     * Define $this->getKeyImpl, $this->rule_defined_key
+     * See Chinese example.
+     * @param @string $lang describes current langauge
+     * @param @book $packed describes how weight and bias would look like
+     */
+    public function __construct($lang)
+    {
+        switch($lang) {
+            case("zh_CN"):
+            case("zh-CH"):
+                $this->lang = "zh_CN";
+                break;
+            default:
+                $this->lang = $lang;
+        }
+    }
+    
+    /**
+     * A function that process the trainning data
+     * @param @mixed $text_files can be a file or an array of file names
+     * @return @array of seperated sentences, each sentenfce have the format of
+     *               [[words...],[tags...]]
+     * Data format MSRA:
+     * 我们/o 是/o 受到/o 郑振铎/nr 先生/o 、/o 阿英/nr 先生/o 著作/o 的/o
+     * 启示/o ，/o 从/o 个人/o 条件/o 出发/o ，/o 瞄准/o 现代/o 出版/o 史/o
+     * 研究/o 的/o 空白/o ，/o 重点/o 集/o 藏/o 解放区/o 、/o 国民党/nt 毁/o
+     * 禁/o 出版物/o 。/o 
+     * To adapt to other language, some modifications are needed
+     */
+    public static function processTexts($text_files, $term_tag_splier="/",
+        $term_process = null, $tag_process = null)
+    {
+        $ret=[];
+        foreach($text_files as $text_file) {
+            if (file_exists($text_file)) {
+                $fn = fopen($text_file,"r");
+                while(! feof($fn))  {
+                    $line = fgets($fn);
+                    if(strpos($line, '<') !== false) {
+                        continue;
+                    }
+                    $word_tag_pairs = preg_split("/[\s　]+/u", $line);
+                    if (!count($word_tag_pairs)) {
+                        continue;
+                    }
+                    $ret[]=[];
+                    $ret[count($ret)-1][0]=[];
+                    $ret[count($ret)-1][1]=[];
+                    foreach ($word_tag_pairs as $word_tag_pair) {
+                        $t = explode("/", $word_tag_pair);
+                        //echo $word_tag_pair;
+                        //print_r($t);
+                        if (count($t) == 2) {
+                            $tag = $tag_process ? $tag_process($t[1]) : $t[1];
+                            foreach(preg_split('//u', $t[0], null, PREG_SPLIT_NO_EMPTY) as $ch) {
+                                $ret[count($ret)-1][0][] =
+                                    $term_process ? $term_process($ch) : $ch;
+                                $ret[count($ret)-1][1][] = $tag;
+                            }
+                        }
+                    }
+                }
+                fclose($fn);
+            }
+        }
+        return $ret;
+    }
+    
+    /**
+    * Function to train a data
+    * Notice: This function might run very long time, depending on training set
+    * @param @mixed $text_files are training data
+    *               can be a file or an array of file names
+    * @param @float $learning_rate 
+    * @param @int  $max_epoch 1200 might be a good one,
+                   the weight will overfit if it's greater than this number
+    * @param @function $term_process is a preporcess on term before training
+    * @param @function $tag_process is a preporcess on tag before training
+    */
+    public function train($text_files, 
+                          $learning_rate=0.1, $max_epoch = 1200,
+                          $term_process = null, $tag_process = null)
+    {
+        if (is_string($text_files)) {
+            $text_files = [$text_files];
+        }
+        echo "Reading files\n";
+        // term_tag_sentences[sentence#]=[[words...],[tags...]]
+        $term_tag_sentences = self::processTexts($text_files,
+            $term_process, $tag_process);
+        $this->word_feature=[];
+        $this->tag_set=[];
+        $tag_index = 0;
+        for ($i = -4; $i <= -1; $i++) {
+            $this->word_feature[$i] = [];
+        }
+        foreach ($term_tag_sentences as $term_tag_pairs) {
+            $terms=$term_tag_pairs[0];
+            $tags=$term_tag_pairs[1];
+            $this->tag_feature["start"]=[];
+            $this->tag_feature["start-start"]=[];
+            for ($i = 0; $i < count($terms); $i++) {
+                if (!isset($this->tag_set[$tags[$i]])) {
+                    $this->tag_set[$tags[$i]] = $tag_index++;
+                }
+                if ($i == 0) {}
+                else if ($i == 1) {
+                    if (!isset($this->tag_feature["start-".$tags[$i-1]])) {
+                        $this->tag_feature["start-".$tags[$i-1]]=[];
+                    }
+                    if (!isset($this->tag_feature[$tags[$i-1]])) {
+                        $this->tag_feature[$tags[$i-1]]=[];
+                    }
+                } else {
+                    if (!isset($this->tag_feature[$tags[$i-2]."-".$tags[$i-1]])) {
+                        $this->tag_feature[$tags[$i-2]."-".$tags[$i-1]]=[];
+                    }
+                    if (!isset($this->tag_feature[$tags[$i-1]])) {
+                        $this->tag_feature[$tags[$i-1]]=[];
+                    }
+                }
+                
+                if (!isset($this->word_feature[$terms[$i]])) {
+                    $this->word_feature[$terms[$i]] = [];
+                }
+            }
+        }
+        foreach (array_keys($this->word_feature) as $key) {
+            for ($i=-2; $i<=2;$i++) {
+                if (!isset($this->word_feature[$key][$i])) {
+                    $this->word_feature[$key][$i] = [];
+                }
+                foreach($this->tag_set as $possiable_tag => $tag_index) {
+                    if (!isset($this->word_feature[$key][$i][$tag_index])) {
+                        $this->word_feature[$key][$i][$tag_index] = 0;
+                    }
+                }
+            }
+        }
+        foreach (array_keys($this->tag_feature) as $key) {
+            foreach($this->tag_set as $possiable_tag => $tag_index) {
+                if (!isset($this->tag_feature[$key][$tag_index])) {
+                    $this->tag_feature[$key][$tag_index] = 0;
+                }
+            }
+        }
+        foreach($this->tag_set as $possiable_tag => $tag_index) {
+            if (!isset($this->bias[$tag_index])) {
+                $this->bias[$tag_index] = 0;
+            }
+        }
+        echo "Training...\n";
+        //train the weight
+        $cross_entropy_loss = 1;
+        $pre_cross_entropy_loss = 2;
+        for ($epoch = 0; ($epoch < $max_epoch) &&
+            $pre_cross_entropy_loss - $cross_entropy_loss > 0.000001; $epoch++) {
+            $this->min_w=0;
+            $this->max_w=0;
+            $time = time();
+            $dy_dw = [];
+            $dy_dw_n = [];
+            $pre_cross_entropy_loss = $cross_entropy_loss;
+            $cross_entropy_loss = 0;
+            $cross_entropy_loss_n = 0;
+            
+            $dy_db=[];
+            $dy_db_n=[];
+            
+            $dy_dt=[];
+            $dy_dt_n=[];
+            for($i = 0; $i < count($this->tag_set); $i++) {
+                $dy_db[$i] = 0;
+                $dy_db_n[$i] = 0;
+            }
+            //for each sentence
+            foreach ($term_tag_sentences as $term_tag_pairs) {
+                $terms=$term_tag_pairs[0];
+                $tags=$term_tag_pairs[1];
+                for ($i = 0; $i < count($terms); $i++) {
+                    $k=[];
+                    for ($j=-2; $j<=2;$j++) {
+                        $k[$j]= $this->getIndex($i+$j,$terms);
+                    }
+                    foreach ($this->tag_set as $possiable_tag => $tag_index) {
+                        $equality = $possiable_tag == $tags[$i] ? 1 : 0;
+                        $sum=0;
+                        //5 words including itself
+                        for ($j=-2; $j<=2;$j++) {
+                            $sum += $this->word_feature[$k[$j]][$j][$tag_index];
+                        }
+                        //previous 2 tags
+                        if ($i == 0) {
+                            $tf1="start";
+                            $tf2="start-start";
+                        } else if ($i == 1) {
+                            $tf1=$tags[$i-1];
+                            $tf2="start-".$tags[$i-1];
+                        } else {
+                            $tf1=$tags[$i-1];
+                            $tf2=$tags[$i-2]."-".$tags[$i-1];
+                        }
+                        $sum += $this->tag_feature[$tf1][$tag_index];
+                        $sum += $this->tag_feature[$tf2][$tag_index];
+                        //bias
+                        $sum += $this->bias[$tag_index];
+                        $sigmoid = 1 / (1 + exp(-1 * $sum));
+                        for ($j=-2; $j<=2;$j++) {
+                            if (!isset($dy_dw[$k[$j]])) {
+                                $dy_dw[$k[$j]] = [];
+                                $dy_dw_n[$k[$j]] = [];
+                            }
+                            if (!isset($dy_dw[$k[$j]][$j])) {
+                                $dy_dw[$k[$j]][$j] = [];
+                                $dy_dw_n[$k[$j]][$j] = [];
+                            }
+                            if (!isset($dy_dw[$k[$j]][$j][$tag_index])) {
+                                $dy_dw[$k[$j]][$j][$tag_index] = 0;
+                                $dy_dw_n[$k[$j]][$j][$tag_index] = 0;
+                            }
+                            
+                            $dy_dw[$k[$j]][$j][$tag_index] +=
+                                ($sigmoid - $equality);
+                            $dy_dw_n[$k[$j]][$j][$tag_index] += 1;
+                            
+                        }
+                        //dy_dt
+                        if (!isset($dy_dt[$tf1])) {
+                            $dy_dt[$tf1] = [];
+                            $dy_dt_n[$tf1] = [];
+                        }
+                        if (!isset($dy_dt[$tf1][$tag_index])) {
+                            $dy_dt[$tf1][$tag_index] = 0;
+                            $dy_dt_n[$tf1][$tag_index] = 0;
+                        }
+                        if (!isset($dy_dt[$tf2])) {
+                            $dy_dt[$tf2] = [];
+                            $dy_dt_n[$tf2] = [];
+                        }
+                        if (!isset($dy_dt[$tf2][$tag_index])) {
+                            $dy_dt[$tf2][$tag_index] = 0;
+                            $dy_dt_n[$tf2][$tag_index] = 0;
+                        }
+                        $dy_dt[$tf1][$tag_index] += ($sigmoid - $equality);
+                        $dy_dt_n[$tf1][$tag_index] += 1;
+                        $dy_dt[$tf2][$tag_index] += ($sigmoid - $equality);
+                        $dy_dt_n[$tf2][$tag_index] += 1;
+                        //dy_db
+                        $dy_db[$tag_index] += ($sigmoid - $equality);
+                        $dy_db_n[$tag_index] += 1;
+                        $cross_entropy_loss+=
+                            - $equality*log($sigmoid)
+                            - (1-$equality)*log(1-$sigmoid);
+                        $cross_entropy_loss_n++;
+                    }
+                }
+            }
+            $cross_entropy_loss /= $cross_entropy_loss_n;
+            $duration = time() - $time;
+            echo "epoch {$epoch} cross_entropy {$cross_entropy_loss}".
+                " Takes {$duration} seconds\n";
+            foreach ($dy_dw as $i =>$v1) {
+                foreach ($v1 as $j =>$v2) {
+                    foreach ($v2 as $k =>$v3) {
+                        $this->word_feature[$i][$j][$k] -=
+                            $dy_dw[$i][$j][$k] /
+                            $dy_dw_n[$i][$j][$k] *
+                            $learning_rate;
+                        if ($this->word_feature[$i][$j][$k] < $this->min_w) {
+                            $this->min_w = $this->word_feature[$i][$j][$k];
+                        }
+                        if ($this->word_feature[$i][$j][$k] > $this->max_w) {
+                            $this->max_w = $this->word_feature[$i][$j][$k];
+                        }
+                    }
+                }
+            }
+            foreach ($dy_dt as $i => $v1) {
+                foreach ($v1 as $j => $v2) {
+                    $this->tag_feature[$i][$j] -=
+                        $dy_dt[$i][$j] /
+                        $dy_dt_n[$i][$j] *
+                        $learning_rate;
+                }
+            }
+            foreach ($dy_db as $k => $v) {
+                $this->bias[$k]-=
+                    $dy_db[$k] / 
+                    $dy_db_n[$k] * 
+                    $learning_rate;
+            }
+            if ($epoch % 10 == 9 ) {
+                $this->save_weight();
+            }
+        }
+        $this->save_weight();
+        return true;
+    }
+    /**
+     * The primary function to predit the tag
+     * @param mixed $sentence is an array of segmented words/terms
+     *     or a string needs to be splited by $splitter
+     * @param function $splitter to process $sentence if $sentence
+     *                 is a string
+     * @return @array all predicted named entities with its tag
+     *                ex. [["郑振铎","nr"],["国民党","nt"]]
+     */
+    public function predict($sentence, $delimiter="",$splitter=null)
+    {
+        if (!is_array($sentence)) {
+            if ($sentence == "") {
+                $terms=[];
+            } else {
+                $terms=preg_split("/[\s]+/",$sentence);
+            }
+        } else {
+            $terms=$sentence;
+        }
+        if (!count($terms)) {
+            return [];
+        }
+        if (!$this->word_feature) {
+            $this->load_weight();
+        }
+        $result = [];
+        for($i = 0; $i < count($terms); $i++) {
+            $term = $terms[$i];
+            $score =[];
+            foreach($this->tag_set as $possiable_tag => $tag_index) {
+                $score[$possiable_tag]=0;
+                for ($j=-2; $j <=2; $j++) {
+                    $k=$this->getIndex($i+$j, $terms);
+                    if (isset($this->word_feature[$k])) {
+                        $score[$possiable_tag] += 
+                                $this->getW($k,$j,$tag_index);
+                    }
+                }
+                if ($i == 0) {
+                    $tf1="start";
+                    $tf2="start-start";
+                } else if ($i == 1) {
+                    $tf1=$result[$i-1];
+                    $tf2="start-".$result[$i-1];
+                } else {
+                    $tf1=$result[$i-1];
+                    $tf2=$result[$i-2]."-".$result[$i-1];
+                }
+                $score[$possiable_tag] += $this->getT($tf1,$tag_index);
+                $score[$possiable_tag] += $this->getT($tf2,$tag_index);
+                $score[$possiable_tag] += $this->getB($tag_index);
+            }
+            $result[]=array_keys($score, max($score))[0];
+        }
+        $pre_tag='o';
+        $current_entity=null;
+        $ret=[];
+        for ($i = 0; $i < count($terms); $i++) {
+            if ($pre_tag != $result[$i] && $pre_tag != "o") {
+                if (mb_strlen($current_entity) < 10) {
+                    $ret[]=[$current_entity,$pre_tag];
+                }
+                $current_entity=null;
+            }
+            if ($result[$i] != "o") {
+                if ($current_entity) {
+                    $current_entity.=$delimiter.$terms[$i];
+                } else {
+                    $current_entity=$terms[$i];
+                }
+            }
+            $pre_tag=$result[$i];
+        }
+        return $ret;
+    }
+    
+    /**
+     * A list of private helper functions
+     * Given a setence ($term), find the key at position $index
+     */
+    private function getIndex($index, $terms)
+    {
+        if ($index < 0) $k = $index - 2;
+        else if ($index >= count($terms)) {
+            $k = $index - count($terms) - 2;
+        }
+        else {
+            $k = $terms[$index];
+        }
+        return $k;
+    }
+    
+    /**
+     * save the trained weight to disk
+     */
+    private function save_weight()
+    {
+        $out_file = C\LOCALE_DIR . "/{$this->lang}/resources/ner_weight.txt.gz";
+        $out = [];
+        $out["min_w"] = $this->min_w;
+        $out["max_w"] = $this->max_w;
+        $out["w"]=[];
+        foreach(array_keys($this->word_feature) as $key) {
+            $out["w"][$key] = $this->pack_w($key);
+        }
+        foreach(array_keys($this->tag_feature) as $key) {
+            $out["t"][$key] = $this->pack_t($key);
+        }
+        $out["b"] = $this->pack_b();
+        $out["tag_set"] = $this->tag_set;
+        echo "Saving...";
+        file_put_contents($out_file,
+            gzencode(serialize($out),9));
+        echo " ok\n";
+    }
+    /**
+     * load the trained weight from disk
+     */
+    private function load_weight($trainning_load=false)
+    {
+        $dic_file
+            = C\LOCALE_DIR . "/{$this->lang}/resources/ner_weight.txt.gz";
+        if (!file_exists($dic_file)) {
+            echo "$dic_file does not exist!";
+            exit();
+        }
+        $f = unserialize(gzdecode(file_get_contents($dic_file))
+            ,['allowed_classes' => false]);
+        $this->word_feature=$f["w"];
+        $this->tag_feature=$f["t"];
+        $this->bias=$f["b"];
+        $this->min_w=$f["min_w"];
+        $this->max_w=$f["max_w"];
+        $this->tag_set=$f["tag_set"];
+        if ($trainning_load) {
+            foreach(array_keys($this->word_feature) as $key) {
+                $this->word_feature[$key] = $this->unpack_w($key);
+            }
+            foreach(array_keys($this->tag_feature) as $key) {
+                $this->tag_feature[$key] = $this->unpack_t($key);
+            }
+            $this->bias = $this->unpack_b();
+        }
+    }
+    /**
+     * Pack the bias
+     */
+    private function pack_b()
+    {
+        return pack("f*", ...$this->bias);
+    }
+    /**
+     * Unpack the bias
+     */
+    private function unpack_b()
+    {
+        return array_merge(unpack("f".strval(count($this->tag_set)),$this->bias));
+    }
+    /**
+     * Pack the tag_feature
+     */
+    private function pack_t($key)
+    {
+        return pack("f*", ...$this->tag_feature[$key]);
+    }
+    /**
+     * Unpack the tag_feature
+     */
+    private function unpack_t($key)
+    {
+        return array_merge(unpack("f".strval(count($this->tag_set)),$this->tag_feature[$key]));
+    }
+    /**
+     * Pack the word_feature
+     */
+    private function pack_w($key)
+    {
+        $bin_str = "";
+        foreach($this->word_feature[$key] as $i => $t) {
+            foreach($t as $u) {
+                $v = 65535 * ($u-$this->min_w) / ($this->max_w-$this->min_w);
+                $bin_str .= pack("S", intval($v));
+            }
+        }
+        return $bin_str;
+    }
+    /**
+     * Unpack the word_feature
+     */
+    private function unpack_w($key)
+    {
+        $tmp = [];
+        $size = count($this->tag_set);
+        for ($i = 0; $i < 5; $i++) {
+            $tmp[$i-2] = array_merge(unpack("S".strval($size),
+                $this->word_feature[$key], 2*$i*count($this->tag_set)));
+            for($j = 0; $j < $size; $j++) {
+                $tmp[$i-2][$j] = $tmp[$i-2][$j] / 65535
+                    * ($this->max_w-$this->min_w) + $this->min_w;
+            }
+        }
+        return $tmp;
+    }
+    /**
+     * Get the bias value for tag
+     */
+    private function getB($tag_index)
+    {
+        return unpack("f",$this->bias,$tag_index*4)[1];
+    }
+    /**
+     * Get the bias value for tag
+     */
+    private function getT($key, $tag_index)
+    {
+        return unpack("f",$this->tag_feature[$key],$tag_index*4)[1];
+    }
+    /**
+     * Get the weight value for term at postion for tag
+     */
+    private function getW($term, $position, $tag_index)
+    {
+        $t = unpack("S",$this->word_feature[$term],
+            2*($position+2)*count($this->tag_set)+$tag_index*2)[1]
+            / 65535
+            * ($this->max_w-$this->min_w) + $this->min_w;;
+        return $t;
+    }
+}
diff --git a/src/library/ContextWeightedPosTagger.php b/src/library/ContextWeightedPosTagger.php
new file mode 100644
index 00000000..140097b9
--- /dev/null
+++ b/src/library/ContextWeightedPosTagger.php
@@ -0,0 +1,599 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * @author Xianghong Sun sxh19911230@gmail.com
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+ 
+namespace seekquarry\yioop\library;
+
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\locale\zh_CN\resources as ZH;
+
+/**
+ * Machine learning based POS tagger. Typically, ContextWeightedPosTagger.php
+ * can train the language with some dataset and predict
+ * the tag given a list of word.
+ *
+ * Instruction to add a new language:
+ * Add a switch case in the constructor.
+ * Define the following functions:
+ * getKeyImpl
+ * See the class function 'getKey' for more information
+ *
+ * @author Xianghong Sun
+ */
+ 
+class ContextWeightedPosTagger {
+    /**
+     * Current Language, only tested on Simplified Chinese
+     * Might be extensable for other languages in the furture
+     * @var string
+     */
+    public $lang;
+    /**
+     * The weight for predicting the pos tag
+     * y = wx + b
+     * Generized by training method
+     * @var array
+     */
+    public $w;
+    /**
+     * The bias for predicting the pos tag
+     * y = wx + b
+     * Generized by training method
+     * @var array
+     */
+    public $b;
+    /**
+     * range of w
+     */
+    private $min_w;
+    private $max_w;
+    
+    /**
+     * All Possiable tag set
+     * Generized by training method
+     * @var associative array [tag => tag index]
+     */
+    private $tag_set;
+    /**
+     * Check if all the chars in the term is not current language
+     * @param $term is a string that to be checked
+     * @return true if all the chars in $term is not current language
+     *         false otherwise
+     */
+    public function notCurrentLang($term)
+    {
+        return preg_match("/^[^\p{Han}]+$/u", $term);
+    }
+    /**
+     * The constructer of the pos tagger
+     * To extend to other languages, some work are needed:
+     * Define $this->getKeyImpl, $this->rule_defined_key
+     * See Chinese example.
+     * @param @string $lang describes current langauge
+     * @param @book $packed describes how weight and bias would look like
+     */
+    public function __construct($lang, $packed = true)
+    {
+        //$this->packed = $packed;
+        switch($lang) {
+            case("zh_CN"):
+            case("zh-CH"):
+                $this->lang = "zh_CN";
+                /*
+                 * Some Exception of Tags. Some tags are detemined by ruls.
+                 * e.x. There are infinity amount of Arabic numerals.
+                 */
+                $this->getKeyImpl = function($term) {
+                    $key = ZH\Tokenizer::POSGetKey($term);
+                    return $key ? $this->tag_set[$key] : $term;
+                };
+                //Tags from above
+                $this->rule_defined_key = ['PU','CD','OD','NT','FW'];
+                break;
+            default:
+                $this->lang = $lang;
+        }
+    }
+    /**
+     * __call  for calling dynamic methods
+     * @param string $method method of this class to call
+     * @param array $args arguments to pass to method
+     * @return mixed result of method calculation
+     */
+    public function __call($method, $args)
+    {
+        return call_user_func_array($this->$method, $args);
+    }
+    /**
+     *  __get  for getting dynamic variables
+     * @param string $var_name variable to retrieve
+     * @return mixed result of retrieval
+     */
+    public function __get($var_name)
+    {
+        return $this->$var_name;
+    }
+    /**
+     *  __set  for assigning dynamic variables
+     * @param string $var_name variable to assign
+     * @param  mixed $value value to assign to it
+     */
+    public function __set($var_name, $value)
+    {
+        $this->$var_name = $value;
+    }
+    
+    /**
+     * check if the term can be determined by algorithm,
+     * usually by regualr expression, because there are infinity 
+     * amount of them.
+     * ex. 13th is an ordinal number, 123 is a cardinal number
+     * then use the determined tag to be the weight key
+     * @param @string $term is the term to be checked
+     * @return right key in feature matrix
+     */
+    public function getKey($term)
+    {
+        if (isset($this->getKeyImpl)) {
+            return $this->getKeyImpl($term);
+        }
+        return $term;
+    }
+    
+    /**
+     * A function that process the trainning data
+     * @param @mixed $text_files can be a file or an array of file names
+     * @return @array of seperated sentences, each sentenfce have the format of
+     *               [[words...],[tags...]]
+     * Currently, the trainning data needs to fit CTB format:
+     * term followed by a underscore and followed by the tag
+     * e.g. "新_VA 的_DEC 南斯拉夫_NR 会国_NN"
+     * To adapt to other language, some modifications are needed
+     */
+    public static function processTexts($text_files, $term_tag_splier="_",
+        $term_process = null, $tag_process = null)
+    {
+        $ret=[];
+        foreach($text_files as $text_file) {
+            if (file_exists($text_file)) {
+                $fn = fopen($text_file,"r");
+                while(! feof($fn))  {
+                    $line = fgets($fn);
+                    if(strpos($line, '<') !== false) {
+                        continue;
+                    }
+                    $word_tag_pairs = preg_split("/[\s　]+/u", $line);
+                    if (!count($word_tag_pairs)) {
+                        continue;
+                    }
+                    $ret[]=[];
+                    $ret[count($ret)-1][0]=[];
+                    $ret[count($ret)-1][1]=[];
+                    foreach ($word_tag_pairs as $word_tag_pair) {
+                        $t = explode($term_tag_splier, $word_tag_pair);
+                        
+                        if (count($t) == 2) {
+                            $ret[count($ret)-1][0][] =
+                                $term_process ? $term_process($t[0]) : $t[0];
+                            $ret[count($ret)-1][1][] = 
+                                $tag_process ? $tag_process($t[1]) : $t[1];
+                        }
+                    }
+                }
+                fclose($fn);
+            }
+        }
+        return $ret;
+    }
+    
+    /**
+    * Function to train a data
+    * Notice: This function might run very long time, depending on training set
+    * @param @mixed $text_files are training data
+    *               can be a file or an array of file names
+    * @param @float $learning_rate 
+    * @param @int  $max_epoch 1200 might be a good one,
+                   the weight will overfit if it's greater than this number
+    * @parama @bool $resume if true, read the weight file and continue training
+                            if false, start from beginning
+    */
+    public function train($text_files, $term_tag_splier="_", 
+                          $learning_rate=0.1, $max_epoch = 1200,
+                          $term_process = null, $tag_process = null,
+                          $resume=false)
+    {
+        if (is_string($text_files)) {
+            $text_files = [$text_files];
+        }
+        echo "Reading files\n";
+        // term_tag_sentences[sentence#]=[[words...],[tags...]]
+        $term_tag_sentences = self::processTexts($text_files, $term_tag_splier,
+            $term_process, $tag_process);
+        if ($resume) {
+            echo "Loading weights... ";
+            $this->load_weight(true);
+            $tag_index = count($this->tag_set);
+            echo "ok\n";
+        } else {
+            $this->w=[];
+            $this->tag_set=[];
+            $tag_index = 0;
+            if (isset($this->rule_defined_key)) {
+                foreach($this->rule_defined_key as $k) {
+                    $this->tag_set[$k] = $tag_index++;
+                }
+            }
+            for ($i = -4; $i <= -1; $i++) {
+                $this->w[$i] = [];
+            }
+        }
+        foreach ($term_tag_sentences as $term_tag_pairs) {
+            $terms=$term_tag_pairs[0];
+            $tags=$term_tag_pairs[1];
+            for ($i = 0; $i < count($terms); $i++) {
+                if (!isset($this->tag_set[$tags[$i]])) {
+                    $this->tag_set[$tags[$i]] = $tag_index++;
+                }
+                $k = $this->getIndex($i,$terms);
+                if (!isset($this->w[$k])) {
+                    $this->w[$k] = [];
+                }
+            }
+        }
+        foreach (array_keys($this->w) as $key) {
+            for ($i=-2; $i<=2;$i++) {
+                if (!isset($this->w[$key][$i])) {
+                    $this->w[$key][$i] = [];
+                }
+                foreach($this->tag_set as $possiable_tag => $tag_index) {
+                    if (!isset($this->w[$key][$i][$tag_index])) {
+                        $this->w[$key][$i][$tag_index] = 0;
+                    }
+                }
+            }
+        }
+        foreach($this->tag_set as $possiable_tag => $tag_index) {
+            if (!isset($this->b[$tag_index])) {
+                $this->b[$tag_index] = 0;
+            }
+        }
+        echo "Training\n";
+        //train the weight
+        $cross_entropy_loss = 1;
+        $pre_cross_entropy_loss = 2;
+        for ($epoch = 0; $epoch < $max_epoch &&
+            $pre_cross_entropy_loss - $cross_entropy_loss > 0.000001; $epoch++) {
+            $this->min_w=0;
+            $this->max_w=0;
+            $time = time();
+            $dy_dw = [];
+            $dy_dw_n = [];
+            $pre_cross_entropy_loss = $cross_entropy_loss;
+            $cross_entropy_loss = 0;
+            $cross_entropy_loss_n = 0;
+            
+            $dy_db=[];
+            $dy_db_n=[];
+            for($i = 0; $i < count($this->tag_set); $i++) {
+                $dy_db[$i] = 0;
+                $dy_db_n[$i] = 0;
+            }
+            //for each sentence
+            foreach ($term_tag_sentences as $term_tag_pairs) {
+                $terms=$term_tag_pairs[0];
+                $tags=$term_tag_pairs[1];
+                for ($i = 0; $i < count($terms); $i++) {
+                    $k=[];
+                    for ($j=-2; $j<=2;$j++) {
+                        $k[$j]= $this->getIndex($i+$j,$terms);
+                    }
+                    foreach ($this->tag_set as $possiable_tag => $tag_index) {
+                        $equality = $possiable_tag == $tags[$i] ? 1 : 0;
+                        $sum=0;
+                        for ($j=-2; $j<=2;$j++) {
+                            $sum += $this->w[$k[$j]][$j][$tag_index];
+                        }
+                        $sum += $this->b[$tag_index];
+                        $sigmoid = 1 / (1 + exp(-1 * $sum));
+                        for ($j=-2; $j<=2;$j++) {
+                            if (!isset($dy_dw[$k[$j]])) {
+                                $dy_dw[$k[$j]] = [];
+                                $dy_dw_n[$k[$j]] = [];
+                            }
+                            if (!isset($dy_dw[$k[$j]][$j])) {
+                                $dy_dw[$k[$j]][$j] = [];
+                                $dy_dw_n[$k[$j]][$j] = [];
+                            }
+                            if (!isset($dy_dw[$k[$j]][$j][$tag_index])) {
+                                $dy_dw[$k[$j]][$j][$tag_index] = 0;
+                                $dy_dw_n[$k[$j]][$j][$tag_index] = 0;
+                            }
+                            
+                            $dy_dw[$k[$j]][$j][$tag_index] +=
+                                ($sigmoid - $equality);
+                            $dy_dw_n[$k[$j]][$j][$tag_index] += 1;
+                            
+                        }
+                        //dy_db
+                        $dy_db[$tag_index] += ($sigmoid - $equality);
+                        $dy_db_n[$tag_index] += 1;
+                        $cross_entropy_loss+=
+                            - $equality*log($sigmoid)
+                            - (1-$equality)*log(1-$sigmoid);
+                        $cross_entropy_loss_n++;
+                    }
+                }
+            }
+            $cross_entropy_loss /= $cross_entropy_loss_n;
+            $duration = time() - $time;
+            echo "epoch {$epoch} cross_entropy {$cross_entropy_loss}".
+                " Takes {$duration} seconds\n";
+            foreach ($dy_dw as $i =>$v1) {
+                foreach ($v1 as $j =>$v2) {
+                    foreach ($v2 as $k =>$v3) {
+                        $this->w[$i][$j][$k] -=
+                            $dy_dw[$i][$j][$k] /
+                            $dy_dw_n[$i][$j][$k] *
+                            $learning_rate;
+                        if ($this->w[$i][$j][$k] < $this->min_w) {
+                            $this->min_w = $this->w[$i][$j][$k];
+                        }
+                        if ($this->w[$i][$j][$k] > $this->max_w) {
+                            $this->max_w = $this->w[$i][$j][$k];
+                        }
+                    }
+                }
+            }
+            foreach ($dy_db as $k =>$v) {
+                $this->b[$k]-=
+                    $dy_db[$k] / 
+                    $dy_db_n[$k] * 
+                    $learning_rate;
+            }
+            if ($epoch % 10 == 9 ) {
+                $this->save_weight();
+            }
+        }
+        $this->save_weight();
+        return true;
+    }
+    /**
+     * The primary function to predit the tag
+     * @param mixed $sentence is an array of segmented words/terms
+     *     or a string with words/terms seperated by space
+     * @return @array of tags 
+     */
+    public function predict($sentence)
+    {
+        if (!is_array($sentence)) {
+            if ($sentence == "") {
+                $terms=[];
+            } else {
+                $terms=preg_split("/[\s]+/",$sentence);
+            }
+        } else {
+            $terms=$sentence;
+        }
+        if (!count($terms)) {
+            return [];
+        }
+        if (!$this->w) {
+            $this->load_weight();
+        }
+        $ret = [];
+        for($i = 0; $i < count($terms); $i++) {
+            $term = $terms[$i];
+            $score =[];
+            $key=$this->getKey($term);
+            foreach($this->tag_set as $possiable_tag => $tag_index) {
+                $score[$possiable_tag]=0;
+                for ($j=-2; $j <=2; $j++) {
+                    $k=$this->getIndex($i+$j, $terms);
+                    if (isset($this->w[$k])) {
+                        $score[$possiable_tag] += 
+                                $this->getW($k,$j,$tag_index);
+                    } else if ($j==0&&in_array($possiable_tag,$this->rule_defined_key)) {
+                        $score[$possiable_tag] += $this->min_w;
+                    }
+                }
+                
+                $score[$possiable_tag] += $this->getB($tag_index);
+                
+                //$score[$possiable_tag]
+                //    += 1 / (1 + exp(-1 * $score[$possiable_tag]));
+            }
+            $ret[]=array_keys($score, max($score))[0];
+        }
+        return $ret;
+    }
+    /**
+     * Wrap function for predict
+     * @param $texts to be a @string of texts
+     * @param $return_string is a boolean to determing if the user
+     *        want it to out put to stdout or a return value
+     * @return @string if $return_string is true;
+               @boolean true otherwise
+     * e.g. 中国_NR 人民_NN 将_AD 满怀信心_VV
+            地_DEV 开创_VV 新_VA 的_DEC 业绩_NN 。_PU
+     */
+    public function tag($texts, $return_string=false)
+    {
+        if ($return_string) {
+            $ret = "";
+        }
+        $sentences = preg_split('/\r\n|\r|\n/', $texts);
+        foreach($sentences as $sentence) {
+            $sentence=explode(" ",trim($sentence));
+            $term_pos = $this->predict($sentence);
+            for($i = 0; $i < count($term_pos); $i++) {
+                $term_pos[$i]=$sentence[$i]."_".$term_pos[$i];
+            }
+            $t = join(" ", $term_pos);
+            if ($return_string) {
+                $ret .= $t;
+            } else {
+                echo $t, "\n";
+            }
+        }
+        if ($return_string) {
+            return $ret;
+        } else {
+            return true;
+        }
+    }
+    /**
+     * A list of private helper functions
+     * Given a setence ($term), find the key at position $index
+     */
+    private function getIndex($index, $terms)
+    {
+        if ($index < 0) $k = $index - 2;
+        else if ($index >= count($terms)) {
+            $k = $index - count($terms) - 2;
+        }
+        else {
+            $k = $this->getKey($terms[$index]);
+        }
+        return $k;
+    }
+    /**
+     * Get the bias value for tag
+     */
+    private function getB($tag_index)
+    {
+        return unpack("f",$this->b,$tag_index*4)[1];
+    }
+    /**
+     * Set the bias value for tag
+     */
+    private function setB($tag_index, $value)
+    {
+        $this->b = substr_replace($this->b,pack("f",$value),$tag_index*4,4);
+    }
+    /**
+     * Get the weight value for term at postion for tag
+     */
+    private function getW($term, $position, $tag_index)
+    {
+        $t = unpack("S",$this->w[$term],
+            2*($position+2)*count($this->tag_set)+$tag_index*2)[1]
+            / 65535
+            * ($this->max_w-$this->min_w) + $this->min_w;;
+        return $t;
+    }
+    /**
+     * save the trained weight to disk
+     */
+    private function save_weight()
+    {
+        $out_file = C\LOCALE_DIR . "/{$this->lang}/resources/pos_weight.txt.gz";
+        $out = [];
+        $out["min_w"] = $this->min_w;
+        $out["max_w"] = $this->max_w;
+        $out["w"]=[];
+        foreach(array_keys($this->w) as $key) {
+            $out["w"][$key] = $this->pack_w($key);
+        }
+        $out["b"] = $this->pack_b();
+        $out["tag_set"] = $this->tag_set;
+        echo "Saving...";
+        file_put_contents($out_file,
+            gzencode(serialize($out),9));
+        echo " ok\n";
+    }
+    /**
+     * load the trained weight from disk
+     */
+    private function load_weight($trainning_load=false)
+    {
+        $dic_file
+            = C\LOCALE_DIR . "/{$this->lang}/resources/pos_weight.txt.gz";
+        if (!file_exists($dic_file)) {
+            echo "$dic_file does not exist!";
+            exit();
+        }
+        $f = unserialize(gzdecode(file_get_contents($dic_file))
+            ,['allowed_classes' => false]);
+        $this->w=$f["w"];
+        $this->b=$f["b"];
+        $this->min_w=$f["min_w"];
+        $this->max_w=$f["max_w"];
+        $this->tag_set=$f["tag_set"];
+        if ($trainning_load) {
+            foreach(array_keys($this->w) as $key) {
+                $this->w[$key] = $this->unpack_w($key);
+            }
+            $this->b = $this->unpack_b($this->b);
+        }
+    }
+    /**
+     * Pack the bias
+     */
+    private function pack_b()
+    {
+        return pack("f*", ...$this->b);
+    }
+    /**
+     * Unpack the bias
+     */
+    private function unpack_b()
+    {
+        return array_merge(unpack("f".strval(count($this->tag_set)),$this->b));
+    }
+    /**
+     * Pack the weight
+     */
+    private function pack_w($key)
+    {
+        $bin_str = "";
+        foreach($this->w[$key] as $i => $t) {
+            foreach($t as $u) {
+                $v = 65535 * ($u-$this->min_w) / ($this->max_w-$this->min_w);
+                $bin_str .= pack("S", intval($v));
+            }
+        }
+        return $bin_str;
+    }
+    /**
+     * Unpack the weight
+     */
+    private function unpack_w($key)
+    {
+        $tmp = [];
+        $size = count($this->tag_set);
+        for ($i = 0; $i < 5; $i++) {
+            $tmp[$i-2] = array_merge(unpack("S".strval($size),
+                $this->w[$key], 2*$i*count($this->tag_set)));
+            for($j = 0; $j < $size; $j++) {
+                $tmp[$i-2][$j] = $tmp[$i-2][$j] / 65535
+                    * ($this->max_w-$this->min_w) + $this->min_w;
+            }
+        }
+        return $tmp;
+    }
+}
diff --git a/src/library/StochasticTermSegmenter.php b/src/library/StochasticTermSegmenter.php
index 474be126..4d64658d 100644
--- a/src/library/StochasticTermSegmenter.php
+++ b/src/library/StochasticTermSegmenter.php
@@ -28,7 +28,6 @@
  */
 namespace seekquarry\yioop\library;
 
-use seekquarry\yioop\library\Trie;
 use seekquarry\yioop\locale\zh_CN\resources as ZH;
 use seekquarry\yioop\configs as C;
 /**
@@ -39,12 +38,13 @@ use seekquarry\yioop\configs as C;
  * Currently only supports Chinese.
  * Instruction to add a new language:
  * Add a switch case in the constructor.
- * Define the following variable:
- * $non_char_preg
  * Define the following function:
  * isExceptionImpl
  * See the class function 'isException' for more information
- *
+ * isPunctuationImpl
+ * See the class function 'isPunctuation' for more information
+ * isNotCurrentLangImpl
+ * See the class function 'notCurrentLang' for more information
  * Chinese example is provided in the constructor
  *
  * @author Xianghong Sun
@@ -66,7 +66,7 @@ class StochasticTermSegmenter
      */
     private $cache_pct;
     /**
-     * Used in memory saving mode
+     * Cache. Will have runtime data for the segmentation
      * @var array
      */
     private $cache=[];
@@ -111,9 +111,8 @@ class StochasticTermSegmenter
         switch($lang)
         {
             case "zh_CN":
-                case "zh-CN":
+            case "zh-CN":
                 $this->lang = "zh_CN";
-                $this->non_char_preg = "/^[^\p{Han}]+$/u";
                 /*
                  * Check if the term passed in is an exception term
                  */
@@ -127,23 +126,22 @@ class StochasticTermSegmenter
                  */
                 $this->isPunctuationImpl = function($term)
                 {
-                    return preg_match(ZH\Tokenizer::$punctuation_preg, $term);
+                    return ZH\Tokenizer::isPunctuation($term);
                 };
-                break;
-            case "ja":
-                $this->lang = $lang;
-                $this->non_char_preg =
-                    "/^[^\x{4E00}-\x{9FBF}\x{3040}-\x{309F}".
-                    "\x{30A0}-\x{30FF}]+$/u";
-                break;
-            case "ko":
-                $this->lang = $lang;
-                $this->non_char_preg = "/^[^\x{3130}-\x{318F}".
-                    "\x{AC00}-\x{D7AF}]+$/u";
+                /*
+                 * Check if all the chars in the term is NOT current language
+                 */
+                $this->isNotCurrentLangImpl = function($term)
+                {
+                    return ZH\Tokenizer::isNotCurrentLang($term);
+                };
+                /*
+                 * named entity recognizer;
+                 */
+                $this->NER = ZH\Tokenizer::getNER();
                 break;
             default:
                 $this->lang = $lang;
-                $this->non_char_preg = "/^[^.]+$/u";
         }
     }
     /**
@@ -208,18 +206,18 @@ class StochasticTermSegmenter
      */
     public function notCurrentLang($term)
     {
-        return preg_match($this->non_char_preg, $term);
+        if (isset($this->isNotCurrentLangImpl))
+            return $this->isNotCurrentLangImpl($term);
+        return false;
     }
     /**
      * Generate a term dictionary file for later segmentation
      * @param mixed $text_files is a string name or an array of files
      *  that to be trained; words in the files need to be segmented by space
      * @param string $format currently only support default and CTB
-     * @param int $max_term_weights maximum number of terms ot keep
      * @return bool true if success
      */
-    public function train($text_files, $format = "default",
-        $max_num_term_weights = 40000)
+    public function train($text_files, $format = "default")
     {
         $ctb_fmt=false;
         switch ($format) {
@@ -254,9 +252,7 @@ class StochasticTermSegmenter
                             && !$this->notCurrentLang($word)) {
                             if (!empty($dictionary[$word])) {
                                 $dictionary[$word]++;
-                            } else if (count($dictionary) <
-                                $max_num_term_weights
-                                && mb_strlen($word) < 7) {
+                            } else if (mb_strlen($word) < 7) {
                                 $dictionary[$word] = 1;
                             }
                         }
@@ -319,6 +315,8 @@ class StochasticTermSegmenter
                     }
                 }
                 fclose($fh);
+            } else {
+                echo "cannot open $text_file\n";
             }
         }
         if ($return_string) {
@@ -392,6 +390,13 @@ class StochasticTermSegmenter
         if (!count($characters)) {
             return [];
         }
+        $ner_dict=[];
+        if (isset($this->NER)) {
+            $named_entities=$this->NER->predict($characters);
+            foreach($named_entities as $e) {
+                $this->add($e[0],1,$ner_dict);
+            }
+        }
         $score = [];
         $path = [];
         //init base
@@ -487,6 +492,22 @@ class StochasticTermSegmenter
                     }
                 }
             }
+            //check NER dictionary
+            if (isset($ner_dict[$characters[$index]])) {
+                $subdic = $ner_dict;
+                for ($j = $index; $j < count($characters); $j++) {
+                    if (!isset($subdic[$characters[$j]])) {
+                        break;
+                    }
+                    $subdic = $subdic[$characters[$j]];
+                    if (isset($subdic['$']) && (!isset($score[$j]) ||
+                        $score[$index - 1] + $subdic['$'] < $score[$j])) {
+                        $score[$j] = $score[$index - 1] +
+                            $this->getScore($subdic['$']);
+                        $path[$j] = $index - 1;
+                    }
+                }
+            }
         }
         //trace path
         $t = max(array_keys($path));
diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index 02113c79..ede2bd98 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -30,6 +30,7 @@ namespace seekquarry\yioop\locale\zh_CN\resources;
 
 use seekquarry\yioop\library\PhraseParser;
 use seekquarry\yioop\library\StochasticTermSegmenter;
+use seekquarry\yioop\library\ContextWeightedNamedEntityRecognizer;
 
 /**
  * Chinese specific tokenization code. Typically, tokenizer.php
@@ -57,23 +58,39 @@ class Tokenizer
         '与', '次', '狗', '决', '金', '史', '姆', '部', '正在', '活', '刚',
         '回家', '贝', '如何', '须', '战', '不會', '夫', '喂', '父', '亚', '肯定',
         '女孩', '世界'];
+    /**
+     * regular expression to determine if the None of the char in this
+     * term is in current language.
+     * @var string
+     */
+    public static $non_char_preg = "/^[^\p{Han}]+$/u";
     /**
      * The dictionary of characters can be used as Chinese Numbers
      * @string
      */
     public static $num_dict =
-       "1234567890○零一二三四五六七八九十百千万亿".
+       "1234567890○〇零一二两三四五六七八九十百千万亿".
        "０１２３４５６７８９壹贰叁肆伍陆柒捌玖拾廿卅卌佰仟萬億";
     /**
      * Dots used in Chinese Numbers
      * @string
      */
-    public static $dot = "\.．";
+    public static $dot = "\.．点";
     /**
      * A list of characters can be used at the end of numbers
      * @string
      */
     public static $num_end = "％%";
+    /**
+     * Exception words of the regex found by functions:
+     * isCardinalNumber, isOrdinalNumber, isDate
+     * ex. "十分" in most of time means "very", but it will
+     * be determined to be "10 minutes" by the function so we
+     * need to remove it
+     * @array of string
+     */
+    public static $exception_list= ["十分","一","一点","千万",
+    "万一", "一一", "拾", "一时", "千千", "万万", "陆"];
     /**
      * A list of characters can be used as Chinese punctuations
      * @string
@@ -82,7 +99,17 @@ class Tokenizer
     "/^([\x{2000}-\x{206F}\x{3000}-\x{303F}\x{FF00}-\x{FF0F}" .
     "\x{FF1A}-\x{FF20}\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}" .
     "\x{FFE0}-\x{FFEE}\x{21}-\x{2F}\x{21}-\x{2F}" .
-    "\x{3A}-\x{40}\x{5B}-\x{60}]|[\—]+|[\.]+)$/u";
+    "\x{3A}-\x{40}\x{5B}-\x{60}\x{25cf}])\\1*$/u";
+    /**
+     * Stochastic Term Segmenter instance
+     * @object
+     */
+    private static $stochasticTermSegmenter;
+    /**
+     * named Entity Recognizer instance
+     * @object
+     */
+    private static $namedEntityRecognizer;
     /**
      * Removes the stop words from the page (used for Word Cloud generation
      * and language detection)
@@ -117,8 +144,8 @@ class Tokenizer
                 ['/^\d+$/', '/^[a-zA-Z]+$/']);
                 break;
             case("STS"):
-                $segmenter = new StochasticTermSegmenter("zh_CN");
-                return $segmenter->segmentText($pre_segment,true);
+                return self::getStochasticTermSegmenter()
+                        ->segmentText($pre_segment,true);
                 break;
         }
     }
@@ -127,25 +154,111 @@ class Tokenizer
      */
     public static function isCardinalNumber($term)
     {
-        return preg_match("/^[" . self::$num_dict .
-            self::$dot . "]+[" . self::$num_end .
-            "]?[余餘]?[百千万亿佰仟萬億]?$/u", $term);
+        return !in_array($term,self::$exception_list)
+            && preg_match("/^([" . self::$num_dict .
+            "]+([" . self::$dot . "][" .self::$num_dict . 
+            "]+)?[" . self::$num_end .
+            "]?[余餘多]?[百千万亿佰仟萬億]*)".
+            "$|^([".self::$num_dict."]+分之[" . self::$num_dict .
+            "]+([" . self::$dot . "][" .self::$num_dict . 
+            "]+)?)$/u", $term);
     }
     /*
      * Check if the term passed in is a Ordinal Number
      */
     public static function isOrdinalNumber($term)
     {
-        return preg_match("/^第[" . self::$num_dict .
-        "]*$/u", $term);
+        return !in_array($term,self::$exception_list)
+            && preg_match("/^第[" . self::$num_dict .
+            "]*$/u", $term);
     }
     /*
      * Check if the term passed in is a date
      */
     public static function isDate($term)
     {
-        return preg_match("/^[" . self::$num_dict .
-        "]+(年|年代|月|日|时|小时|時|小時|" .
-        "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
+        return !in_array($term,self::$exception_list)
+            && preg_match("/^[" . self::$num_dict .
+            "]+(年|年代|月|日|时|小时|時|小時|" .
+            "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
+    }
+    /*
+     * Check if the term is a punctuation
+     */
+    public static function isPunctuation($term)
+    {
+        return preg_match(self::$punctuation_preg, $term);
+    }
+    /**
+     * Check if all the chars in the term is NOT current language
+     * @param $term is a string that to be checked
+     * @return bool true if all the chars in $term is NOT current language
+     *         false otherwise
+     */
+    public static function isNotCurrentLang($term)
+    {
+        return preg_match(self::$non_char_preg, $term);
+    }
+    /*
+     * Create stochastic term segmenter
+     */
+    public static function createStochasticTermSegmenter($cache_pct=0.06)
+    {
+        self::$stochasticTermSegmenter
+            = new StochasticTermSegmenter("zh_CN", $cache_pct);
+    }
+    /*
+     * Destory stochastic term segmenter
+     */
+    public static function destoryStochasticTermSegmenter()
+    {
+        self::$stochasticTermSegmenter = null;
+    }
+    /*
+     * Get the segmenter instance
+     */
+    public static function getStochasticTermSegmenter() {
+        if (!self::$stochasticTermSegmenter) {
+            self::createStochasticTermSegmenter();
+        }
+        return self::$stochasticTermSegmenter;
+    }
+    public static function POSGetKey($term) {
+        if (self::isPunctuation($term)) {
+            return 'PU';
+        } else if (self::isCardinalNumber($term)) {
+            return 'CD';
+        } else if (self::isOrdinalNumber($term)) {
+            return 'OD';
+        } else if (self::isDate($term)) {
+            return 'NT';
+        } else if (self::isNotCurrentLang($term)) {
+            return 'FW';
+        }
+        return null;
+    }
+    /*
+     * Create named entity recognizer instance
+     */
+    public static function createNER()
+    {
+        self::$namedEntityRecognizer
+            = new ContextWeightedNamedEntityRecognizer("zh_CN");
+    }
+    /*
+     * Destory named entity recognizer instance
+     */
+    public static function destoryNER()
+    {
+        self::$namedEntityRecognizer = null;
+    }
+    /*
+     * Get the named entity recognizer instance
+     */
+    public static function getNER() {
+        if (!self::$namedEntityRecognizer) {
+            self::createNER();
+        }
+        return self::$namedEntityRecognizer;
     }
 }
-- 
2.21.0.windows.1

