_binary From 6fd6fd950beeeb5099ffa75199c2475534ce9267 Mon Sep 17 00:00:00 2001
From: Xianghong Sun <sxh19911230@gmail.com>
Date: Thu, 31 Oct 2019 10:44:30 -0700
Subject: [PATCH 1/2] Chinese/(Japan/Korean) term segmenter r=chris

---
 src/configs/TokenTool.php               |  21 ++
 src/library/StochasticTermSegmenter.php | 422 ++++++++++++++++++++++++
 src/library/Trie.php                    |   4 +-
 3 files changed, 445 insertions(+), 2 deletions(-)
 create mode 100644 src/library/StochasticTermSegmenter.php

diff --git a/src/configs/TokenTool.php b/src/configs/TokenTool.php
index 46ddbdae..f032b9d3 100644
--- a/src/configs/TokenTool.php
+++ b/src/configs/TokenTool.php
@@ -40,6 +40,7 @@ namespace seekquarry\yioop\configs;
 
 use seekquarry\yioop\library\NWordGrams;
 use seekquarry\yioop\library\Trie;
+use seekquarry\yioop\library\StochasticTermSegmenter;
 
 if (php_sapi_name() != 'cli' ||
     defined("seekquarry\\yioop\\configs\\IS_OWN_WEB_SERVER")) {
@@ -170,6 +171,26 @@ switch ($argv[1]) {
         }
         NWordGrams::makeSegmentFilterFile($file_path.$argv[2], $argv[3]);
         break;
+    case "train":
+        if (!isset($argv[2])) {
+           echo $usage;
+        }
+        $texts=[];
+        for($i=3;$i<count($argv);++$i) {
+            if (file_exists($argv[$i])) {
+                $texts[]=$argv[$i];
+            } else {
+                echo $argv[$i]." does not exist in ".$file_path;
+                exit();
+            }
+        }
+        $segmenter = new StochasticTermSegmenter($argv[2]);
+        if ($segmenter->train($texts)) {
+          echo "success";
+        } else {
+          echo "failed";
+        }
+        break;
     default:
         echo $usage;
         exit();
diff --git a/src/library/StochasticTermSegmenter.php b/src/library/StochasticTermSegmenter.php
new file mode 100644
index 00000000..ff60bd7c
--- /dev/null
+++ b/src/library/StochasticTermSegmenter.php
@@ -0,0 +1,422 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * @author Xianghong Sun sxh19911230@gmail.com
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+ 
+namespace seekquarry\yioop\library;
+
+use seekquarry\yioop\library\Trie;
+use seekquarry\yioop\configs as C;
+//define("LOCALE_DIR", __DIR__ ."/../locale/");
+//require_once __DIR__ . '../configs/Config.php';
+//require_once "Trie.php";
+ini_set("memory_limit","1500M");
+
+
+
+/**
+ * A Stochastic Finite-State Word-Segmenter.
+ * This class contains necessary tools to segment terms 
+ * from sentences.
+ *
+ * @author Xianghong Sun
+ */
+class StochasticTermSegmenter {
+    
+    /**
+     * Which language?
+     */
+    public $lang;
+    
+    /**
+     * regular expression to determine if the non of the char in this 
+     * term is in current language
+     * Recommanded expression for:
+     * Chinese:  \p{Han}
+     * Japanese: \x{4E00}-\x{9FBF}\x{3040}-\x{309F}\x{30A0}-\x{30FF}
+     * korean:   \x{3130}-\x{318F}\x{AC00}-\x{D7AF}
+     */
+    public $non_char_preg;
+    
+    /**
+     * Default score for any unknown term,
+     */
+    public $unknown_term_score;
+    
+    /**
+     * A dictionary file that contains the statistic infomation of
+     * the terms
+     */
+    public $dictionary_file;
+    
+    /**
+     * Dynamic variables, methods
+     */
+    private $data;
+    
+    /**
+     * Constructor
+     */ 
+    function __construct($lang) {
+        $this->lang=$lang;
+        /* Add different attribute for different Languages
+         * Currenly only Chinese
+        */
+        switch($lang) {
+        case("zh_CN"):
+            $this->non_char_preg="/^[^\p{Han}]+$/u";
+            $this->num_dict =
+               "1234567890○零一二三四五六七八九十百千万亿".
+               "０１２３４５６７８９壹贰叁肆伍陆柒捌玖拾廿卅卌佰仟萬億";
+            $this->dot="\.．";
+            $this->num_end = "％%";
+            $this->punctuation="\x{3000}-\x{303F}\x{FF00}-\x{FF0F}\x{FF1A}-\x{FF20}" .
+            "\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}\x{FFE0}-\x{FFEE}\x{21}-\x{2F}" .
+            "\x{21}-\x{2F}\x{3A}-\x{40}\x{5B}-\x{60}";
+            /**
+             * Check if the term passed in is a Cardinal Number
+             */
+            $this->isCardinalNumber = function($term)
+            {
+                return preg_match("/^[" . $this->num_dict .
+                    $this->dot . "]+[" . $this->num_end .
+                    "]?[余餘]?[百千万亿佰仟萬億]?$/u", $term);
+            };
+            /**
+             * Check if the term passed in is a Ordinal Number
+             */
+            $this->isOrdinalNumber = function($term)
+            {
+                return preg_match("/^第[" . $this->num_dict .
+                "]*$/u", $term);
+            };
+            /**
+             * Check if the term passed in is a date
+             */
+            $this->isDate = function($term)
+            {
+                return preg_match("/^[" . $this->num_dict .
+                "]+(年|年代|月|日|时|小时|時|小時|" .
+                "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
+            };
+            /**
+             * Check if the term passed in is an exception term
+             */
+            $this->_isException = function($term)
+            {
+                return $this->isCardinalNumber($term)
+                || $this->isOrdinalNumber($term)
+                || $this->isDate($term);
+            };
+            /**
+             * Check if the term passed in is a punctuation
+             */
+            $this->isPunctuation = function($term)
+            {
+                return preg_match("/^[" . $this->punctuation
+                . "]$/u",$term);
+            };
+            break;
+        case("ja"):
+            $this->non_char_preg
+              ="/^[\x{4E00}-\x{9FBF}\x{3040}-\x{309F}\x{30A0}-\x{30FF}]+$/u";
+            break;
+        case("ko"):
+            $this->non_char_preg="/^[\x{3130}-\x{318F}\x{AC00}-\x{D7AF}]+$/u";
+            break;
+        default:
+            $this->non_char_preg="/^[^.]+$/u";
+        }
+    }
+    
+    /**
+     * harness __call  __get __set for adding dynamic methods, variables
+     */
+    public function __call($method, $args)
+    {
+        return call_user_func_array($this->$method, $args);
+    }
+    
+    public function __get($varName)
+    {
+        return $this->$varName;
+    }
+
+    public function __set($varName,$value)
+    {
+        $this->$varName = $value;
+    }
+    
+    /**
+     * Check if the term passed in is an exception term
+     * Not all valid terms should be indexed.
+     * e.g. there are infinite combinations of numbers in the world.
+     * _isException should be defined in constructor if needed
+     * @param $term is a string that to be checked
+     * @return true if $term is an exception term, false otherwise
+     */
+    public function isException($term){
+        if (isset($this->_isException))
+            return $this->_isException($term);
+        return false;
+    }
+    
+    /**
+     * Check if all the chars in the term is not current language
+     * @param $term is a string that to be checked
+     * @return true if all the chars in $term is not current language
+     *         false otherwise
+     */
+    public function notCurrentLang($term)
+    {
+        return preg_match($this->non_char_preg, $term);
+    }
+    
+    /**
+     * Generate a term dictionary file for later segmentation
+     * @param $text_files is a string or an array of files
+     *         that to be trained;
+     *         words in the files need to be segmented by space
+     * @return true if success
+     */
+    public function train($text_files)
+    {
+        $out_file = C\LOCALE_DIR . "/{$this->lang}/resources/term_weight.txt.gz";
+        echo "saving file to: $out_file\n";
+        $dictionary=array();
+        $N = 0;
+        if (is_string($text_files)) $text_files = [$text_files];
+        foreach($text_files as $text_file){
+            if (file_exists($text_file)) {
+                $fn = fopen($text_file,"r");
+                while(! feof($fn))  {
+                    $line = fgets($fn);
+                    $words = preg_split("/[\s　]+/u", $line);
+                    foreach ($words as $word) {
+                        if ($word!="" && !$this->isException($word)
+                            && !$this->notCurrentLang($word)) {
+                            if (array_key_exists($word,$dictionary))
+                                ++$dictionary[$word];
+                            else $dictionary[$word] = 1;
+                            
+                        }
+                    }
+                }
+                fclose($fn);
+            }
+        }
+        $trie = new Trie('$');
+        foreach ($dictionary as $key => $value) {
+            $trie->add($key." ".$value,false);
+            $N++;
+        }
+        $this->dictionary_file=[];
+        $this->dictionary_file["N"]=$N;
+        $this->dictionary_file["dic"]=$trie->trie_array;
+        $this->unknown_term_score=$this->getScore(1);
+        file_put_contents($out_file,
+            gzencode(json_encode($this->dictionary_file),9));
+        return true;
+    }
+    
+    /**
+     * segment texts
+     * words are seperated by space
+     * @param $text_files is a filename or a list of file names
+     *        to be segmented
+     * $param $return_string return segmented string if true,
+     *        print otherwise
+     * @return segmented words with space or true/false;
+     */
+    public function segmentFiles($text_files,$return_string=false)
+    {
+        if ($return_string) $result="";
+        if (is_string($text_files)) $text_files = [$text_files];
+        foreach($text_files as $text_file){
+            if (file_exists($text_file)) {
+                $fn = fopen($text_file,"r");
+                while(! feof($fn))  {
+                    $line = fgets($fn);
+                    if (mb_strlen($line)){
+                        $t = $this->segmentSentence($line);
+                        if ($return_string)
+                            $result.=(join( " ", $t)."\n");
+                        else
+                            echo join( " ", $t)."\n";
+                    }
+                }
+                fclose($fn);
+            }
+        }
+        if ($return_string) return $result;
+        return true;
+    }
+    
+    /**
+     * segment texts
+     * words are seperated by space
+     * @param $text is a string to be segmented
+     * $param $return_string return segmented string if true,
+     *        print otherwise
+     * @return segmented words with space or true/false;
+     */
+    public function segmentText($text, $return_string=false)
+    {
+        if ($return_string) $result="";
+        $sentences = explode("\n",$text);
+        foreach($sentences as $line){
+            if (mb_strlen($line)){
+                $t = $this->segmentSentence($line);
+                if ($return_string)
+                    $result.=(join( " ", $t)."\n");
+                else
+                    echo join( " ", $t)."\n";
+            }
+        }
+        if ($return_string) return $result;
+        return true;
+    }
+    
+    /**
+     * Segment a sentence into arrays of words.
+     * Need not contain any new line characters.
+     * words are seperated by space
+     * @param $sentence is a string to be segmented
+     * @return a string of segmented words
+     */
+    public function segmentSentence($sentence)
+    {
+        if (!$this->dictionary_file) {
+            $dic_file = C\LOCALE_DIR . "/{$this->lang}/resources/term_weight.txt.gz";
+            if (!file_exists($dic_file)) {
+                echo "$dic_file does not exist!";
+                return null;
+            };
+            $this->dictionary_file
+              =json_decode(gzdecode(file_get_contents($dic_file)),true);
+            $this->unknown_term_score=$this->getScore(1);
+        }
+        
+        preg_match_all('/./u', trim($sentence), $matches);
+        $characters=$matches[0];
+        if (!count($characters)) {
+            return "";
+        }
+        $score=[];
+        $path=[];
+        //init base
+        $score[-1]=0;
+        for($index=0; $index < count($characters); ++$index) {
+            //if not current language
+            if ($this->notCurrentLang($characters[$index]) ) {
+                $current_char=$characters[$index];
+                for($j=$index+1;$j<count($characters);++$j) {
+                    if ($this->notCurrentLang($current_char.$characters[$j]))
+                      $current_char.=$characters[$j];
+                    else break;
+                }
+                if (!isset($score[$j-1]) ||
+                  $score[$j-1]>$score[$index-1]+$this->unknown_term_score) {
+                    $score[$j-1]=$score[$index-1]+$this->unknown_term_score;;
+                    $path[$j-1]=$index-1;
+                }
+            }
+            //if date or number
+            if ($this->isException($characters[$index]) ) {
+                $current_char=$characters[$index];
+                for($j=$index+1; $j<count($characters);++$j) {
+                    if ($this->isException(
+                        $current_char.$characters[$j]))
+                            $current_char.=$characters[$j];
+                  else break;
+                }
+                if (!isset($score[$j-1]) ||
+                    $score[$j-1]>$score[$index-1]+$this->unknown_term_score) {
+                    $score[$j-1]=$score[$index-1]+$this->unknown_term_score;
+                    $path[$j-1]=$index-1;
+                }
+            }
+            
+            //if is punctuation. Give slitely better score than unknown words
+            if ($this->isPunctuation($characters[$index])) {
+                if (!isset($score[$index]) ||
+                  $score[$index]>$score[$index-1]+$this->unknown_term_score/1.1) {
+                    $score[$index]=$score[$index-1]+$this->unknown_term_score/1.1;
+                    $path[$index]=$index-1;
+                }
+            }
+            
+            //All case (Even not in current lang because dictionary may contains those terms
+            //check the first char, give score even nothing matches
+            if (!isset($score[$index]) || 
+                $score[$index-1]+$this->unknown_term_score < $score[$index]) {
+                    $score[$index] = $score[$index-1]+$this->unknown_term_score;
+                    $path[$index] = $index-1;
+            }
+            $subdic=$this->dictionary_file["dic"];
+            for ($j=$index;$j<count($characters);++$j) {
+                if (!isset($subdic[$characters[$j]])) break;
+                $subdic=$subdic[$characters[$j]];
+                if (isset($subdic['$']) && 
+                  (!isset($score[$j]) || 
+                  $score[$index-1]+$subdic['$'] < $score[$j])){
+                      $score[$j]=$score[$index-1]+$this->getScore($subdic['$']);
+                      $path[$j] = $index-1;
+                }
+            }
+        }
+        //trace path
+        $t=max(array_keys($path));
+        $tmp=[];
+        while($t!=-1) {
+            $tmp[]=$t;
+            $t=$path[$t];
+        }
+        $result=[];
+        $t=0;
+        foreach(array_reverse($tmp) as $nextnode){
+            $result_word="";
+            while($t<=$nextnode) {
+              $result_word.=$characters[$t];
+              ++$t;
+            }
+            $result[]=$result_word;
+        }
+        $result_word="";
+        while($t < count($characters)){
+          $result_word.=$characters[$t];
+          ++$t;
+        }
+        $result[]=$result_word;
+        return $result;
+    }
+    
+    private function getScore($appearanace)
+    {
+      return -log($appearanace/$this->dictionary_file["N"]);
+    }      
+}
diff --git a/src/library/Trie.php b/src/library/Trie.php
index ffb365e8..c32a8b42 100644
--- a/src/library/Trie.php
+++ b/src/library/Trie.php
@@ -66,7 +66,7 @@ class Trie
      * @param string $term the term to be inserted
      * @return array $trie_array beneath last letter of term inserted
      */
-    public function add($term)
+    public function add($term,$rawurldec=true)
     {
         $trie_array = & $this->trie_array;
         $term_arr = explode(" ",$term);
@@ -75,7 +75,7 @@ class Trie
         }
         for ($i = 0; $i < mb_strlen($term_arr[0],"utf-8"); $i++) {
             $character = mb_substr($term_arr[0], $i, 1, "utf-8");
-            $enc_char = rawurlencode($character);
+            $enc_char = $rawurldec? rawurlencode($character) : $character;
             // To avoid encoding the linefeed
             if ($enc_char == "%0A"){
                 continue;
-- 
2.19.0.windows.1


From 62ae1058ef66dc93ff0a4edcdf954b437fc11717 Mon Sep 17 00:00:00 2001
From: Xianghong Sun <sxh19911230@gmail.com>
Date: Wed, 6 Nov 2019 16:23:39 -0800
Subject: [PATCH 2/2] Added, fixed some comments, styles of the code for
 readability.

---
 src/configs/TokenTool.php               |  13 +
 src/library/StochasticTermSegmenter.php | 382 +++++++++++++-----------
 src/library/Trie.php                    |   5 +-
 3 files changed, 223 insertions(+), 177 deletions(-)

diff --git a/src/configs/TokenTool.php b/src/configs/TokenTool.php
index f032b9d3..2f9183ee 100644
--- a/src/configs/TokenTool.php
+++ b/src/configs/TokenTool.php
@@ -109,6 +109,19 @@ php TokenTool.php segment-filter dictionary_file locale
 Here dictionary_file should be a text file with one word/line,
 locale is the IANA language tag of the locale to store the results for.
 
+TokenTool can be used to train some language. In some asian languages,
+such as Chinese, Japanese, there are no spaces between words.
+In order to segment terms/words from a sentence, a dictionary with some
+statistic is needed.
+
+To make a dictionary for segment the specified language:
+
+php TokenTool.php train <locale> <dictionary_files...> 
+
+Here locale is the IANA language tag of the locale to store the results for,
+dictionary_files... should be a list of text files
+with space or new line between each term.
+
 
 Obtaining Data
 ==============
diff --git a/src/library/StochasticTermSegmenter.php b/src/library/StochasticTermSegmenter.php
index ff60bd7c..a33662f1 100644
--- a/src/library/StochasticTermSegmenter.php
+++ b/src/library/StochasticTermSegmenter.php
@@ -31,24 +31,33 @@ namespace seekquarry\yioop\library;
 
 use seekquarry\yioop\library\Trie;
 use seekquarry\yioop\configs as C;
-//define("LOCALE_DIR", __DIR__ ."/../locale/");
-//require_once __DIR__ . '../configs/Config.php';
-//require_once "Trie.php";
 ini_set("memory_limit","1500M");
 
-
-
 /**
  * A Stochastic Finite-State Word-Segmenter.
  * This class contains necessary tools to segment terms 
  * from sentences.
  *
+ * Currently only support Chinese.
+ * Instruction to add a new language:
+ * Add a switch case in the constructor.
+ * Define the following variable:
+ * $non_char_preg
+ * Define the following function:
+ * isExceptionImpl
+ * See the class function 'isException' for more information
+ * 
+ * Chinese example is provided in the constructor
+ * 
  * @author Xianghong Sun
  */
-class StochasticTermSegmenter {
+class StochasticTermSegmenter
+{
     
     /**
-     * Which language?
+     * The language current using
+     * e.g. zh_CN, ja
+     * @var string  
      */
     public $lang;
     
@@ -59,97 +68,99 @@ class StochasticTermSegmenter {
      * Chinese:  \p{Han}
      * Japanese: \x{4E00}-\x{9FBF}\x{3040}-\x{309F}\x{30A0}-\x{30FF}
      * korean:   \x{3130}-\x{318F}\x{AC00}-\x{D7AF}
+     * @var string
      */
     public $non_char_preg;
     
     /**
-     * Default score for any unknown term,
+     * Default score for any unknown term
+     * @var float
      */
     public $unknown_term_score;
     
     /**
      * A dictionary file that contains the statistic infomation of
      * the terms
+     * @var array
      */
     public $dictionary_file;
     
-    /**
-     * Dynamic variables, methods
-     */
-    private $data;
     
     /**
      * Constructor
+     * @para $lang is a string to indicate the language
      */ 
-    function __construct($lang) {
-        $this->lang=$lang;
+    function __construct($lang)
+    {
+        $this->lang = $lang;
         /* Add different attribute for different Languages
-         * Currenly only Chinese
-        */
-        switch($lang) {
-        case("zh_CN"):
-            $this->non_char_preg="/^[^\p{Han}]+$/u";
-            $this->num_dict =
-               "1234567890○零一二三四五六七八九十百千万亿".
-               "０１２３４５６７８９壹贰叁肆伍陆柒捌玖拾廿卅卌佰仟萬億";
-            $this->dot="\.．";
-            $this->num_end = "％%";
-            $this->punctuation="\x{3000}-\x{303F}\x{FF00}-\x{FF0F}\x{FF1A}-\x{FF20}" .
-            "\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}\x{FFE0}-\x{FFEE}\x{21}-\x{2F}" .
-            "\x{21}-\x{2F}\x{3A}-\x{40}\x{5B}-\x{60}";
-            /**
-             * Check if the term passed in is a Cardinal Number
-             */
-            $this->isCardinalNumber = function($term)
-            {
-                return preg_match("/^[" . $this->num_dict .
-                    $this->dot . "]+[" . $this->num_end .
-                    "]?[余餘]?[百千万亿佰仟萬億]?$/u", $term);
-            };
-            /**
-             * Check if the term passed in is a Ordinal Number
-             */
-            $this->isOrdinalNumber = function($term)
-            {
-                return preg_match("/^第[" . $this->num_dict .
-                "]*$/u", $term);
-            };
-            /**
-             * Check if the term passed in is a date
-             */
-            $this->isDate = function($term)
-            {
-                return preg_match("/^[" . $this->num_dict .
-                "]+(年|年代|月|日|时|小时|時|小時|" .
-                "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
-            };
-            /**
-             * Check if the term passed in is an exception term
-             */
-            $this->_isException = function($term)
-            {
-                return $this->isCardinalNumber($term)
-                || $this->isOrdinalNumber($term)
-                || $this->isDate($term);
-            };
-            /**
-             * Check if the term passed in is a punctuation
-             */
-            $this->isPunctuation = function($term)
-            {
-                return preg_match("/^[" . $this->punctuation
-                . "]$/u",$term);
-            };
-            break;
-        case("ja"):
-            $this->non_char_preg
-              ="/^[\x{4E00}-\x{9FBF}\x{3040}-\x{309F}\x{30A0}-\x{30FF}]+$/u";
-            break;
-        case("ko"):
-            $this->non_char_preg="/^[\x{3130}-\x{318F}\x{AC00}-\x{D7AF}]+$/u";
-            break;
-        default:
-            $this->non_char_preg="/^[^.]+$/u";
+          * Currenly only Chinese
+          */
+        switch($lang)
+        {
+            case("zh_CN"):
+                $this->non_char_preg = "/^[^\p{Han}]+$/u";
+                $this->num_dict = 
+                   "1234567890○零一二三四五六七八九十百千万亿".
+                   "０１２３４５６７８９壹贰叁肆伍陆柒捌玖拾廿卅卌佰仟萬億";
+                $this->dot = "\.．";
+                $this->num_end = "％%";
+                $this->punctuation = "\x{3000}-\x{303F}\x{FF00}-\x{FF0F}\x{FF1A}-\x{FF20}" .
+                "\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}\x{FFE0}-\x{FFEE}\x{21}-\x{2F}" .
+                "\x{21}-\x{2F}\x{3A}-\x{40}\x{5B}-\x{60}";
+                /**
+                 * Check if the term passed in is a Cardinal Number
+                 */
+                $this->isCardinalNumber = function($term)
+                {
+                    return preg_match("/^[" . $this->num_dict .
+                        $this->dot . "]+[" . $this->num_end .
+                        "]?[余餘]?[百千万亿佰仟萬億]?$/u", $term);
+                };
+                /**
+                 * Check if the term passed in is a Ordinal Number
+                 */
+                $this->isOrdinalNumber = function($term)
+                {
+                    return preg_match("/^第[" . $this->num_dict .
+                    "]*$/u", $term);
+                };
+                /**
+                 * Check if the term passed in is a date
+                 */
+                $this->isDate = function($term)
+                {
+                    return preg_match("/^[" . $this->num_dict .
+                    "]+(年|年代|月|日|时|小时|時|小時|" .
+                    "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
+                };
+                /**
+                 * Check if the term passed in is an exception term
+                 */
+                $this->isExceptionImpl = function($term)
+                {
+                    return $this->isCardinalNumber($term)
+                    || $this->isOrdinalNumber($term)
+                    || $this->isDate($term);
+                };
+                /**
+                 * Check if the term passed in is a punctuation
+                 */
+                $this->isPunctuation = function($term)
+                {
+                    return preg_match("/^[" . $this->punctuation
+                    . "]$/u",$term);
+                };
+                break;
+            case("ja"):
+                $this->non_char_preg
+                  = "/^[^\x{4E00}-\x{9FBF}\x{3040}-\x{309F}\x{30A0}-\x{30FF}]+$/u";
+                break;
+            case("ko"):
+                $this->non_char_preg = "/^[^\x{3130}-\x{318F}\x{AC00}-\x{D7AF}]+$/u";
+                break;
+            default:
+                $this->non_char_preg = "/^[^.]+$/u";
         }
     }
     
@@ -175,20 +186,21 @@ class StochasticTermSegmenter {
      * Check if the term passed in is an exception term
      * Not all valid terms should be indexed.
      * e.g. there are infinite combinations of numbers in the world.
-     * _isException should be defined in constructor if needed
+     * isExceptionImpl should be defined in constructor if needed
      * @param $term is a string that to be checked
      * @return true if $term is an exception term, false otherwise
      */
-    public function isException($term){
-        if (isset($this->_isException))
-            return $this->_isException($term);
+    public function isException($term)
+    {
+        if (isset($this->isExceptionImpl))
+            return $this->isExceptionImpl($term);
         return false;
     }
     
     /**
-     * Check if all the chars in the term is not current language
+     * Check if all the chars in the term is NOT current language
      * @param $term is a string that to be checked
-     * @return true if all the chars in $term is not current language
+     * @return true if all the chars in $term is NOT current language
      *         false otherwise
      */
     public function notCurrentLang($term)
@@ -207,20 +219,22 @@ class StochasticTermSegmenter {
     {
         $out_file = C\LOCALE_DIR . "/{$this->lang}/resources/term_weight.txt.gz";
         echo "saving file to: $out_file\n";
-        $dictionary=array();
+        $dictionary = array();
         $N = 0;
-        if (is_string($text_files)) $text_files = [$text_files];
-        foreach($text_files as $text_file){
+        if (is_string($text_files)) {
+            $text_files = [$text_files];
+        }
+        foreach($text_files as $text_file) {
             if (file_exists($text_file)) {
                 $fn = fopen($text_file,"r");
-                while(! feof($fn))  {
+                while(!feof($fn))  {
                     $line = fgets($fn);
                     $words = preg_split("/[\s　]+/u", $line);
                     foreach ($words as $word) {
-                        if ($word!="" && !$this->isException($word)
+                        if ($word != "" && !$this->isException($word)
                             && !$this->notCurrentLang($word)) {
                             if (array_key_exists($word,$dictionary))
-                                ++$dictionary[$word];
+                                $dictionary[$word]++;
                             else $dictionary[$word] = 1;
                             
                         }
@@ -234,79 +248,89 @@ class StochasticTermSegmenter {
             $trie->add($key." ".$value,false);
             $N++;
         }
-        $this->dictionary_file=[];
-        $this->dictionary_file["N"]=$N;
-        $this->dictionary_file["dic"]=$trie->trie_array;
-        $this->unknown_term_score=$this->getScore(1);
+        $this->dictionary_file = [];
+        $this->dictionary_file["N"] = $N;
+        $this->dictionary_file["dic"] = $trie->trie_array;
+        $this->unknown_term_score = $this->getScore(1);
         file_put_contents($out_file,
             gzencode(json_encode($this->dictionary_file),9));
         return true;
     }
     
     /**
-     * segment texts
-     * words are seperated by space
-     * @param $text_files is a filename or a list of file names
+     * This function is used to segment a list of files
+     * @param $text_files can be a file name or a list of file names
      *        to be segmented
      * $param $return_string return segmented string if true,
-     *        print otherwise
+     *        print to stdout otherwise
+     *        user can use > filename to output it to a file
      * @return segmented words with space or true/false;
      */
-    public function segmentFiles($text_files,$return_string=false)
+    public function segmentFiles($text_files,$return_string = false)
     {
-        if ($return_string) $result="";
-        if (is_string($text_files)) $text_files = [$text_files];
-        foreach($text_files as $text_file){
+        if ($return_string) {
+            $result = "";
+        }
+        if (is_string($text_files)) {
+            $text_files = [$text_files];
+        }
+        foreach($text_files as $text_file) {
             if (file_exists($text_file)) {
                 $fn = fopen($text_file,"r");
                 while(! feof($fn))  {
                     $line = fgets($fn);
-                    if (mb_strlen($line)){
+                    if (mb_strlen($line)) {
                         $t = $this->segmentSentence($line);
                         if ($return_string)
-                            $result.=(join( " ", $t)."\n");
+                            $result .= (join( " ", $t)."\n");
                         else
-                            echo join( " ", $t)."\n";
+                            echo join(" ", $t)."\n";
                     }
                 }
                 fclose($fn);
             }
         }
-        if ($return_string) return $result;
+        if ($return_string) {
+            return $result;
+        }
         return true;
     }
     
     /**
-     * segment texts
+     * Segment texts.
+     * Similiar to the previous function.
      * words are seperated by space
      * @param $text is a string to be segmented
      * $param $return_string return segmented string if true,
      *        print otherwise
      * @return segmented words with space or true/false;
      */
-    public function segmentText($text, $return_string=false)
+    public function segmentText($text, $return_string = false)
     {
-        if ($return_string) $result="";
+        if ($return_string) {
+            $result = "";
+        }
         $sentences = explode("\n",$text);
-        foreach($sentences as $line){
-            if (mb_strlen($line)){
+        foreach($sentences as $line) {
+            if (mb_strlen($line)) {
                 $t = $this->segmentSentence($line);
                 if ($return_string)
-                    $result.=(join( " ", $t)."\n");
+                    $result .= (join( " ", $t)."\n");
                 else
                     echo join( " ", $t)."\n";
             }
         }
-        if ($return_string) return $result;
+        if ($return_string) {
+            return $result;
+        }
         return true;
     }
     
     /**
      * Segment a sentence into arrays of words.
-     * Need not contain any new line characters.
-     * words are seperated by space
-     * @param $sentence is a string to be segmented
-     * @return a string of segmented words
+     * Need NOT contain any new line characters.
+     * @param $sentence is a string without newline to be segmented
+     * @return an array of segmented words
      */
     public function segmentSentence($sentence)
     {
@@ -317,106 +341,114 @@ class StochasticTermSegmenter {
                 return null;
             };
             $this->dictionary_file
-              =json_decode(gzdecode(file_get_contents($dic_file)),true);
-            $this->unknown_term_score=$this->getScore(1);
+              = json_decode(gzdecode(file_get_contents($dic_file)),true);
+            $this->unknown_term_score = $this->getScore(1);
         }
         
         preg_match_all('/./u', trim($sentence), $matches);
-        $characters=$matches[0];
+        $characters = $matches[0];
         if (!count($characters)) {
-            return "";
+            return [];
         }
-        $score=[];
-        $path=[];
+        $score = [];
+        $path = [];
         //init base
-        $score[-1]=0;
-        for($index=0; $index < count($characters); ++$index) {
+        $score[-1] = 0;
+        for($index = 0; $index < count($characters); $index++) {
             //if not current language
             if ($this->notCurrentLang($characters[$index]) ) {
-                $current_char=$characters[$index];
-                for($j=$index+1;$j<count($characters);++$j) {
+                $current_char = $characters[$index];
+                for($j = $index + 1; $j < count($characters); $j++) {
                     if ($this->notCurrentLang($current_char.$characters[$j]))
-                      $current_char.=$characters[$j];
+                      $current_char .= $characters[$j];
                     else break;
                 }
-                if (!isset($score[$j-1]) ||
-                  $score[$j-1]>$score[$index-1]+$this->unknown_term_score) {
-                    $score[$j-1]=$score[$index-1]+$this->unknown_term_score;;
-                    $path[$j-1]=$index-1;
+                if (!isset($score[$j - 1]) ||
+                  $score[$j - 1]>$score[$index - 1] + $this->unknown_term_score) {
+                    $score[$j - 1]
+                        = $score[$index - 1] + $this->unknown_term_score;
+                    $path[$j - 1] = $index - 1;
                 }
             }
             //if date or number
             if ($this->isException($characters[$index]) ) {
-                $current_char=$characters[$index];
-                for($j=$index+1; $j<count($characters);++$j) {
+                $current_char = $characters[$index];
+                for($j = $index+1; $j<count($characters);$j++) {
                     if ($this->isException(
                         $current_char.$characters[$j]))
-                            $current_char.=$characters[$j];
-                  else break;
+                        $current_char .= $characters[$j];
+                    else {
+                        break;
+                    }
                 }
                 if (!isset($score[$j-1]) ||
-                    $score[$j-1]>$score[$index-1]+$this->unknown_term_score) {
-                    $score[$j-1]=$score[$index-1]+$this->unknown_term_score;
-                    $path[$j-1]=$index-1;
+                    $score[$j - 1]>$score[$index-1]+$this->unknown_term_score) {
+                    $score[$j - 1] = $score[$index-1]+$this->unknown_term_score;
+                    $path[$j - 1] = $index-1;
                 }
             }
             
             //if is punctuation. Give slitely better score than unknown words
             if ($this->isPunctuation($characters[$index])) {
                 if (!isset($score[$index]) ||
-                  $score[$index]>$score[$index-1]+$this->unknown_term_score/1.1) {
-                    $score[$index]=$score[$index-1]+$this->unknown_term_score/1.1;
-                    $path[$index]=$index-1;
+                    $score[$index]>$score[$index - 1] 
+                    + $this->unknown_term_score / 1.1) {
+                    $score[$index]
+                        = $score[$index - 1] + $this->unknown_term_score / 1.1;
+                    $path[$index] = $index-1;
                 }
             }
             
-            //All case (Even not in current lang because dictionary may contains those terms
+            //All case (Even not in current lang because dictionary may contains
+            //those terms
             //check the first char, give score even nothing matches
             if (!isset($score[$index]) || 
-                $score[$index-1]+$this->unknown_term_score < $score[$index]) {
-                    $score[$index] = $score[$index-1]+$this->unknown_term_score;
-                    $path[$index] = $index-1;
+                $score[$index-1] + $this->unknown_term_score < $score[$index]) {
+                    $score[$index] = $score[$index-1] + $this->unknown_term_score;
+                    $path[$index] = $index - 1;
             }
-            $subdic=$this->dictionary_file["dic"];
-            for ($j=$index;$j<count($characters);++$j) {
-                if (!isset($subdic[$characters[$j]])) break;
-                $subdic=$subdic[$characters[$j]];
+            $subdic = $this->dictionary_file["dic"];
+            for ($j = $index; $j < count($characters); $j++) {
+                if (!isset($subdic[$characters[$j]])) {
+                    break;
+                }
+                $subdic = $subdic[$characters[$j]];
                 if (isset($subdic['$']) && 
                   (!isset($score[$j]) || 
-                  $score[$index-1]+$subdic['$'] < $score[$j])){
-                      $score[$j]=$score[$index-1]+$this->getScore($subdic['$']);
+                  $score[$index-1] + $subdic['$'] < $score[$j])) {
+                      $score[$j] = $score[$index-1]+$this->getScore($subdic['$']);
                       $path[$j] = $index-1;
                 }
             }
         }
         //trace path
-        $t=max(array_keys($path));
-        $tmp=[];
-        while($t!=-1) {
-            $tmp[]=$t;
-            $t=$path[$t];
+        $t = max(array_keys($path));
+        $tmp = [];
+        while($t != -1) {
+            $tmp[] = $t;
+            $t = $path[$t];
         }
-        $result=[];
-        $t=0;
-        foreach(array_reverse($tmp) as $nextnode){
-            $result_word="";
-            while($t<=$nextnode) {
-              $result_word.=$characters[$t];
-              ++$t;
+        $result = [];
+        $t = 0;
+        foreach(array_reverse($tmp) as $nextnode) {
+            $result_word = "";
+            while($t <= $nextnode) {
+              $result_word .= $characters[$t];
+              $t++;
             }
-            $result[]=$result_word;
+            $result[] = $result_word;
         }
-        $result_word="";
-        while($t < count($characters)){
-          $result_word.=$characters[$t];
-          ++$t;
-        }
-        $result[]=$result_word;
+        
         return $result;
     }
     
-    private function getScore($appearanace)
+    /*
+     * This is the function to calculate scores for each word
+     * @para $frequency is an integer tells the frequency of a word
+     * @return the score of the term.
+     */
+    private function getScore($frequency)
     {
-      return -log($appearanace/$this->dictionary_file["N"]);
+      return -log($frequency / $this->dictionary_file["N"]);
     }      
 }
diff --git a/src/library/Trie.php b/src/library/Trie.php
index c32a8b42..69178dd6 100644
--- a/src/library/Trie.php
+++ b/src/library/Trie.php
@@ -64,9 +64,10 @@ class Trie
      * Adds a term to the Trie
      *
      * @param string $term the term to be inserted
+     @ @param bool $raw_url_decode decide term should be encoded in utf-8 or url
      * @return array $trie_array beneath last letter of term inserted
      */
-    public function add($term,$rawurldec=true)
+    public function add($term,$raw_url_decode=true)
     {
         $trie_array = & $this->trie_array;
         $term_arr = explode(" ",$term);
@@ -75,7 +76,7 @@ class Trie
         }
         for ($i = 0; $i < mb_strlen($term_arr[0],"utf-8"); $i++) {
             $character = mb_substr($term_arr[0], $i, 1, "utf-8");
-            $enc_char = $rawurldec? rawurlencode($character) : $character;
+            $enc_char = $raw_url_decode? rawurlencode($character) : $character;
             // To avoid encoding the linefeed
             if ($enc_char == "%0A"){
                 continue;
-- 
2.19.0.windows.1

