_binary From b3e620502882227adb4b44c93c04e460d9c35c32 Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Fri, 31 Jan 2020 22:48:50 -0800
Subject: [PATCH] updated cache for stochastictermsegmenter

---
 src/configs/TokenTool.php                |  30 ++--
 src/library/StochasticTermSegmenter.php  | 174 ++++++++++++++++++-----
 src/locale/zh_CN/resources/Tokenizer.php |   8 +-
 3 files changed, 164 insertions(+), 48 deletions(-)

diff --git a/src/configs/TokenTool.php b/src/configs/TokenTool.php
index a91b07e5..c8c5eeeb 100644
--- a/src/configs/TokenTool.php
+++ b/src/configs/TokenTool.php
@@ -177,7 +177,7 @@ which is a less accurate way to do segmentation.
 To create such files, TokenTool.php is run from the
 command line as:
 
-php TokenTool.php stochastic-segmenter locale dictionary_files...
+php TokenTool.php stochastic-segmenter locale files_format dataset_files...
 
 or
 
@@ -186,8 +186,16 @@ php TokenTool.php segment-filter locale dictionary_file
 respectively.
 
 Here locale is the IANA language tag of the locale to store the results for,
-dictionary_files... is a list of one or more text file names with one word/line
+files_format is the format of the files,
+currently supported format can be "default", or "CTB" (Chinese Three Bank),
+default format has all word segmented by space,
+CTB information can be found at:
+http://www.cs.brandeis.edu/~clp/ctb/
+
+dataset_files... should be a list of text files with the format described above
 and dictionary_file is a text file with one word/line.
+An example is:
+php TokenTool.php stochastic-segmenter zh-CN CTB segmented/*
 
 Localizing Yioop's web app strings to a new language can be done manually
 via the Manage Locale activity within Yioop. Alternatively, TokenTool can
@@ -274,23 +282,23 @@ switch ($argv[1]) {
         break;
     case "stochastic-segmenter":
         $file_path = PREP_DIR . "/";
-        if (!isset($argv[2])) {
+        if (!isset($argv[3])) {
            echo $usage;
         }
         $texts=[];
-        for($i = 3; $i < count($argv); $i++) {
-            if (file_exists($file_path . $argv[$i])) {
-                $texts[] = $file_path . $argv[$i];
-            } else {
-                echo $argv[$i] . " does not exist in " . $file_path;
+        for($i = 4; $i < count($argv); $i++) {
+            $files = glob($file_path . $argv[$i]);
+            if (count($files) == 0) {
+                echo "error: {$file_path}{$argv[i]}: File not found\n";
                 exit();
             }
+            $texts = array_merge ($texts, $files);
         }
         $segmenter = new StochasticTermSegmenter($argv[2]);
-        if ($segmenter->train($texts)) {
-            echo "success";
+        if ($segmenter->train($texts, $argv[3])) {
+            echo "Success\n";
         } else {
-            echo "failed";
+            echo "Failed\n";
         }
         break;
     case "translate-locale":
diff --git a/src/library/StochasticTermSegmenter.php b/src/library/StochasticTermSegmenter.php
index 3026c5f2..e7765f6c 100644
--- a/src/library/StochasticTermSegmenter.php
+++ b/src/library/StochasticTermSegmenter.php
@@ -51,6 +51,25 @@ use seekquarry\yioop\configs as C;
  */
 class StochasticTermSegmenter
 {
+    /**
+     * Percentage for cache entries. Value should be between 0 and 1.0
+     * Set to small number when running on memory limited machines
+     * Here is a general comparison when setting it to 0 and 1:
+     * In the test of Chinese Segmentation on pku dataset,
+     * the peak usage of memory is 26.288MB vs. 151.46MB
+     * The trade off is some efficiency,
+     * In the test of Chinese Segmentation on pku dataset,
+     * the speed is 43.803s vs. 1.540s
+     * Default value = 0.06
+     * The time and Peak Memory are 5.094 s and 98.97MB
+     * @var number from 0 - 1.0
+     */
+    private $cache_pct;
+    /**
+     * Used in memory saving mode
+     * @var array
+     */
+    private $cache=[];
     /**
      * The language currently being used  e.g. zh_CN, ja
      * @var string
@@ -83,15 +102,17 @@ class StochasticTermSegmenter
      * segmentation possibilities.
      * @param string $lang is a string to indicate the language
      */
-    function __construct($lang)
+    function __construct($lang, $cache_pct = 0.06)
     {
-        $this->lang = $lang;
+        $this->cache_pct = $cache_pct;
         /* Add different attribute for different languages
          * Currently only Chinese
          */
         switch($lang)
         {
             case "zh_CN":
+                case "zh-CN":
+                $this->lang = "zh_CN";
                 $this->non_char_preg = "/^[^\p{Han}]+$/u";
                 /*
                  * Check if the term passed in is an exception term
@@ -104,22 +125,24 @@ class StochasticTermSegmenter
                 /*
                  * Check if the term passed in is a punctuation
                  */
-                $this->isPunctuation = function($term)
+                $this->isPunctuationImpl = function($term)
                 {
-                    return preg_match("/^[" . ZH\Tokenizer::$punctuation .
-                        "]+$/u", $term);
+                    return preg_match(ZH\Tokenizer::$punctuation_preg, $term);
                 };
                 break;
             case "ja":
+                $this->lang = $lang;
                 $this->non_char_preg =
                     "/^[^\x{4E00}-\x{9FBF}\x{3040}-\x{309F}".
                     "\x{30A0}-\x{30FF}]+$/u";
                 break;
             case "ko":
+                $this->lang = $lang;
                 $this->non_char_preg = "/^[^\x{3130}-\x{318F}".
                     "\x{AC00}-\x{D7AF}]+$/u";
                 break;
             default:
+                $this->lang = $lang;
                 $this->non_char_preg = "/^[^.]+$/u";
         }
     }
@@ -165,6 +188,18 @@ class StochasticTermSegmenter
             return $this->isExceptionImpl($term);
         return false;
     }
+    /**
+     * Check if the term passed in is a punctuation
+     * isPunctuationImpl should be defined in constructor if needed
+     * @param $term is a string that to be checked
+     * @return true if $term is a punctuation, false otherwise
+     */
+    public function isPunctuation($term)
+    {
+        if (isset($this->isPunctuationImpl))
+            return $this->isPunctuationImpl($term);
+        return false;
+    }
     /**
      * Check if all the chars in the term is NOT current language
      * @param $term is a string that to be checked
@@ -179,11 +214,24 @@ class StochasticTermSegmenter
      * Generate a term dictionary file for later segmentation
      * @param mixed $text_files is a string name or an array of files
      *  that to be trained; words in the files need to be segmented by space
+     * @param string $format currently only support default and CTB
      * @param int $max_term_weights maximum number of terms ot keep
      * @return bool true if success
      */
-    public function train($text_files, $max_num_term_weights = 40000)
+    public function train($text_files, $format = "default",
+        $max_num_term_weights = 40000)
     {
+        $ctb_fmt=false;
+        switch ($format) {
+            case("default"):
+                break;
+            case("CTB"):
+                $ctb_fmt=true;
+                break;
+            default:
+                echo "Unrecognized format";
+                exit();
+        }
         $out_file = C\LOCALE_DIR .
             "/{$this->lang}/resources/term_weight.txt.gz";
         echo "Saving file to: $out_file\n";
@@ -193,10 +241,13 @@ class StochasticTermSegmenter
             $text_files = [$text_files];
         }
         foreach($text_files as $text_file) {
-            if (file_exists($text_file)) {
+            if (file_exists($text_file) && !is_dir($text_file)) {
                 $fh = fopen($text_file, "r");
                 while(!feof($fh))  {
                     $line = fgets($fh);
+                    if ($ctb_fmt and preg_match('/^<.*>$/', trim($line))) {
+                        continue;
+                    }
                     $words = preg_split("/[\s　]+/u", $line);
                     foreach ($words as $word) {
                         if ($word != "" && !$this->isException($word)
@@ -204,7 +255,8 @@ class StochasticTermSegmenter
                             if (!empty($dictionary[$word])) {
                                 $dictionary[$word]++;
                             } else if (count($dictionary) <
-                                $max_num_term_weights) {
+                                $max_num_term_weights
+                                && mb_strlen($word) < 7) {
                                 $dictionary[$word] = 1;
                             }
                         }
@@ -213,14 +265,22 @@ class StochasticTermSegmenter
                 fclose($fh);
             }
         }
-        $trie = new Trie('$');
+        $this->dictionary_file = [];
+        $this->dictionary_file["N"] = 0;
+        $this->dictionary_file["dic"] = [];
+        ksort ($dictionary);
+        $start_char = null;
+        $tmp_array=[];
         foreach ($dictionary as $key => $value) {
-            $trie->add($key . " " . $value, false);
-            $N++;
+            if (mb_substr($key,0,1)!=$start_char) {
+                $this->dictionary_file["dic"][$start_char]
+                    = json_encode($tmp_array[$start_char]);
+                $tmp_array=[];
+                $start_char=mb_substr($key,0,1);
+            }
+            $this->add($key, $value, $tmp_array);
+            $this->dictionary_file["N"]++;
         }
-        $this->dictionary_file = [];
-        $this->dictionary_file["N"] = $N;
-        $this->dictionary_file["dic"] = $trie->trie_array;
         $this->unknown_term_score = $this->getScore(1);
         file_put_contents($out_file,
             gzencode(json_encode($this->dictionary_file), 9));
@@ -290,7 +350,7 @@ class StochasticTermSegmenter
             }
         }
         if ($return_string) {
-            return $result;
+            return mb_substr($result, 0, -1);
         }
         return true;
     }
@@ -302,7 +362,7 @@ class StochasticTermSegmenter
      */
     public function segmentSentence($sentence)
     {
-        $t = explode(" ",$sentence);
+        $t=preg_split("/[\s　]+/u", trim($sentence));
         if(count($t) > 1) {
             $ret = [];
             foreach($t as $s) {
@@ -319,9 +379,14 @@ class StochasticTermSegmenter
             }
             $this->dictionary_file =
                 json_decode(gzdecode(file_get_contents($dic_file)), true);
-            gc_collect_cycles();
+            //Maybe no longer need? gc_collect_cycles();  
             $this->unknown_term_score = $this->getScore(1);
         }
+        $cache_size =
+            floor(count($this->dictionary_file['dic']) * $this->cache_pct);
+        if ($cache_size == 0) {
+            $cache_size = 1;
+        }
         preg_match_all('/./u', trim($sentence), $matches);
         $characters = $matches[0];
         if (!count($characters)) {
@@ -371,12 +436,20 @@ class StochasticTermSegmenter
             }
             //if is punctuation. Give slitely better score than unknown words
             if ($this->isPunctuation($characters[$index])) {
-                if (!isset($score[$index]) ||
-                    $score[$index]>$score[$index - 1]
-                    + $this->unknown_term_score / 1.1) {
-                    $score[$index] =
-                        $score[$index - 1] + $this->unknown_term_score / 1.1;
-                    $path[$index] = $index - 1;
+                $current_char = $characters[$index];
+                for($j = $index+1; $j<count($characters); $j++) {
+                    if (!$this->isPunctuation(
+                        $current_char . $characters[$j])) {
+                        break;            
+                    }
+                    $current_char .= $characters[$j];
+                }
+                if (!isset($score[$j - 1]) ||
+                    $score[$j - 1] > $score[$index - 1] +
+                        $this->unknown_term_score / 1.1) {
+                    $score[$j - 1] = $score[$index - 1] +
+                        $this->unknown_term_score  / 1.1;
+                    $path[$j - 1] = $index - 1;
                 }
             }
             /* All case (Even not in current lang because dictionary may
@@ -389,17 +462,29 @@ class StochasticTermSegmenter
                     $this->unknown_term_score;
                 $path[$index] = $index - 1;
             }
-            $subdic = $this->dictionary_file["dic"];
-            for ($j = $index; $j < count($characters); $j++) {
-                if (!isset($subdic[$characters[$j]])) {
-                    break;
+            //if entry exists, look for the term
+            if (isset($this->dictionary_file["dic"][$characters[$index]])) {
+                if (!isset($this->cache[$characters[$index]])) {
+                    $this->cache = [$characters[$index] => 
+                        json_decode(
+                        $this->dictionary_file["dic"][$characters[$index]],
+                        true)] + $this->cache;
+                    while (count($this->cache) > $cache_size) {
+                        array_pop($this->cache);
+                    }
                 }
-                $subdic = $subdic[$characters[$j]];
-                if (isset($subdic['$']) && (!isset($score[$j]) ||
-                    $score[$index - 1] + $subdic['$'] < $score[$j])) {
-                    $score[$j] = $score[$index - 1] +
-                        $this->getScore($subdic['$']);
-                    $path[$j] = $index - 1;
+                $subdic = $this->cache;
+                for ($j = $index; $j < count($characters); $j++) {
+                    if (!isset($subdic[$characters[$j]])) {
+                        break;
+                    }
+                    $subdic = $subdic[$characters[$j]];
+                    if (isset($subdic['$']) && (!isset($score[$j]) ||
+                        $score[$index - 1] + $subdic['$'] < $score[$j])) {
+                        $score[$j] = $score[$index - 1] +
+                            $this->getScore($subdic['$']);
+                        $path[$j] = $index - 1;
+                    }
                 }
             }
         }
@@ -431,4 +516,27 @@ class StochasticTermSegmenter
     {
       return -log($frequency / $this->dictionary_file["N"]);
     }
+    /**
+     * Adds a term to the dictionary
+     *
+     * @param string $key the term to be inserted
+     * @param string $value the frequency to be inserted
+     * @param array $array for insertion
+     */
+    private function add($key, $value, & $array)
+    {
+        $trie_array = & $array;
+        for ($i = 0; $i < mb_strlen($key,"utf-8"); $i++) {
+            $character = mb_substr($key, $i, 1, "utf-8");
+            $enc_char = $character;
+            // If letter doesnt exist then create one by
+            // assigning new array
+            if (!isset($trie_array[$enc_char])) {
+                $trie_array[$enc_char] = [];
+            }
+            $trie_array = & $trie_array[$enc_char];
+        }
+        // Set end of term marker
+        $trie_array['$'] = $value;
+    }
 }
diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index 5b5cd881..02113c79 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -78,11 +78,11 @@ class Tokenizer
      * A list of characters can be used as Chinese punctuations
      * @string
      */
-    public static $punctuation =
-    "\x{2000}-\x{206F}\x{3000}-\x{303F}\x{FF00}-\x{FF0F}" .
+    public static $punctuation_preg =
+    "/^([\x{2000}-\x{206F}\x{3000}-\x{303F}\x{FF00}-\x{FF0F}" .
     "\x{FF1A}-\x{FF20}\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}" .
     "\x{FFE0}-\x{FFEE}\x{21}-\x{2F}\x{21}-\x{2F}" .
-    "\x{3A}-\x{40}\x{5B}-\x{60}";
+    "\x{3A}-\x{40}\x{5B}-\x{60}]|[\—]+|[\.]+)$/u";
     /**
      * Removes the stop words from the page (used for Word Cloud generation
      * and language detection)
@@ -114,7 +114,7 @@ class Tokenizer
         switch($method) {
             case("RMM"):
                 return PhraseParser::reverseMaximalMatch($pre_segment, "zh-CN",
-                ['/\d+/', '/[a-zA-Z]+/']);
+                ['/^\d+$/', '/^[a-zA-Z]+$/']);
                 break;
             case("STS"):
                 $segmenter = new StochasticTermSegmenter("zh_CN");
-- 
2.21.0.windows.1

