_binary From ae58c7eba8e679f04122b374feeb473a443e6dcb Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Tue, 4 Feb 2020 22:17:37 -0800
Subject: [PATCH] make the segmenter static instead of creating and destorying
 it everytime r=chris

---
 src/library/StochasticTermSegmenter.php  |  4 +--
 src/locale/zh_CN/resources/Tokenizer.php | 33 ++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/library/StochasticTermSegmenter.php b/src/library/StochasticTermSegmenter.php
index 474be126..c933be1b 100644
--- a/src/library/StochasticTermSegmenter.php
+++ b/src/library/StochasticTermSegmenter.php
@@ -66,7 +66,7 @@ class StochasticTermSegmenter
      */
     private $cache_pct;
     /**
-     * Used in memory saving mode
+     * Cache. Will have runtime data for the segmentation
      * @var array
      */
     private $cache=[];
@@ -111,7 +111,7 @@ class StochasticTermSegmenter
         switch($lang)
         {
             case "zh_CN":
-                case "zh-CN":
+            case "zh-CN":
                 $this->lang = "zh_CN";
                 $this->non_char_preg = "/^[^\p{Han}]+$/u";
                 /*
diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index 02113c79..2b54402e 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -83,6 +83,11 @@ class Tokenizer
     "\x{FF1A}-\x{FF20}\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}" .
     "\x{FFE0}-\x{FFEE}\x{21}-\x{2F}\x{21}-\x{2F}" .
     "\x{3A}-\x{40}\x{5B}-\x{60}]|[\—]+|[\.]+)$/u";
+    /**
+     * Stochastic Term Segmenter instance
+     * @object
+     */
+    private static $stochasticTermSegmenter;
     /**
      * Removes the stop words from the page (used for Word Cloud generation
      * and language detection)
@@ -117,8 +122,8 @@ class Tokenizer
                 ['/^\d+$/', '/^[a-zA-Z]+$/']);
                 break;
             case("STS"):
-                $segmenter = new StochasticTermSegmenter("zh_CN");
-                return $segmenter->segmentText($pre_segment,true);
+                return self::getStochasticTermSegmenter()
+                        ->segmentText($pre_segment,true);
                 break;
         }
     }
@@ -148,4 +153,28 @@ class Tokenizer
         "]+(年|年代|月|日|时|小时|時|小時|" .
         "点|点钟|點|點鐘|分|分鐘|秒|秒鐘)$/u",$term);
     }
+    /*
+     * Create stochastic term segmenter
+     */
+    public static function createStochasticTermSegmenter($cache_pct=0.06)
+    {
+        self::$stochasticTermSegmenter
+            = new StochasticTermSegmenter("zh_CN", $cache_pct);
+    }
+    /*
+     * Destory stochastic term segmenter
+     */
+    public static function destoryStochasticTermSegmenter()
+    {
+        self::$stochasticTermSegmenter = null;
+    }
+    /*
+     * Get the segmenter instance
+     */
+    public static function getStochasticTermSegmenter() {
+        if (!self::$stochasticTermSegmenter) {
+            self::createStochasticTermSegmenter();
+        }
+        return self::$stochasticTermSegmenter;
+    }
 }
-- 
2.21.0.windows.1

