_binary From ada1c4cac067476de19b9f1dfe2a408cee0cb41a Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Tue, 21 Jul 2020 14:46:18 -0700
Subject: [PATCH] add the missing tokenizer.php r=chris

---
 src/locale/zh_CN/resources/Tokenizer.php | 33 ++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index 5bf9ef0a..e82edcef 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -213,6 +213,12 @@ class Tokenizer
      * @var object
      */
     private static $pos_tagger;
+    /**
+     * Trandition Character to Simplified for normalization
+     * Assigned once up on use
+     * @var array
+     */
+    private static $zh2HansChar;
     /**
      * Removes the stop words from the page (used for Word Cloud generation
      * and language detection)
@@ -380,6 +386,33 @@ class Tokenizer
         }
         return self::$pos_tagger;
     }
+    /**
+     * Normalize the text
+     * @param  string $text is a string of Chinese Char
+     * @return string normalized form of the text
+     */
+    public static function normalize($text)
+    {
+        if (!self::$zh2HansChar) {
+            $path = C\LOCALE_DIR . "/zh_CN/resources/zh2HansChar.txt.gz";
+            if (!file_exists($path)) {
+                //crawlLog("{$path} does not exist!");
+                return $text;
+            }
+            $chars=gzdecode(file_get_contents($path));
+            $chars=preg_split('//u', $chars, null, PREG_SPLIT_NO_EMPTY);
+            for($i=0, self::$zh2HansChar=[]; $i<count($chars)/2; ++$i) {
+                self::$zh2HansChar[$chars[$i*2]]=$chars[$i*2+1];
+            }
+        }
+        $chars=preg_split('//u', $text, null, PREG_SPLIT_NO_EMPTY);
+        for($i=0; $i<count($chars); ++$i) {
+            if (isset(self::$zh2HansChar[$chars[$i]])) {
+                $chars[$i] = self::$zh2HansChar[$chars[$i]];
+            }
+        }
+        return implode($chars);
+    }
     /**
      * Scans a word list for phrases. For phrases found generate
      * a list of question and answer pairs at two levels of granularity:
-- 
2.21.0.windows.1

