_binary From dc540829398053ce05af821b8efd4c41db94f0c0 Mon Sep 17 00:00:00 2001
From: Forrest Sun <sxh19911230@gmail.com>
Date: Thu, 14 May 2020 02:57:17 -0700
Subject: [PATCH] Chinese QA version 1.0. r=chris

---
 src/library/ContextWeightedPosTagger.php |  10 +-
 src/library/PhraseParser.php             |   2 +-
 src/locale/zh_CN/resources/Tokenizer.php | 916 ++++++++++++++++++++++-
 3 files changed, 920 insertions(+), 8 deletions(-)

diff --git a/src/library/ContextWeightedPosTagger.php b/src/library/ContextWeightedPosTagger.php
index d49f252b..d354d3fd 100644
--- a/src/library/ContextWeightedPosTagger.php
+++ b/src/library/ContextWeightedPosTagger.php
@@ -77,6 +77,10 @@ class ContextWeightedPosTagger
      * @var associative array [tag => tag index]
      */
     private $tag_set;
+    /**
+     * The unknown words should be picked from these tags
+     */
+    private $unknown_word_possiable_tags=[];
     /**
      * Check if all the chars in the term is not current language
      * @param $term is a string that to be checked
@@ -112,6 +116,8 @@ class ContextWeightedPosTagger
                 };
                 //Tags from above
                 $this->rule_defined_key = ['PU','CD','OD','NT','FW'];
+                //Unknown word possiable tag
+                $this->unknown_word_possiable_tags=["NN","NR","VV","VA"];
                 break;
             default:
                 $this->lang = $lang;
@@ -412,8 +418,8 @@ class ContextWeightedPosTagger
                     if (isset($this->w[$k])) {
                         $score[$possiable_tag] +=
                                 $this->getW($k,$j,$tag_index);
-                    } else if ($j==0&&in_array($possiable_tag,
-                        $this->rule_defined_key)) {
+                    } else if ($j==0&&!in_array($possiable_tag,
+                        $this->unknown_word_possiable_tags)) {
                         $score[$possiable_tag] += $this->min_w;
                     }
                 }
diff --git a/src/library/PhraseParser.php b/src/library/PhraseParser.php
index 185e9b4b..6ded427d 100755
--- a/src/library/PhraseParser.php
+++ b/src/library/PhraseParser.php
@@ -417,7 +417,7 @@ class PhraseParser
             "tagTokenizePartOfSpeech") &&
             !isset(self::$programming_language_map[$lang])) {
             $string = mb_strtolower($string);
-            $pre_sentences = preg_split("/(\n\n+)|\.|\!|\?|。/u", $string);
+            $pre_sentences = preg_split("/(\n\n+)|\.|\!|\?|。|！|？/u", $string);
             $pos = 1;
             $sentences_pos = [];
             $sentences = [];
diff --git a/src/locale/zh_CN/resources/Tokenizer.php b/src/locale/zh_CN/resources/Tokenizer.php
index ede2bd98..cd45cdf3 100755
--- a/src/locale/zh_CN/resources/Tokenizer.php
+++ b/src/locale/zh_CN/resources/Tokenizer.php
@@ -27,10 +27,9 @@
  * @filesource
  */
 namespace seekquarry\yioop\locale\zh_CN\resources;
-
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
 use seekquarry\yioop\library\PhraseParser;
-use seekquarry\yioop\library\StochasticTermSegmenter;
-use seekquarry\yioop\library\ContextWeightedNamedEntityRecognizer;
 
 /**
  * Chinese specific tokenization code. Typically, tokenizer.php
@@ -100,6 +99,103 @@ class Tokenizer
     "\x{FF1A}-\x{FF20}\x{FF3B}-\x{FF40}\x{FF5B}-\x{FF65}" .
     "\x{FFE0}-\x{FFEE}\x{21}-\x{2F}\x{21}-\x{2F}" .
     "\x{3A}-\x{40}\x{5B}-\x{60}\x{25cf}])\\1*$/u";
+    /**
+     * Any unique identifier corresponding to the component of a triplet which
+     * can be answered using a question answer list
+     * @string
+     */
+    public static $question_token = "qqq";
+    /**
+     * Words array that determine if a sentence passed in is a question
+     * @array
+     */
+    public static $question_words=[
+        "any"=>["谁"=>"who",
+                "哪儿|哪里"=>"where",
+                "哪个"=>"which",
+                "哪些"=>"list",
+                "哪"=>["after"=>[   "1|一"=>"which",
+                                    "[2-9]|[1-9][0-9]+"=>"list"
+                                ],
+                       "other"=>"where"
+                        ],
+                "什么|啥|咋"=>[ "after"=>[    "地方"=>"where",
+                                            "地点"=>"where",
+                                            "时\w*"=>"when"
+                                       ],
+                                "other"=>"what"],
+                "怎么|怎样|怎么样|如何"=>"how", 
+                "为什么"=>"why", 
+                "多少"=>"how many",
+                "几\w*"=>["any"=>["吗|\?|？"=>"how many"], "other"=>false],
+                "多久"=>"how long", 
+                "多大"=>"how big"
+                ],
+        "other"=>[  "any"=>[    "吗"=>"yesno", 
+                                "呢"=>"what about"
+                           ],
+                    "other"=>[  "other"=>false,
+                                "any"=>["\?|？"=>"yesno"]
+                             ]
+                 ]
+        ];
+    /**
+     * List of adjective-like parts of speech that might appear in lexicon file
+     * Predicative adjective: VA
+     * other noun-modifier: JJ
+     * @array
+     */
+    public static $adjective_type = ["VA","JJ"];
+    /**
+     * List of adverb-like parts of speech that might appear in lexicon file
+     * @array
+     */
+    public static $adverb_type = ["AD"];
+    /**
+     * List of conjunction-like parts of speech that might appear in lexicon
+     * file
+     * Coordinating conjunction: CC
+     * Subordinating conjunction: CS
+     * @array
+     */
+    public static $conjunction_type = ["CC","CS"];
+    /**
+     * List of determiner-like parts of speech that might appear in lexicon
+     * file
+     * Determiner: DT
+     * Cardinal Number: CD
+     * Ordinal Number: OD
+     * Measure word: M
+     * @array
+     */
+    public static $determiner_type = ["DT", "CD", "OD", "M"];
+    /**
+     * List of noun-like parts of speech that might appear in lexicon file
+     * Proper Noun: NR
+     * Temporal Noun: NT
+     * Other Noun: NN
+     * Pronoun: PN
+     * @array
+     */
+    public static $noun_type = ["NR", "NT", "NN", "PN"];
+    /**
+     * List of verb-like parts of speech that might appear in lexicon file
+     * Copula: VC
+     * you3 as the main verb: VE
+     * Other verb: VV
+     * Short passive voice: SB
+     * Long passive voice: LB
+     * @array
+     */
+    public static $verb_type = ["VC", "VE", "VV", "SB", "LB"];
+    /**
+     * List of particle-like parts of speech that might appear in lexicon file
+     * No meaning words that can appear anywhere
+     * @array
+     */
+    public static $particle_type = [
+        "AS", "ETC", "DEC", "DEG", "DEV", "MSP",
+        "DER", "SP", "IJ", "FW"];
     /**
      * Stochastic Term Segmenter instance
      * @object
@@ -110,6 +206,11 @@ class Tokenizer
      * @object
      */
     private static $namedEntityRecognizer;
+    /**
+     * PosTagger instance
+     * @object
+     */
+    private static $posTagger;
     /**
      * Removes the stop words from the page (used for Word Cloud generation
      * and language detection)
@@ -205,7 +306,7 @@ class Tokenizer
     public static function createStochasticTermSegmenter($cache_pct=0.06)
     {
         self::$stochasticTermSegmenter
-            = new StochasticTermSegmenter("zh_CN", $cache_pct);
+            = new L\StochasticTermSegmenter("zh_CN", $cache_pct);
     }
     /*
      * Destory stochastic term segmenter
@@ -243,7 +344,7 @@ class Tokenizer
     public static function createNER()
     {
         self::$namedEntityRecognizer
-            = new ContextWeightedNamedEntityRecognizer("zh_CN");
+            = new L\ContextWeightedNamedEntityRecognizer("zh_CN");
     }
     /*
      * Destory named entity recognizer instance
@@ -261,4 +362,809 @@ class Tokenizer
         }
         return self::$namedEntityRecognizer;
     }
+    /*
+     * Create POSTagger instance
+     */
+    public static function createPosTagger()
+    {
+        self::$posTagger
+            = new L\ContextWeightedPosTagger("zh_CN");
+    }
+    /*
+     * Destory POSTagger instance
+     */
+    public static function destoryPosTagger()
+    {
+        self::$posTagger = null;
+    }
+    /*
+     * Get PosTagger instance
+     */
+    public static function getPosTagger() {
+        if (!self::$posTagger) {
+            self::createPosTagger();
+        }
+        return self::$posTagger;
+    }
+    /**
+     * Scans a word list for phrases. For phrases found generate
+     * a list of question and answer pairs at two levels of granularity:
+     * CONCISE (using all terms in orginal phrase) and RAW (removing
+     * (adjectives, etc).
+     *
+     * @param array $word_and_phrase_list of statements
+     * @return array with two fields: QUESTION_LIST consisting of triplets
+     *      (SUBJECT, PREDICATES, OBJECT) where one of the components has been
+     *      replaced with a question marker.
+     */
+    public static function extractTripletsPhrases($word_and_phrase_list)
+    {
+        $triplets_list = [];
+        $question_list = [];
+        $question_answer_list = [];
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($word_and_phrase_list as $word_and_phrase => $position_list) {
+            // strip parentheticals
+            $word_and_phrase = preg_replace("/[\{\[\(【（][^\}\]）】\)]+[\}\]\)）】]/u",
+                "", $word_and_phrase);
+            $tagged_phrase = self::tagTokenizePartOfSpeech($word_and_phrase);
+            $parse_tree = ['cur_node' => 0];
+            $extracted_triplets=[];
+            $pre_sub=[];
+            while ($parse_tree['cur_node'] < count($tagged_phrase)) {
+                $parse_tree = self::parseWholePhrase($tagged_phrase,
+                ['cur_node'=>$parse_tree['cur_node']],$pre_sub);
+                $triplets = self::extractTripletsParseTree($parse_tree);
+                if (isset($parse_tree['NP'])) {
+                    $pre_sub = $parse_tree['NP'];
+                }
+                $extracted_triplets_set[] = self::rearrangeTripletsByType($triplets);
+                // next partial sentence
+                while($parse_tree['cur_node'] < count($tagged_phrase)
+                    && $tagged_phrase[$parse_tree['cur_node']]["tag"] != "PU") {
+                    $parse_tree['cur_node']++;
+                }
+                $parse_tree['cur_node']++;
+            }
+            foreach($extracted_triplets_set as $extracted_triplets) {
+                foreach ($triplet_types as $type) {
+                    if (!empty($extracted_triplets[$type])) {
+                        $triplets = $extracted_triplets[$type];
+                        $questions = $triplets['QUESTION_LIST'];
+                        foreach ($questions as $question) {
+                            $question_list[$question] = $position_list;
+                        }
+                        $question_answer_list = array_merge($question_answer_list,
+                            $triplets['QUESTION_ANSWER_LIST']);
+                    }
+                }
+            }
+        }
+        $out_triplets['QUESTION_LIST'] = $question_list;
+        $out_triplets['QUESTION_ANSWER_LIST'] = $question_answer_list;
+        return $out_triplets;
+    }
+    /**
+     * Split input text into terms and output an array with one element
+     * per term, that element consisting of array with the term token
+     * and the part of speech tag.
+     *
+     * @param string $text string to tag and tokenize
+     * @return array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term) for one each token in $text
+     */
+    public static function tagTokenizePartOfSpeech($text)
+    {
+        $segmented = self::getStochasticTermSegmenter()->segmentSentence($text);
+        $tags = self::getPosTagger()->predict($segmented);
+        $result=[];
+        for($i=0; $i<count($segmented); $i++) {
+            $result[$i]=[];
+            $result[$i]["token"]=$segmented[$i];
+            $result[$i]["tag"]=$tags[$i];
+        }
+        return $result;
+    }
+    /**
+     * Starting at the $cur_node in a $tagged_phrase parse tree for an English
+     * sentence, create a phrase string for each of the next nodes
+     * which belong to part of speech group $type.
+     *
+     * @param array& $cur_node node within parse tree
+     * @param array $tagged_phrase parse tree for phrase
+     * @param string $type self::$noun_type, self::$verb_type, etc
+     * @return string phrase string involving only terms of that $type
+     */
+    public static function parseTypeList(&$cur_node, $tagged_phrase, $type)
+    {
+        $string = "";
+        $previous_string = "";
+        $previous_tag = "";
+        $start_node = $cur_node;
+        $next_tag = (empty($tagged_phrase[$cur_node]['tag'])) ? "" :
+            trim($tagged_phrase[$cur_node]['tag']);
+        $allowed_conjuncts = [];
+        while ($next_tag && (in_array($next_tag, $type) ||
+            in_array($next_tag, $allowed_conjuncts))) {
+            $previous_string = $string;
+            $string .= $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+            $allowed_conjuncts = self::$conjunction_type;
+            $previous_tag = $next_tag;
+            $next_tag = (empty($tagged_phrase[$cur_node]['tag'])) ? "" :
+                trim($tagged_phrase[$cur_node]['tag']);
+        }
+        if (in_array($previous_tag, $allowed_conjuncts) && $start_node <
+            $cur_node) {
+            $cur_node--;
+            $string = $previous_string;
+        }
+        return $string;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for an adjective if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "JJ" a subarray with a token node for the adjective that was
+     *      parsed
+     */
+    public static function parseAdjective($tagged_phrase, $tree)
+    {
+        $adjective_string = self::parseTypeList($tree['cur_node'],
+            $tagged_phrase, self::$adjective_type);
+        if (!empty($adjective_string)) {
+            $tree["JJ"] = $adjective_string;
+            $partical_string = self::parseTypeList($tree['cur_node'],
+                $tagged_phrase,self::$particle_type);
+            if ($partical_string) {
+                $tree["PARTICAL"] = $partical_string;
+            }
+        }
+        return $tree;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a determiner if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "DT" a subarray with a token node for the determiner that was
+     *      parsed
+     */
+    public static function parseDeterminer($tagged_phrase, $tree)
+    {
+        $determiner_string = "";
+        /* In: All the cows low, "All the" is considered a determiner.
+           That is, we will mush together the predeterminer with the determiner
+         */
+        $determiner_string = self::parseTypeList($tree['cur_node'],
+            $tagged_phrase, self::$determiner_type);
+        if (!empty($determiner_string)) {
+            $tree["DT"] = $determiner_string;
+            $partical_string = self::parseTypeList($tree['cur_node'],
+                $tagged_phrase,self::$particle_type);
+            if ($partical_string) {
+                $tree["PARTICAL"] = $partical_string;
+            }
+        }
+        return $tree;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "NN" a subarray with a token node for the noun string that was
+     *      parsed
+     */
+    public static function parseNoun($tagged_phrase, $tree)
+    {
+        //Combining multiple noun into one
+        $noun_string = self::parseTypeList($tree['cur_node'], $tagged_phrase,
+            self::$noun_type);
+        if (!empty($noun_string)) {
+            $tree["NN"] = $noun_string;
+            $partical_string = self::parseTypeList($tree['cur_node'],
+                $tagged_phrase,self::$particle_type);
+            if ($partical_string) {
+                $tree["PARTICAL"] = $partical_string;
+            }
+        }
+        return $tree;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VB" a subarray with a token node for the verb string that was
+     *      parsed
+     */
+    public static function parseVerb($tagged_phrase, $tree)
+    {
+        $verb_string = self::parseTypeList($tree['cur_node'], $tagged_phrase,
+            self::$verb_type);
+        if (!empty($verb_string)) {
+            $tree["VB"] = $verb_string;
+            $partical_string = self::parseTypeList($tree['cur_node'],
+                $tagged_phrase,self::$particle_type);
+            if ($partical_string) {
+                $tree["PARTICAL"] = $partical_string;
+            }
+        }
+        return $tree;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a sequence of
+     * prepositional phrases if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @param int $index which term in $tagged_phrase to start to try to parse
+     *      a preposition from
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      parsed followed by additional possible fields (here i
+     *      represents the ith clause found):
+     *      "IN_i" with value a preposition subtree
+     *      "DT_i" with value a determiner subtree
+     *      "JJ_i" with value an adjective subtree
+     *      "NN_i"  with value an additional noun subtree
+     */
+    public static function parsePrepositionalPhrases($tagged_phrase, $tree,
+        $index = 1)
+    {
+        $cur_node = $tree['cur_node'];
+        // There are two forms of prepostion.
+        // First one has lc only
+        // 之前(lc) 他在看书
+        if (isset($tagged_phrase[$cur_node]['tag']) &&
+            trim($tagged_phrase[$cur_node]['tag']) == "LC") {
+            $tree["LC"] = $tagged_phrase[$cur_node]['token'];
+            $tree['cur_node']+=1;
+            return $tree;
+        }
+        // Second form:
+        // format: prep [anything] [locolizer|punctuation]
+        // 在(p)今天早上，(pu) 他 在(p) 车 里(lc) 睡觉。
+        // In the morning today, he was sleeping in the car.
+        if (isset($tagged_phrase[$cur_node]['tag']) &&
+            trim($tagged_phrase[$cur_node]['tag']) == "P") {
+            /* can have multiple prep's in a row, for example,
+               it is known in over 20 countries*/
+            $preposition_string = self::parseTypeList($cur_node, $tagged_phrase,
+                ["P"]);
+            if (!empty($preposition_string)) {
+                $tree["P"] = $preposition_string;
+            }
+            while(isset($tagged_phrase[$cur_node]) &&
+                isset($tagged_phrase[$cur_node]['tag']) &&
+                !in_array($tagged_phrase[$cur_node]['tag'],["PU","LC"])) {
+                $tree["P"] .= $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            $lc_string = self::parseTypeList($cur_node, $tagged_phrase,
+                ["LC"]);
+            if ($lc_string) {
+                $tree["P"] .= $lc_string;
+            }
+            $partical_string = self::parseTypeList($cur_node,
+                $tagged_phrase,self::$particle_type);
+            if ($partical_string) {
+                $tree["PARTICAL"] = $partical_string;
+            }
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "NP" a subarray with possible fields
+     *      "DT" with value a determiner subtree
+     *      "JJ" with value an adjective subtree
+     *      "NN" with value a noun tree
+     */
+    public static function parseNounPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_dt = self::parseDeterminer($tagged_phrase,
+            ['cur_node' => $cur_node]);
+        $tree_jj = self::parseAdjective($tagged_phrase,
+            ['cur_node' => $tree_dt['cur_node']]);
+        $tree_pp = self::parsePrepositionalPhrases($tagged_phrase,
+            ['cur_node' => $tree_jj['cur_node']]);
+        $tree_nn = self::parseNoun($tagged_phrase,
+            ['cur_node' => $tree_pp['cur_node']]);
+        if ($tree_nn['cur_node'] == $tree_pp['cur_node']) {
+            $tree['NP'] = "";
+            return $tree;
+        }
+        $tree_pp2 = self::parsePrepositionalPhrases($tagged_phrase,
+            ['cur_node' => $tree_nn['cur_node']]);
+        $cur_node = $tree_pp2['cur_node'];
+        $cc = "";
+        if (!empty($tagged_phrase[$cur_node]['tag']) &&
+            in_array($tagged_phrase[$cur_node]['tag'],
+            self::$conjunction_type)) {
+            $cc = $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        $tree_np = self::parseNounPhrase($tagged_phrase,
+            ['cur_node' => $cur_node]);
+        if ($tree_np['cur_node'] == $cur_node && $cc) {
+            $cur_node--;
+            $tree_np = [];
+            $cc = "";
+        } else {
+            $cur_node = $tree_np['cur_node'];
+        }
+        unset($tree_dt['cur_node'], $tree_jj['cur_node'],
+            $tree_nn['cur_node'], $tree_pp['cur_node'],
+            $tree_np['cur_node'], $tree_pp2['cur_node']);
+        $sub_tree = ['DT' => $tree_dt, 'JJ' => $tree_jj, 'PRP' => $tree_pp,
+        'NN' => $tree_nn, 'PRP2'=>$tree_pp2,'CC' => $cc,
+            'ADD_NP' => $tree_np];
+        return ['cur_node' => $cur_node, 'NP' => $sub_tree];
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VP" a subarray with possible fields
+     *      "VB" with value a verb subtree
+     *      "NP" with value an noun phrase subtree
+     */
+    public static function parseVerbPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $adverb_string="";
+        do {
+            $start_node=$cur_node;
+            $adverb_string .= self::parseTypeList($cur_node, $tagged_phrase,
+                self::$adverb_type);
+            $adverb_string .= self::parseTypeList($cur_node, $tagged_phrase,
+                self::$particle_type);
+        } while($start_node!=$cur_node);
+        if (!empty($adverb_string)) {
+            $tree_vb["RB"] = $adverb_string;
+        } else {
+            $tree_vb=[];
+        }
+        $tree_vb = array_merge($tree_vb,
+            self::parseVerb($tagged_phrase, ['cur_node' => $cur_node]));
+        if ($cur_node == $tree_vb['cur_node']) {
+            // if no verb return what started with
+            return $tree;
+        }
+        $cur_node = $tree_vb['cur_node'];
+        $tree_np = self::parseNounPhrase($tagged_phrase,
+            ['cur_node' => $tree_vb['cur_node']]);
+        $cur_node = $tree_np['cur_node'];
+        if (!empty($tree_np['NP'])) {
+            unset($tree_vb['cur_node'], $tree_np['cur_node']);
+            return ['VP' => ['VB' => $tree_vb, 'NP' => $tree_np['NP']],
+                'cur_node' => $cur_node];
+        }
+        unset($tree_vb['cur_node']);
+        return ['VP' => ['VB' => $tree_vb], 'cur_node' => $cur_node];
+    }
+    /**
+     * Given a part-of-speeech tagged phrase array generates a parse tree
+     * for the phrase using a recursive descent parser.
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @param $tree_np_pre subject found from previous sub-sentence
+     * @return array used to represent a tree. The array has up to three fields
+     *      $tree["cur_node"] index of how far we parsed our$tagged_phrase
+     *      $tree["NP"] contains a subtree for a noun phrase
+     *      $tree["VP"] contains a subtree for a verb phrase
+     */
+    public static function parseWholePhrase($tagged_phrase, $tree, $tree_np_pre=[])
+    {
+        //remove heading adverbs
+        $cur_node = $tree['cur_node'];
+        do {
+            $start_node=$cur_node;
+            self::parseTypeList($cur_node, $tagged_phrase,
+                self::$adverb_type);
+            self::parseTypeList($cur_node, $tagged_phrase,
+                self::$particle_type);
+        } while($start_node!=$cur_node);
+        $tree_np = self::parseNounPhrase($tagged_phrase,
+            ["cur_node" => $cur_node]);
+        if ($tree_np['cur_node'] == $cur_node) {
+            if (!empty($tree_np_pre)) {
+                $tree_np['NP']=$tree_np_pre;
+                $tree_np["cur_node"]=$cur_node;
+            } else {
+                return $tree;
+            }
+        }
+        $tree_vp = self::parseVerbPhrase($tagged_phrase,
+            ["cur_node" => $tree_np['cur_node']]);
+        if ($tree_np['cur_node'] == $tree_vp['cur_node']) {
+            return $tree;
+        }
+        $cur_node = $tree_vp['cur_node'];
+        unset($tree_np['cur_node'], $tree_vp['cur_node']);
+        if (!empty($tree_start) && !empty($tree_np['NP'])) {
+            $tree_np['NP']['PRP-1'] = $tree_start;
+        }
+        return ['cur_node' => $cur_node, 'NP' => $tree_np['NP'],
+            'VP' => $tree_vp['VP']];
+    }
+    /**
+     * Takes a parse tree of a phrase and computes subject, predicate, and
+     * object arrays. Each of these array consists of two components CONCISE and
+     * RAW, CONCISE corresponding to something more similar to the words in the
+     * original phrase and RAW to the case where extraneous words have been
+     * removed
+     *
+     * @param are $tree a parse tree for a sentence
+     * @return array triplet array
+     */
+    public static function extractTripletsParseTree($tree)
+    {
+        $triplets = [];
+        $triplets['subject'] = self::extractSubjectParseTree($tree);
+        $triplets['predicate'] = self::extractPredicateParseTree($tree);
+        $triplets['object'] = self::extractObjectParseTree($tree);
+        return $triplets;
+    }
+    /**
+     * Takes phrase tree $tree and a part-of-speech $pos returns
+     * the deepest $pos only path in tree.
+     *
+     * @param array $tree phrase to extract type from
+     * @param string $pos the part of speech to extract
+     * @return string the label of deepest $pos only path in $tree
+     */
+    public static function extractDeepestSpeechPartPhrase($tree, $pos)
+    {
+        $extract = "";
+        if (!empty($tree[$pos])) {
+            $extract = self::extractDeepestSpeechPartPhrase($tree[$pos], $pos);
+        }
+        if (!$extract && !empty($tree[$pos]) && !empty($tree[$pos][$pos])) {
+            $extract = $tree[$pos][$pos];
+        }
+        return $extract;
+    }
+    /**
+     * Takes a parse tree of a phrase or statement and returns an array
+     * with two fields CONCISE and RAW the former having the object of
+     * the original phrase (as a string) the latter having the importart
+     * parts of the object
+     *
+     * @param array representation of a parse tree of a phrase
+     * @return array with two fields CONCISE and RAW as described above
+     */
+    public static function extractObjectParseTree($tree)
+    {
+        $object = [];
+        if (!empty($tree['VP'])) {
+            $tree_vp = $tree['VP'];
+            if (!empty($tree_vp['NP'])) {
+                $nb = $tree_vp['NP'];
+                $object['CONCISE'] = $tree_vp['NP'];
+                while (is_array($object['CONCISE']["ADD_NP"]["NP"])) {
+                    $object['CONCISE'] = $object['CONCISE']["ADD_NP"]["NP"];
+                }
+                $object['CONCISE']=$object['CONCISE']["NN"]["NN"];
+                $raw_object = "";
+                $it = new \RecursiveIteratorIterator(
+                    new \RecursiveArrayIterator($nb));
+                foreach ($it as $v) {
+                    $raw_object .= $v;
+                }
+                $object['RAW'] = $raw_object;
+            } else {
+                $object['CONCISE'] = "";
+                $object['RAW'] = "";
+            }
+        } else {
+            $object['CONCISE'] = "";
+            $object['RAW'] = "";
+        }
+        return $object;
+    }
+    /**
+     * Takes a parse tree of a phrase or statement and returns an array
+     * with two fields CONCISE and RAW the former having the predicate of
+     * the original phrase (as a string) the latter having the importart
+     * parts of the predicate
+     *
+     * @param array representation of a parse tree of a phrase
+     * @return array with two fields CONCISE and RAW as described above
+     */
+    public static function extractPredicateParseTree($tree)
+    {
+        $predicate = [];
+        if (!empty($tree['VP'])) {
+            $tree_vp = $tree['VP'];
+            $predicate['CONCISE'] = self::extractDeepestSpeechPartPhrase(
+                $tree_vp, "VB");
+            $raw_predicate = "";
+            if (!empty($tree_vp['VB'])) {
+                $tree_vb = $tree_vp['VB'];
+                $it = new \RecursiveIteratorIterator(
+                    new \RecursiveArrayIterator($tree_vb));
+                foreach ($it as $v) {
+                    $raw_predicate .= $v;
+                }
+                $predicate['RAW'] = $raw_predicate;
+            }
+        } else {
+            $predicate['CONCISE'] = "";
+            $predicate['RAW'] = "";
+        }
+        return $predicate;
+    }
+    /**
+     * Takes a parse tree of a phrase or statement and returns an array
+     * with two fields CONCISE and RAW the former having the subject of
+     * the original phrase (as a string) the latter having the importart
+     * parts of the subject
+     *
+     * @param array representation of a parse tree of a phrase
+     * @return array with two fields CONCISE and RAW as described above
+     */
+    public static function extractSubjectParseTree($tree)
+    {
+        $subject = [];
+        if (!empty($tree['NP'])) {
+            $subject['CONCISE'] = $tree['NP'];
+            while (is_array($subject['CONCISE']["ADD_NP"]["NP"])) {
+                $subject['CONCISE'] = $subject['CONCISE']["ADD_NP"]["NP"];
+            }
+            $subject['CONCISE']=$subject['CONCISE']["NN"]["NN"];
+            $raw_subject = "";
+            $it = new \RecursiveIteratorIterator(
+                new \RecursiveArrayIterator($tree['NP']));
+            foreach ($it as $v) {
+                $raw_subject .= $v ;
+            }
+            $subject['RAW'] = $raw_subject;
+        } else {
+            $subject['CONCISE'] = "";
+            $subject['RAW'] = "";
+        }
+        return $subject;
+    }
+    /**
+     * Takes a triplets array with subject, predicate, object fields with
+     * CONCISE and RAW subfields and rearranges it to have two fields CONCISE
+     * and RAW with subject, predicate, object, and QUESTION_ANSWER_LIST
+     * subfields
+     *
+     * @param array $sub_pred_obj_triplets in format described above
+     * @return array $processed_triplets in format described above
+     */
+    public static function rearrangeTripletsByType($sub_pred_obj_triplets)
+    {
+        $processed_triplet = [];
+        $processed_triplets['CONCISE'] =
+            self::extractTripletByType($sub_pred_obj_triplets, "CONCISE");
+        $processed_triplets['RAW'] =
+            self::extractTripletByType($sub_pred_obj_triplets, "RAW");
+        return $processed_triplets;
+    }
+    /**
+     * Takes a triplets array with subject, predicate, object fields with
+     * CONCISE, RAW subfields and produces a triplits with $type subfield (where
+     * $type is one of CONCISE and RAW) and with subject, predicate, object,
+     * and QUESTION_ANSWER_LIST subfields
+     *
+     * @param array $sub_pred_obj_triplets  in format described above
+     * @param string $type either CONCISE or RAW
+     * @return array $triplets in format described above
+     */
+    public static function extractTripletByType($sub_pred_obj_triplets, $type)
+    {
+        $triplets = [];
+        if (!empty($sub_pred_obj_triplets['subject'][$type])
+            && !empty($sub_pred_obj_triplets['predicate'][$type])
+            && !empty($sub_pred_obj_triplets['object'][$type])) {
+            $question_answer_triplets = [];
+            $sentence = [ trim($sub_pred_obj_triplets['subject'][$type]),
+                trim($sub_pred_obj_triplets['predicate'][$type]),
+                trim($sub_pred_obj_triplets['object'][$type])];
+            $question_triplets = [];
+            for ($j = 0; $j < 2; $j++) {
+                if ($j == 1 && in_array($sentence[1], ['是'])) {
+                    $tmp = $sentence[2];
+                    $sentence[2] = $sentence[0];
+                    $sentence[0] = $tmp;
+                } else if ($j == 1) {
+                    break;
+                }
+                for ($i = 0; $i < 3; $i++) {
+                    $q_sentence = $sentence;
+                    $q_sentence[$i] = self::$question_token;
+                    $q_sentence_string = implode(" ", $q_sentence);
+                    $question_triplets[] = $q_sentence_string;
+                    $question_answer_triplets[$q_sentence_string] =
+                        preg_replace('/\s+/u', ' ',$sentence[$i]);
+                }
+            }
+            $triplets['QUESTION_LIST'] = $question_triplets;
+            $triplets['QUESTION_ANSWER_LIST'] = $question_answer_triplets;
+        }
+        return $triplets;
+    }
+    /**
+     * Takes any question started with WH question and returns the
+     * triplet from the question
+     *
+     * @param string $question question to parse
+     * @return array question triplet
+     */
+    public static function questionParser($question)
+    {
+        $tagged_question = self::tagTokenizePartOfSpeech($question);
+        $generated_questions = [];
+        $keywords = self::isQuestion($question);
+        if ($keywords) {
+            $generated_questions=self::parseQuestion(
+                        $tagged_question, 1, $keywords);
+        }
+        return $generated_questions;
+    }
+    /**
+     * Takes a phrase query entered by user and return true if it is question
+     * and false if not
+     *
+     * @param $phrase any statement
+     * @return bool returns question word if statement is question
+     */
+    public static function isQuestion($phrase)
+    {
+        $terms=self::getStochasticTermSegmenter()->segmentSentence($phrase);
+        $qt=self::questionType($terms, self::$question_words);
+        if (in_array($qt["types"], ["who","what","which","where","when",
+            "whose","how", "how many", "how long", "how big"])) {
+            return $qt["ques_words"];
+        }
+        return false;
+    }
+    /**
+     * Takes tagged question string starts with Who
+     * and returns question triplet from the question string
+     *
+     * @param string $tagged_question part-of-speech tagged question
+     * @param int $index current index in statement
+     * @param string $question_word is the question word need to be replaced
+     * @return array parsed triplet
+     */
+    public static function parseQuestion($tagged_question, $index, $question_word)
+    {
+        $generated_questions = [];
+        $tree = ["cur_node" => 0];
+        $parse_tree = self::parseWholePhrase($tagged_question,
+                $tree);
+        $triplets = self::extractTripletsParseTree($parse_tree);
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($triplet_types as $type) {
+            if (!empty($triplets['subject'][$type])
+                && !empty($triplets['predicate'][$type])
+                && !empty($triplets['object'][$type])) {
+                $sub=trim($triplets['subject'][$type]);
+                $sub=preg_replace("/^.*".$question_word.".*$/",self::$question_token, $sub);
+                $pre=trim($triplets['predicate'][$type]);
+                $pre=preg_replace("/^.*".$question_word.".*$/",self::$question_token, $pre);
+                $obj=trim($triplets['object'][$type]);
+                $obj=preg_replace("/^.*".$question_word.".*$/",self::$question_token, $obj);
+                $generated_questions[$type][] = $obj . " " . $pre . " " . $sub;
+                $generated_questions[$type][] = $sub . " " . $pre . " " . $obj;
+            }
+        }
+        return $generated_questions;
+    }
+    /**
+     * Helper function for isQuestion
+     * @param $term_array segmented Chinese terms
+     * @param $type_list currect trace of self::$question_words
+     * return ["ques_words"=>ques_words,"types"=>types]
+     */
+    public static function questionType($term_array, $type_list)
+    {
+        if (!isset($type_list["any"])) {
+            return ["ques_words"=>"","types"=>""];
+        }
+        $types="";
+        $ques_words="";
+        for($i=0; $i < count($term_array); $i++ ) {
+            foreach($type_list["any"] as $key => $value) {
+                if (preg_match('/^('.$key.')$/u',$term_array[$i])) {
+                    if (is_array($value)) {
+                        if(isset($value["after"])) {
+                            $found_after=false;
+                            if (array_key_exists($i+1,$term_array)) {
+                                foreach($value["after"] as $key2 => $value2) {
+                                    if (preg_match('/^('.$key2.')$/u',
+                                        $term_array[$i+1])) {
+                                        $ques_words=$term_array[$i].
+                                            " ".$term_array[$i+1];
+                                        $types=$value2;
+                                        $found_after=true;
+                                        break;
+                                    }
+                                }
+                            }
+                            if (!$found_after && isset($type_list["other"])
+                                    && $value["other"]) {
+                                $ques_words=$term_array[$i];
+                                $types=$value["other"];
+                            }
+                        } elseif(isset($value["any"])) {
+                            $t=self::questionType($term_array,$value);
+                            $ques_words[]=$term_array[$i];
+                            $types=$t["types"];
+                        }
+                    } elseif ($value) {
+                        $ques_words=$term_array[$i];
+                        $types=$value;
+                    }
+                }
+            }
+        }
+        if ($types == "" && isset($type_list["other"])){
+            if (is_array($type_list["other"])) {
+                $t=self::questionType($term_array, $type_list["other"]);
+                $ques_words=$t["ques_words"];
+                $types=$t["types"];
+            } elseif ( $type_list["other"]) {
+                $types=$type_list["other"];
+            }
+        }
+        return ["ques_words"=>$ques_words,"types"=>$types];
+    }
 }
-- 
2.21.0.windows.1

