_binary From 917551294ac51725094b7280a76f1e42420a69c5 Mon Sep 17 00:00:00 2001
From: Salil Shenoy <salilshenoy@gmail.com>
Date: Tue, 14 Mar 2017 12:34:42 -0700
Subject: [PATCH] Patch: Triplet Extractor for Hindi

---
 src/locale/hi/resources/Tokenizer.php | 397 +++++++++++++++++++++++++++++++++-
 1 file changed, 389 insertions(+), 8 deletions(-)

diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php
index 771f1e3..2bf00d5 100755
--- a/src/locale/hi/resources/Tokenizer.php
+++ b/src/locale/hi/resources/Tokenizer.php
@@ -28,6 +28,10 @@
  */
 namespace seekquarry\yioop\locale\hi\resources;
 
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library\PhraseParser;
+use seekquarry\yioop\library as L;
+
 /**
  * Hindi specific tokenization code. In particular, it has a stemmer,
  * The stemmer is my stab at porting Ljiljana Dolamic (University of Neuchatel,
@@ -42,6 +46,12 @@ namespace seekquarry\yioop\locale\hi\resources;
  */
 class Tokenizer
 {
+    public static $verb_phrases = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ"];
+
+    public static $noun_phrases = ["NN", "NNS", "NNP", "NNPS", "PRP"];
+
+    public static $adjective_phrases = ["JJ", "JJR", "JJS"];
+
     /**
      * Words we don't want to be stemmed
      * @var array
@@ -165,25 +175,28 @@ class Tokenizer
      * @param string $text input phrase which is to be tagged
      * @return string $result which is an array of token => tag
      */
-    public static function tagTokenizePartsofSpeech($text)
+    public static function tagTokenizePartofSpeech($text)
     {
         static $dictionary = [];
         if (empty($dictionary)) {
             $serial_lex_file = C\LOCALE_DIR .
                 "/hi/resources/serial_lexicon.txt";
-            if (file_exists($serial_lex_file)) {
+            $lex_file = C\LOCALE_DIR . "/hi/resources/lexicon.txt.gz";
+            if (file_exists($serial_lex_file) &&
+                filemtime($serial_lex_file) > filemtime($lex_file)) {
                 $dictionary = unserialize(file_get_contents($serial_lex_file));
             } else {
-                $lines = gzfile(C\LOCALE_DIR . "/hi/resources/lexicon.txt.gz");
+                $lines = gzfile($lex_file);
                 foreach ($lines as $line) {
-                    $tags = explode(',', trim($line, ' '));
-                    $dictionary[array_shift($tags)] = $tags;
+                    $tags = explode(',', $line);
+                    $dictionary[strtolower(array_shift($tags))] = $tags;
                 }
                 $dictionary_string = serialize($dictionary);
                 file_put_contents($serial_lex_file, $dictionary_string);
-                chmod(0777, $serial_lex_file);
+                chmod($serial_lex_file, 0777);
             }
         }
+
         preg_match_all("/[\w\d]+/", $text, $matches);
         $nouns = ['NN','NNP','NNS'];
         $verbs = ['VBZ','VBD','VBN'];
@@ -197,6 +210,8 @@ class Tokenizer
              * UNKNOWN
              */
             $current = ['token' => $token, 'tag' => 'UNKNOWN'];
+            $previous = ['token' => $token, 'tag' => 'UNKNOWN'];
+
             if (!empty($dictionary[$token])) {
                 $tag_list = $dictionary[$token];
                 $current['tag'] = $tag_list[0];
@@ -266,6 +281,25 @@ class Tokenizer
             * RULE: If the word ends with <tar>, <tam>, <thik> then we tag it
             * as a Adjective
             */
+            $ends_with = substr($token, -2);
+            switch ($ends_with) {
+                case 'तर':
+                    // Convert noun to past particle if ends with 'ed'
+                    if ($current['tag'][0] == 'N') {
+                        $current['tag'] = 'JJ';
+                    }
+                break;
+                case 'तम':
+                    // Anything that ends 'ly' is an adverb
+                    $current['tag'] = 'JJ';
+                break;
+                case 'त्हिक':
+                    // Common noun to adjective if it ends with al
+                    if (in_array($current['tag'], $nouns)) {
+                        $current['tag'] = 'JJ';
+                    }
+                break;
+            }
             /**
              * VERB IDENTIFICATION
              * RULE: If the current word is tagged as Auxilary verb and
@@ -296,7 +330,6 @@ class Tokenizer
         $with_tokens = true)
     {
         $tagged_phrase = [];
-        $with_tokens = $with_tokens;
         $simplified_parts_of_speech = [
           "NN" => "NN", "NNS" => "NN", "NNP" => "NN", "NNPS" => "NN",
           "PRP" => "NN", 'PRP$' => "NN", "WP" => "NN",
@@ -312,6 +345,354 @@ class Tokenizer
             $token = ($with_tokens) ? $t['token'] . "~" : "";
             $tagged_phrase .= $token . $tag .  " ";
         }
+
         return $tagged_phrase;
     }
-}
+
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "NN" a subarray with a token node for the noun string that was
+     *      parsed
+     */
+    public static function extractNoun($tagged_phrase, $tree)
+    {
+        //Combining multiple noun into one
+        $noun_string = "";
+        $cur_node = $tree['cur_node'];
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            (in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$noun_phrases))) {
+            $noun_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($noun_string)) {
+            $tree["NN"] = $noun_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a sequence of
+     * postpositional phrases if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     */
+    public static function extractPostposition($tagged_phrase, $tree)
+    {
+        $postpos_string = "";
+        $cur_node = $tree['cur_node'];
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$noun_phrases)) {
+            $postpos_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($adjective_string)) {
+            $tree["IN"] = $adjective_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "JJ" with value an adjective subtree
+     *      "POST" with value a post position subtree
+     */
+    public static function extractNounPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_nn =
+            self::extractNoun($tagged_phrase, ['cur_node' => $cur_node]);
+        $tree_jj = self::extractAdjective($tagged_phrase,
+            ['cur_node' => $tree_nn['cur_node']]);
+        $tree_post = self::extractPostposition($tagged_phrase,
+            ['cur_node' => $tree_jj['cur_node']]);
+        unset($tree_nn['cur_node']);
+        $tree['NN'] = $tree_nn;
+        unset($tree_jj['cur_node']);
+        $tree['JJ'] = $tree_jj;
+        unset($tree_post['cur_node']);
+        $tree['POST'] = $tree_post;
+        return $tree;
+    }
+
+     /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VB" a subarray with a token node for the verb string that was
+     *      parsed
+     */
+    public static function extractVerb($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        // skip stuff before verb (intensifiers and adverbs)
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            !in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$verb_phrases)) {
+            $cur_node++;
+        }
+        $verb_string = "";
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$verb_phrases)) {
+            $verb_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($verb_string)) {
+            $tree["VB"] = $verb_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+
+     /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VP" a subarray with possible fields
+     *      "VB" with value a verb subtree
+     */
+    public static function extractVerbPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_vb = self::extractVerb($tagged_phrase, ['cur_node' => $cur_node]);
+        if ($tree_vb['cur_node'] == $cur_node) {
+            return $tree;
+        }
+        return $tree_vb;
+    }
+
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for an adjective if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "JJ" a subarray with a token node for the adjective that was
+     *      parsed
+     */
+    public static function extractAdjective($tagged_phrase, $tree)
+    {
+        $adjective_string = "";
+        $cur_node = $tree['cur_node'];
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$adjective_phrases)) {
+            $adjective_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($adjective_string)) {
+            $tree["JJ"] = $adjective_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+
+    public static function extractSubject($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_np = 
+            self::extractNounPhrase($tagged_phrase, ['cur_node' => $cur_node]);
+        $tree_jj = self::extractAdjective($tagged_phrase, $tree_np);
+        unset($tree_np['cur_node']);
+        $tree['NP'] = $tree_np;
+        unset($tree_jj['cur_node']);
+        $tree['JJ'] = $tree_jj;
+        return $tree;
+    }
+
+    public static function extractComplement($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_np = 
+            self::extractNounPhrase($tagged_phrase, ['cur_node' => $cur_node]);
+        $tree_vp = 
+            self::extractVerbPhrase($tagged_phrase, $tree_np['cur_node']);
+        $tree_ap = 
+            self::extractAdjective($tagged_phrase,
+                ['cur_node' => $tree_vp['cur_node']]);
+        unset($tree_np['cur_node']);
+        $tree['NP'] = $tree_np;
+        unset($tree_vp['cur_node']);
+        $tree['VP'] = $tree_vp;
+        unset($tree_ap['cur_node']);
+        $tree['AP'] = $tree_ap;
+        return $tree;
+    }
+
+     /**
+     * Given a part-of-speeech tagged phrase array generates a parse tree
+     * for the phrase using a recursive descent parser.
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @return array used to represent a tree. The array has up to three fields
+     *      $tree["cur_node"] index of how far we parsed our$tagged_phrase
+     *      $tree["NP"] contains a subtree for a noun phrase
+     *      $tree["VP"] contains a subtree for a verb phrase
+     */
+    public static function generatePhraseParseTree($tagged_phrase)
+    {
+        $tree = [];
+        $tree_subject =
+            self::extractSubject($tagged_phrase, ['cur_node' => 0]);
+        $tree = ['cur_node' => $tree_subject['cur_node']];
+        $tree_complement = self::extractComplement($tagged_phrase, $tree);
+        $tree['cur_node'] = $tree_complement['cur_node'];
+        unset($tree_subject['cur_node']);
+        unset($tree_complement['cur_node']);
+        $tree['NP'] = $tree_subject['NP'];
+        $tree['VP'] = $tree_complement['VP'];
+        return $tree;
+    }
+
+     /**
+     * Scans a word list for phrases. For phrases found generate
+     * a list of question and answer pairs at two levels of granularity:
+     * CONCISE (using all terms in orginal phrase) and RAW (removing
+     * (adjectives, etc).
+     *
+     * @param array $word_and_phrase_list of statements
+     * @return array with two fields: QUESTION_LIST consisting of
+     *      (SUBJECT, COMPLEMENT) where one of the components has been
+     *      replaced with a question marker.
+     */
+    public static function extractTripletsPhrases($word_and_phrase_list)
+    {
+        $triplets_list = [];
+        $question_list = [];
+        $question_answer_list = [];
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($word_and_phrase_list as $word_and_phrase => $position_list) {
+            $tagged_phrase = self::tagTokenizePartOfSpeech($word_and_phrase);
+            $parse_tree = self::generatePhraseParseTree($tagged_phrase);
+            $triplets = self::extractTripletsParseTree($parse_tree);
+        }
+        $out_triplets['QUESTION_LIST'] = $question_list;
+        $out_triplets['QUESTION_ANSWER_LIST'] = $question_answer_list;
+        return $out_triplets;
+    }
+
+    public static function extractDeepestSpeechPartPhrase($tree, $pos)
+    {
+        $extract = "";
+        if (!empty($tree[$pos])) {
+            $extract = self::extractDeepestSpeechPartPhrase($tree[$pos], $pos);
+        }
+        if (!$extract && !empty($tree[$pos]) && !empty($tree[$pos][$pos])) {
+            $extract = $tree[$pos][$pos];
+        }
+        return $extract;
+    }
+
+    public static function extractSubjectParseTree($tree)
+    {
+        $subject = [];
+        if (!empty($tree['NP'])) {
+            $subject['CONCISE'] = self::extractDeepestSpeechPartPhrase(
+                $tree['NP'], "NN");
+            $raw_subject = "";
+            $it = new \RecursiveIteratorIterator(
+                new \RecursiveArrayIterator($tree['NP']));
+            foreach ($it as $v) {
+                $raw_subject .= $v . " ";
+            }
+            $subject['RAW'] = $raw_subject;
+        } else {
+            $subject['CONCISE'] = "";
+            $subject['RAW'] = "";
+        }
+        return $subject;
+    }
+
+    public static function extractComplementParseTree($tree)
+    {
+        $predicate = [];
+        if (!empty($tree['VP'])) {
+            $tree_vp = $tree['VP'];
+            $predicate['CONCISE'] = self::extractDeepestSpeechPartPhrase(
+                $tree_vp, "VB");
+            $raw_predicate = "";
+            if (!empty($tree_vp['VB'])) {
+                $tree_vb = $tree_vp['VB'];
+                $it = new \RecursiveIteratorIterator(
+                    new \RecursiveArrayIterator($tree_vb));
+                foreach ($it as $v) {
+                    $raw_predicate .= $v . " ";
+                }
+                $predicate['RAW'] = $raw_predicate;
+            }
+        } else {
+            $predicate['CONCISE'] = "";
+            $predicate['RAW'] = "";
+        }
+        return $predicate;
+    }
+
+    /**
+     * Takes a parse tree of a phrase and computes subject, predicate, and
+     * object arrays. Each of these array consists of two components CONCISE and
+     * RAW, CONCISE corresponding to something more similar to the words in the
+     * original phrase and RAW to the case where extraneous words have been
+     * removed
+     *
+     * @param are $tree a parse tree for a sentence
+     * @return array triplet array
+     */
+    public static function extractTripletsParseTree($parse_tree)
+    {
+        $triplets = [];
+        $triplets['subject'] = self::extractSubjectParseTree($parse_tree);
+        $triplets['complement'] =
+            self::extractComplementParseTree($parse_tree);
+        return $triplets;
+    }
+}
\ No newline at end of file
-- 
2.10.0.windows.1

