_binary From e110d699d218ee4e2d3f5c78e9cd4b1b1ccf69cb Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Wed, 13 Sep 2017 14:40:11 -0700
Subject: [PATCH 1/4] 1. Triplet Extractor 2. Lexicon stored in database

---
 src/configs/Config.php                         |   4 +-
 src/configs/Createdb.php                       |  45 +-
 src/library/VersionFunctions.php               |  16 +-
 src/library/summarizers/CentroidSummarizer.php |   4 +-
 src/locale/en_US/resources/Tokenizer.php       |   8 +-
 src/locale/hi/resources/Tokenizer.php          | 713 +++++++++++++++++++++----
 src/locale/hi/resources/lexicon.txt.gz         | Bin 200022 -> 1791332 bytes
 src/models/ProfileModel.php                    |   8 +-
 8 files changed, 691 insertions(+), 107 deletions(-)

diff --git a/src/configs/Config.php b/src/configs/Config.php
index 5fda5686..a962cc64 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -112,7 +112,7 @@ function nsconddefine($constant, $value)
  * Version number for upgrade function
  * @var int
  */
-nsdefine('YIOOP_VERSION', 53);
+nsdefine('YIOOP_VERSION', 54);
 /**
  * Minimum Version fo Yioop for which keyword ad script
  * still works with this version
@@ -959,4 +959,4 @@ nsconddefine('AD_LOGO','resources/adv-logo.png');
 /** sentence compression enabled or not*/
 nsconddefine('SENTENCE_COMPRESSION_ENABLED', false);
 /** Define cipher to be used in AES */
-nsconddefine('AES_256_CBC', 'aes-256-cbc');
+nsconddefine('AES_256_CBC', 'aes-256-cbc');
\ No newline at end of file
diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php
index bc91d867..38fb8302 100755
--- a/src/configs/Createdb.php
+++ b/src/configs/Createdb.php
@@ -237,6 +237,49 @@ foreach ($locales as $locale) {
     $locale_index[$locale[0]] = $i;
     $i++;
 }
+//Insert the lexicon for all locales in the database
+foreach ($locales as $locale) {
+    $folderName = $locale[0];
+    if (strstr($locale[0], "-")) {
+        $localeName = explode("-", $locale[0]);
+        $folderName = $localeName[0] . "_" . $localeName[1];
+    }
+    $lexicon_file = C\LOCALE_DIR . "/" . $folderName .
+        "/resources/lexicon.txt.gz";
+    if (file_exists($lexicon_file))
+    {
+        $lines = gzfile($lexicon_file);
+        $insert_values = "";
+        $count = 0;
+        foreach ($lines as $line) {
+            $line = trim($line, " ");
+            $line = explode(" ", $line);
+            if (empty($line[0]) || empty($line[1]))
+                continue;
+            $insert_values .= '(\'' . trim($line[0]) . '\',\'' . $locale[0] .
+                '\',\'' . trim($line[1]) . '\'),';
+            $count++;
+            if ($count >= 10000) {
+                $insert_values = rtrim($insert_values, ',');
+                $query = 'INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                    {$insert_values}';
+                $db->exec($query);
+                $insert_values = "";
+                $count = 0;
+                if ($db->affectedRows() == 0) {
+                    continue;
+                }
+            }
+        }
+        if ($count > 0)
+        {
+            $insert_values = rtrim($insert_values, ',');
+            $query = "INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                {$insert_values}";
+            $db->exec($query);
+        }
+    }
+}
 $activities = [
     "manageAccount" => ['db_activity_manage_account',
         [
@@ -583,4 +626,4 @@ if (in_array(DBMS, ['sqlite','sqlite3'])){
 }
 $db->execute("INSERT INTO AVERAGE_RATING VALUES (0)");
 $db->execute("INSERT INTO LIVE_TABLE VALUES (1,1,1)");
-echo "Create DB succeeded\n";
+echo "Create DB succeeded\n";
\ No newline at end of file
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index 4198c824..8632e53e 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1439,12 +1439,11 @@ function upgradeDatabaseVersion51(&$db){
          INTENT_ID INTEGER, PRIMARY KEY(TERM, INTENT_ID))");
 }
 /**
-+ * Upgrades a Version 51 version of the Yioop! database to a Version 52 version
-+ * @param object $db datasource to use to upgrade.
-+ */
+ * Upgrades a Version 51 version of the Yioop! database to a Version 52 version
+ * @param object $db datasource to use to upgrade.
+ */
 function upgradeDatabaseVersion52(&$db)
 {
-
     $db->execute("CREATE TABLE ITEM_IMPRESSION_STAT(ITEM_ID INTEGER,
         ITEM_TYPE INTEGER, UPDATE_PERIOD INTEGER, SUM INTEGER DEFAULT -1,
         FUZZY_NUM_VIEWS INTEGER DEFAULT -1");
@@ -1511,3 +1510,12 @@ function upgradeDatabaseVersion53(&$db)
     $db->execute("CREATE TABLE USER_THREAD_GROUP
                 (USER_ID INT, ITEM_ID INT, GROUP_ID INT)");
 }
+/**
+ * Upgrades a Version 53 version of the Yioop! database to a Version 54 version
+ * @param object $db datasource to use to upgrade.
+ */
+function upgradeDatabaseVersion54(&$db)
+{
+    $db->execute("CREATE TABLE LEXICON(WORD VARCHAR, LOCALE VARCHAR,
+                POS VARCHAR, PRIMARY KEY(WORD, LOCALE))";
+}
\ No newline at end of file
diff --git a/src/library/summarizers/CentroidSummarizer.php b/src/library/summarizers/CentroidSummarizer.php
index 1c807d1f..2e131d50 100644
--- a/src/library/summarizers/CentroidSummarizer.php
+++ b/src/library/summarizers/CentroidSummarizer.php
@@ -105,7 +105,7 @@ class CentroidSummarizer extends Summarizer
         } else {
             $sentences = $out_sentences;
         }
-print_r($sentences);
+
         /*  Splitting into terms */
         $terms = [];
         foreach ($sentences as $sentence) {
@@ -357,4 +357,4 @@ print_r($sentences);
         $page = preg_replace("/\n\n\n+/", "\n\n", $page);
         return $page;
     }
-}
+}
\ No newline at end of file
diff --git a/src/locale/en_US/resources/Tokenizer.php b/src/locale/en_US/resources/Tokenizer.php
index a2d611e6..315d7229 100755
--- a/src/locale/en_US/resources/Tokenizer.php
+++ b/src/locale/en_US/resources/Tokenizer.php
@@ -1153,15 +1153,15 @@ class Tokenizer
                 ["WRB", "WP"])) {
                 $token = strtoupper(trim($tagged_question[0]['token']));
                 if ($token == "WHO") {
-                    $generated_questions = self::parseWhoQuestion(
+                    $generated_question_array = self::parseWhoQuestion(
                         $tagged_question, 1);
                 } else if (in_array($token, ["WHERE", "WHEN", "WHAT"])) {
-                    $generated_questions = self::parseWHPlusQuestion(
+                    $generated_question_array = self::parseWHPlusQuestion(
                         $tagged_question, 1);
                 }
             }
         }
-        return $generated_questions;
+        return $generated_question_array;
     }
     /**
      * Takes a triplets array with subject, predicate, object fields with
@@ -1687,4 +1687,4 @@ class Tokenizer
             "(,\s?when[^,]*,)|(,\s?where[^,]*,)/i", "", $result);
         return $result;
     }
-}
+}
\ No newline at end of file
diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php
index 771f1e3c..6133a7a2 100755
--- a/src/locale/hi/resources/Tokenizer.php
+++ b/src/locale/hi/resources/Tokenizer.php
@@ -165,7 +165,7 @@ class Tokenizer
      * @param string $text input phrase which is to be tagged
      * @return string $result which is an array of token => tag
      */
-    public static function tagTokenizePartsofSpeech($text)
+    public static function tagTokenizePartofSpeech($text)
     {
         static $dictionary = [];
         if (empty($dictionary)) {
@@ -185,104 +185,97 @@ class Tokenizer
             }
         }
         preg_match_all("/[\w\d]+/", $text, $matches);
-        $nouns = ['NN','NNP','NNS'];
-        $verbs = ['VBZ','VBD','VBN'];
         $tokens = explode(' ', $text);
         $result = [];
         $tag_list = [];
         $i = 0;
-        foreach ($tokens as $token) {
-            /**
-             * Tag the tokens as found in the Lexicon, else use default tag as
-             * UNKNOWN
-             */
+
+        foreach ($tokens as $token)
+        {
+            //Tag the tokens as found in the Lexicon
             $current = ['token' => $token, 'tag' => 'UNKNOWN'];
-            if (!empty($dictionary[$token])) {
+            if (is_numeric($token)) {
+                $current['tag'] = "Q";
+            } elseif (!empty($dictionary[$token])) {
                 $tag_list = $dictionary[$token];
                 $current['tag'] = $tag_list[0];
             }
-            /**
-             * NOUN IDENTIFICATION
-             * RULE 1: If the previous word tagged is a Adjective / Pronoun /
-             * Postposition then the current word is likely to be a noun
-             */
-            if ($previous['tag'] == 'JJ' ||
-                $previous['tag'] == 'PRP' ||
-                $previous['tag'] == 'POST_POS') {
-                $current['tag'] = 'NN';
-            }
-            /**
-             * NOUN INDENTIFICATION
-             * RULE 2: If the current word is a verb then the previous word is
-             * likely to be a noun
-             */
-            if (in_array($current['tag'], $verbs)) {
-                $previous['tag'] = 'NN';
-                $result[$i-1] = $previous;
-            }
-            /**
-             * NOUN INDENTIFICATION
-             * RULE 3: If the current tag is a noun then next / previous is
-             * likely to be a noun
-             */
-            /**
-             * DEMONSTRATIVE IDENTIFICATION
-             * RULE 1: If the current and previous words are tagged as pronouns
-             * then the previous word is likley to be a demonstrative
-             */
-            if ($current['tag'] == 'PRP' && $previous['tag'] == 'PRP') {
-                $previous['tag'] = 'DEM';
-                $result[$i-1] = $previous;
-            }
-            /**
-             * DEMONSTRATIVE IDENTIFICATION
-             * RULE 2: If current word is a noun and previous word is a pronoun
-             * then the current word is liklely to be demonstrative
-             */
-            if ($current['tag'] == 'NN' && $previous['tag'] == 'PRP') {
-                $current['tag'] = 'DEM';
-            }
-            /**
-             * PRONOUN IDENTIFICATION
-             * RULE 1: If the previous word is unknown and cuurent word is a
-             * noun then the previous word is most likely to be a pronoun
-             */
-            if ($previous['tag'] == 'UNKNOWN' && $current['tag'] == 'NN') {
-                $previous['tag'] = 'PRP';
-                $result[$i - 1] = $previous;
-            }
-            /**
-             * NAME Identification
-             * RULE: If we get two words which are untagged the most probably
-             * they form a name and will be tagged as noun
-             */
-            if ($previous['tag'] == 'UNKNOWN' && $current['tag'] == 'UNKNOWN') {
-                $current['tag'] = 'NN';
-                $previous['tag'] = 'NN';
-                $result[$i - 1] = $previous;
-            }
-           /**
-            * ADJECTIVE IDENTIFCATION
-            * RULE: If the word ends with <tar>, <tam>, <thik> then we tag it
-            * as a Adjective
-            */
-            /**
-             * VERB IDENTIFICATION
-             * RULE: If the current word is tagged as Auxilary verb and
-             * previous word is tagged as Unknown then most likely that the
-             * previous word is a verb
-             */
-            if ($current['tag'] == 'VAUX' && $previous['tag'] == 'UNKNOWN') {
-                $previous['tag'] = 'VB';
-                $result[$i - 1] = $previous;
-            }
             $result[$i] = $current;
             $i++;
+        }
+        return tagUnknownWords($result);
+    }
+    /**
+     *    This method tags the remaining words from the text.
+    */
+    public static function tagUnknownWords($partiallyTaggedText)
+    {
+        $result = $partiallyTaggedText;
+        $nouns = ['NN','NNP','NNS'];
+        $verbs = ['VBZ','VBD','VBN'];
+        $length = count($result);
+        $previous = $result[0];
+        for ($i = 1; $i < $length; $i++)
+        {
+            $current = $result[$i];
+            $current['token'] = trim($current['token']);
+            $current['tag'] = trim($current['tag']);
+            if ($current['tag'] == "UNKNOWN" || $previous['tag'] == "UNKNOWN")
+            {
+                //RULE 1: If the previous word tagged is a Adjective / Pronoun
+                // Postposition then the current word is likely to be a noun
+                if ($previous['tag'] == 'JJ'     ||
+                    $previous['tag'] == 'PRO_NN' ||
+                    $previous['tag'] == 'POST_POS') {
+                    $current['tag'] = 'NN';
+                    $result[$i] = $current;
+                }
+                //RULE 2: If the current word is a verb then the previous word 
+                //is likely to be a noun
+                elseif (in_array($current['tag'], $verbs)) {
+                    $previous['tag'] = 'NN';
+                    $result[$i] = $previous;
+                }
+                //PRONOUN IDENTIFICATION
+                //RULE 3: If the previous word is unknown and cuurent word is a
+                //noun then the previous word is most likely to be a pronoun
+                elseif ($previous['tag'] == 'UNKNOWN' &&
+                    $current['tag'] == 'NN') {
+                    $previous['tag'] = 'PRP';
+                    $result[$i] = $previous;
+                }
+                //VERB IDENTIFICATION
+                //RULE 4: If the current word is tagged as Auxilary verb  and
+                //previous word is tagged as Unknown then most likely that the
+                //previous word is a verb
+                elseif ($current['tag'] == 'VAUX' &&
+                    $previous['tag'] == 'UNKNOWN') {
+                    $previous['tag'] = 'VB';
+                    $result[$i] = $previous;
+                }
+                //ADJECTIVE IDENTIFIATION
+                //RULE 5: if the currennt word ends with 'तम' or 'इक' or 'िक'
+                //or 'तर' then the word is an adjective
+                elseif (mb_substr($current['token'], -2, 2) == "इक" ||
+                       mb_substr($current['token'], -2, 2) == "िक" ||
+                       mb_substr($current['token'], -2, 2) == "तर"  ||
+                       mb_substr($current['token'], -2, 2) == "तम") {
+                    $current['tag'] = 'AJ';
+                    $result[$i] = $current;
+                } else {
+                    if ($current['tag'] == "UNKNOWN") {
+                        $current['tag'] = 'NN';
+                        $result[$i] = $current;
+                    } else {
+                        $previous['tag'] = 'NN';
+                        $result[$i-1] = $previous;
+                    }
+                }
+            }
             $previous = $current;
-            $previous_token = $token;
         }
         return $result;
-    }
+     }
     /**
      * This method is used to simplify the different tags of speech to a
      * common form
@@ -298,13 +291,14 @@ class Tokenizer
         $tagged_phrase = [];
         $with_tokens = $with_tokens;
         $simplified_parts_of_speech = [
-          "NN" => "NN", "NNS" => "NN", "NNP" => "NN", "NNPS" => "NN",
-          "PRP" => "NN", 'PRP$' => "NN", "WP" => "NN",
+          "NNS" => "NN", "NNP" => "NN", "NNPS" => "NN","WP" => "NN",
           "VB" => "VB", "VBD" => "VB", "VBN" => "VB", "VBP" => "VB",
           "VBZ" => "VB",
           "JJ" => "AJ", "JJR" => "AJ", "JJS" => "AJ",
-          "RB" => "AV", "RBR" => "AV", "RBS" => "AV", "WRB" => "AV"
-        ];
+          "RB" => "AV", "RBR" => "AV", "RBS" => "AV", "WRB" => "AV",
+          "case" => "CONJ",
+          "direct_DT" => "DT",
+       ];
         foreach ($tagged_tokens as $t) {
             $tag = trim($t['tag']);
             $tag = (isset($simplified_parts_of_speech[$tag])) ?
@@ -314,4 +308,541 @@ class Tokenizer
         }
         return $tagged_phrase;
     }
-}
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "NN" a subarray with a token node for the noun string that was
+     *      parsed
+     */
+    public static function extractNoun($tagged_phrase, $tree)
+    {
+        //Combining multiple noun into one
+        $noun_string = "";
+        $cur_node = $tree['cur_node'];
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            (in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$noun_phrases))) {
+            $noun_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($noun_string)) {
+            $tree["NN"] = $noun_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a sequence of
+     * postpositional phrases if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     */
+    public static function extractPostposition($tagged_phrase, $tree,
+        $index = 1)
+    {
+        $cur_node = $tree['cur_node'];
+        $prep_noun_string = "";
+        if (isset($tagged_phrase[$cur_node]['tag']) &&
+            trim($tagged_phrase[$cur_node]['tag'] == "inj")) {
+            while (isset($tagged_phrase[$cur_node]['tag']) &&
+                in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$noun_phrases)) {
+                $prep_noun_string .= " " . $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            if ($prep_noun_string) {
+                $tree["NP_$index"] = $prep_noun_string;
+            }
+            $tree_next = self::extractPostposition($tagged_phrase,
+                    ["cur_node" => $cur_node], $index + 1);
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a noun phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "JJ" with value an adjective subtree
+     *      "POST" with value a post position subtree
+     */
+    public static function extractNounPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_jj = self::extractAdjective($tagged_phrase,
+            ['cur_node' => $tree['cur_node']]);
+         $tree_nn =self::extractNoun($tagged_phrase,
+            ['cur_node' => $tree_jj['cur_node']]);
+        $tree_post = self::extractPostposition($tagged_phrase,
+            ['cur_node' => $tree_nn['cur_node']]);
+        if ($tree_nn['cur_node'] == $cur_node) {
+            $tree['NP'] = "";
+        } else {
+            $cur_node = $tree_post['cur_node'];
+            unset($tree_jj['cur_node']);
+            $tree_new_sub['JJ'] = $tree_jj;
+            unset($tree_nn['cur_node']);
+            $tree_new_sub['NN'] = $tree_nn;
+            unset($tree_post['cur_node']);
+            $tree_new_sub['POST'] = $tree_post;
+            $tree_new['cur_node'] = $cur_node;
+            $tree_new['NP'] = $tree_new_sub;
+            return $tree_new;
+        }
+        return $tree;
+    }
+     /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VB" a subarray with a token node for the verb string that was
+     *      parsed
+     */
+    public static function extractVerb($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $verb_string = "";
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$verb_phrases)) {
+            $verb_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($verb_string)) {
+            $tree["VB"] = $verb_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+     /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for a verb phrase if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["curnode" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "VP" a subarray with possible fields
+     *      "VB" with value a verb subtree
+     */
+    public static function extractVerbPhrase($tagged_phrase, $tree)
+    {
+        $cur_node = $tree['cur_node'];
+        $tree_vb = self::extractVerb($tagged_phrase, ['cur_node' => $cur_node]);
+        $cur_node = $tree_vb['cur_node'];
+        $tree_np = self::extractNounPhrase($tagged_phrase,
+            ['cur_node' => $cur_node]);
+        $tree_new = [];
+        $tree_new_sub = [];
+        if ($tree_np['cur_node'] !=  $cur_node) {
+            $cur_node = $tree_np['cur_node'];
+            unset($tree_vb['cur_node']);
+            unset($tree_np['cur_node']);
+            $tree_new_sub['VB'] = $tree_vb;
+            $tree_new_sub['NP'] = $tree_np['NP'];
+            $tree_new['cur_node'] = $cur_node;
+            $tree_new['VP'] = $tree_new_sub;
+            return $tree_new;
+        }
+        unset($tree_vb['cur_node']);
+        $tree_new_sub['VB'] = $tree_vb;
+        $tree_new['cur_node'] = $cur_node;
+        $tree_new['VP'] = $tree_new_sub;
+        return $tree_new;
+    }
+    /**
+     * Takes a part-of-speech tagged phrase and pre-tree with a
+     * parse-from position and builds a parse tree for an adjective if possible
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @param array $tree that consists of ["cur_node" =>
+     *      current parse position in $tagged_phrase]
+     * @return array has fields
+     *      "cur_node" index of how far we parsed $tagged_phrase
+     *      "JJ" a subarray with a token node for the adjective that was
+     *      parsed
+     */
+    public static function extractAdjective($tagged_phrase, $tree)
+    {
+        $adjective_string = "";
+        $cur_node = $tree['cur_node'];
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$adjective_phrases)) {
+            $adjective_string .= " " . $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($adjective_string)) {
+            $tree["JJ"] = $adjective_string;
+        }
+        $tree['cur_node'] = $cur_node;
+        return $tree;
+    }
+     /**
+     * Given a part-of-speeech tagged phrase array generates a parse tree
+     * for the phrase using a recursive descent parser.
+     *
+     * @param array $tagged_phrase
+     *      an array of pairs of the form ("token" => token_for_term,
+     *     "tag"=> part_of_speech_tag_for_term)
+     * @return array used to represent a tree. The array has up to three fields
+     *      $tree["cur_node"] index of how far we parsed our$tagged_phrase
+     *      $tree["NP"] contains a subtree for a noun phrase
+     *      $tree["VP"] contains a subtree for a verb phrase
+     */
+    public static function generatePhraseParseTree($tagged_phrase)
+    {
+        $tree = [];
+        $tree_np = self::extractNounPhrase($tagged_phrase,['cur_node' => 0]);
+        $tree = ["cur_node" => $tree_np['cur_node']];
+        $tree_vp = self::extractVerbPhrase($tagged_phrase, $tree);
+        $tree['cur_node'] = $tree_vp['cur_node'];
+        if ($tree == $tree_vp) {
+            unset($tree_np['cur_node']);
+            unset($tree_vp['cur_node']);
+            $tree['NP'] = $tree_np['NP'];
+            return $tree;
+        }
+        unset($tree_np['cur_node']);
+        unset($tree_vp['cur_node']);
+        $tree['NP'] = $tree_np['NP'];
+        $tree['VP'] = $tree_vp['VP'];
+        return $tree;
+    }
+     /**
+     * Scans a word list for phrases. For phrases found generate
+     * a list of question and answer pairs at two levels of granularity:
+     * CONCISE (using all terms in orginal phrase) and RAW (removing
+     * (adjectives, etc).
+     *
+     * @param array $word_and_phrase_list of statements
+     * @return array with two fields: QUESTION_LIST consisting of
+     *      (SUBJECT, COMPLEMENT) where one of the components has been
+     *      replaced with a question marker.
+     */
+    public static function extractTripletsPhrases($word_and_phrase_list)
+    {
+        $triplets_list = [];
+        $question_list = [];
+        $question_answer_list = [];
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($word_and_phrase_list as $word_and_phrase => $position_list) {
+            $tagged_phrase = self::tagTokenizePartOfSpeech($word_and_phrase);
+            $parse_tree = self::generatePhraseParseTree($tagged_phrase);
+            $triplets = self::extractTripletsParseTree($parse_tree);
+            $extracted_triplets = self::rearrangeTripletsByType($triplets);
+            foreach ($triplet_types as $type) {
+                if (!empty($extracted_triplets[$type])) {
+                    $triplets = $extracted_triplets[$type];
+                    $questions = $triplets['QUESTION_LIST'];
+                    foreach ($questions as $question) {
+                        $question_list[$question] = $position_list;
+                    }
+                    $question_answer_list = array_merge($question_answer_list,
+                        $triplets['QUESTION_ANSWER_LIST']);
+                }
+            }
+        }
+        $out_triplets['QUESTION_LIST'] = $question_list;
+        $out_triplets['QUESTION_ANSWER_LIST'] = $question_answer_list;
+        return $out_triplets;
+    }
+    public static function extractDeepestSpeechPartPhrase($tree, $pos)
+    {
+        $extract = "";
+        if (!empty($tree[$pos])) {
+            $extract = self::extractDeepestSpeechPartPhrase($tree[$pos], $pos);
+        }
+        if (!$extract && !empty($tree[$pos]) && !empty($tree[$pos][$pos])) {
+            $extract = $tree[$pos][$pos];
+        }
+        return $extract;
+    }
+    public static function extractSubjectParseTree($tree)
+    {
+        $subject = [];
+        if (!empty($tree['NP'])) {
+            $subject['CONCISE'] = self::extractDeepestSpeechPartPhrase(
+                $tree['NP'], "NN");
+            $raw_subject = "";
+            $it = new \RecursiveIteratorIterator(
+                new \RecursiveArrayIterator($tree['NP']));
+            foreach ($it as $v) {
+                $raw_subject .= $v . " ";
+            }
+            $subject['RAW'] = $raw_subject;
+        } else {
+            $subject['CONCISE'] = "";
+            $subject['RAW'] = "";
+        }
+        return $subject;
+    }
+    public static function extractComplementParseTree($tree)
+    {
+        $predicate = [];
+        if (!empty($tree['VP'])) {
+            $tree_vp = $tree['VP'];
+            $predicate['CONCISE'] = self::extractDeepestSpeechPartPhrase(
+                $tree_vp, "VB");
+            $raw_predicate = "";
+            if (!empty($tree_vp['VB'])) {
+                $tree_vb = $tree_vp['VB'];
+                $it = new \RecursiveIteratorIterator(
+                    new \RecursiveArrayIterator($tree_vb));
+                foreach ($it as $v) {
+                    $raw_predicate .= $v . " ";
+                }
+                $predicate['RAW'] = $raw_predicate;
+            }
+        } else {
+            $predicate['CONCISE'] = "";
+            $predicate['RAW'] = "";
+        }
+        return $predicate;
+    }
+    public static function extractObjectParseTree($tree)
+    {
+        $object = [];
+        if (!empty($tree['VP'])) {
+            $tree_vp = $tree['VP'];
+            if (!empty($tree_vp['NP'])) {
+                $nb = $tree_vp['NP'];
+                $object['CONCISE'] = self::extractDeepestSpeechPartPhrase($nb,
+                    "NN");
+                $raw_object = "";
+                $it = new \RecursiveIteratorIterator(
+                    new \RecursiveArrayIterator($nb));
+                foreach ($it as $v) {
+                    $raw_object .= $v . " ";
+                }
+                $object['RAW'] = $raw_object;
+            } else {
+                $object['CONCISE'] = "";
+                $object['RAW'] = "";
+            }
+        } else {
+            $object['CONCISE'] = "";
+            $object['RAW'] = "";
+        }
+        return $object;
+    }
+    /**
+     * Takes a parse tree of a phrase and computes subject, predicate, and
+     * object arrays. Each of these array consists of two components CONCISE and
+     * RAW, CONCISE corresponding to something more similar to the words in the
+     * original phrase and RAW to the case where extraneous words have been
+     * removed
+     *
+     * @param are $tree a parse tree for a sentence
+     * @return array triplet array
+     */
+    public static function extractTripletsParseTree($parse_tree)
+    {
+        $triplets = [];
+        $triplets['subject'] = self::extractSubjectParseTree($parse_tree);
+        $triplets['object'] = self::extractObjectParseTree($parse_tree);
+        $triplets['complement'] = self::extractComplementParseTree($parse_tree);
+        return $triplets;
+    }
+    public static function rearrangeTripletsByType($sub_pred_obj_triplets)
+    {
+        $processed_triplet = [];
+        $processed_triplets['CONCISE'] =
+            self::extractTripletByType($sub_pred_obj_triplets, "CONCISE");
+        $processed_triplets['RAW'] =
+            self::extractTripletByType($sub_pred_obj_triplets, "RAW");
+        return $processed_triplets;
+    }
+    public static function extractTripletByType($sub_pred_obj_triplets, $type)
+    {
+        $triplets = [];
+        $isSubjectEmpty = empty($sub_pred_obj_triplets['subject'][$type]);
+        $isComplementEmpty = empty($sub_pred_obj_triplets['complement'][$type]);
+        $isObjectEmpty = empty($sub_pred_obj_triplets['object'][$type]);
+        if (!$isSubjectEmpty || !$isComplementEmpty || !$isObjectEmpty) {
+            $question_answer_triplets = [];
+            $question_marker = self::$question_marker;
+
+            if ($isSubjectEmpty === true)
+                $sub_pred_obj_triplets['subject'][$type] = " ";
+
+            if ($isComplementEmpty === true)
+                $sub_pred_obj_triplets['complement'][$type] = " ";
+
+            if ($isObjectEmpty === true)
+                $sub_pred_obj_triplets['object'][$type] = " ";
+
+            $sentence = [trim($sub_pred_obj_triplets['subject'][$type]),
+                         trim($sub_pred_obj_triplets['complement'][$type]),
+                         trim($sub_pred_obj_triplets['object'][$type])];
+            $question_triplets = [];
+            for ($j = 0; $j < 2; $j++) {
+                for ($i = 0; $i < 3; $i++) {
+                    $q_sentence = $sentence;
+                    $q_sentence[$i] = $question_marker;
+                    $q_sentence_string = implode(" ", $q_sentence);
+                    $question_triplets[] = $q_sentence_string;
+                    $question_answer_triplets[$q_sentence_string] =
+                        preg_replace('/\s+/u', ' ',$sentence[$i]);
+                }
+            }
+            $triplets['QUESTION_LIST'] = $question_triplets;
+            $triplets['QUESTION_ANSWER_LIST'] = $question_answer_triplets;
+        }
+        return $triplets;
+    }
+        /**
+     * Takes tagged question string starts with Who
+     * and returns question triplet from the question string
+     *
+     * @param string $tagged_question part-of-speech tagged question
+     * @param int $index current index in statement
+     * @return array parsed triplet
+     */
+    public static function parseWhoQuestion($tagged_question, $index)
+    {
+        $generated_questions = [];
+        $question_marker = self::getQuestionMarker();
+        $tree = ["cur_node" => $index];
+        $tree['NP'] = "कौन";
+        $triplets = [];
+        $tree_vp = self::extractVerbPhrase($tagged_question, $tree);
+        $triplets['complement'] = self::extractComplementParseTree(
+            $tree_vp);
+        $triplets['object'] = self::extractObjectParseTree(
+            $tree_vp);
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($triplet_types as $type) {
+            if (!empty($triplets['object'][$type])
+                && !empty($triplets['complement'][$type])) {
+                $generated_questions[$type][] =
+                    trim($triplets['object'][$type]) .
+                    " " . trim($triplets['complement'][$type]) . " " .
+                    $question_marker;
+                $generated_questions[$type][] = $question_marker .
+                    " " . trim($triplets['complement'][$type]) .
+                    " " . trim($triplets['object'][$type]);
+            }
+        }
+        return $generated_questions;
+    }
+    /**
+     * Takes tagged question string starts with Wh+ except Who
+     * and returns question triplet from the question string
+     * Unlike the WHO case, here we assume there is an auxliary verb
+     * followed by a noun phrase then the rest of the verb phrase. For example,
+     * Where is soccer played?
+     *
+     * @param string $tagged_question part-of-speech tagged question
+     * @param $index current index in statement
+     * @return array parsed triplet suitable for query look-up
+     */
+    public static function parseWHPlusQuestion($tagged_question, $index)
+    {
+        $generated_questions = [];
+        $aux_verb = "";
+        $question_marker = self::getQuestionMarker();
+        while (isset($tagged_question[$index]) &&
+            in_array(trim($tagged_question[$index]['tag']),
+            self::$verb_phrases)) {
+            $token = trim($tagged_question[$index]['token']);
+            $aux_verb .= " " . $token;
+            $index++;
+        }
+        $tree = ["cur_node" => $index];
+        $tree['NP'] = $tagged_question;
+        $triplets = [];
+        $tree_np = self::extractNounPhrase($tagged_question, $tree);
+        $triplets['subject'] = self::extractSubjectParseTree($tree_np);
+        $tree_vp = self::extractVerbPhrase($tagged_question, $tree_np);
+        $triplets['predicate'] = self::extractPredicateParseTree($tree_vp);
+        if (!empty($aux_verb)) {
+            if (!isset($triplets['complement']['RAW'])) {
+                $triplets['complement']['RAW'] = "";
+            }
+            $triplets['predicate']['RAW'] = trim($aux_verb) .
+                " " . $triplets['complement']['RAW'];
+        }
+        $triplet_types = ['CONCISE', 'RAW'];
+        foreach ($triplet_types as $type) {
+            if (!empty($triplets['subject'][$type])&&
+                !empty($triplets['complement'][$type])) {
+                $generated_questions[$type][] =
+                    trim($triplets['subject'][$type]) .
+                    " " . trim($triplets['complement'][$type]) .
+                    " " . $question_marker;
+                $generated_questions[$type][] = $question_marker.
+                    " " . trim($triplets['complement'][$type]) .
+                    " " . trim($triplets['subject'][$type]);
+            }
+        }
+        return $generated_questions;
+    }
+    /**
+    * Takes a phrase query entered by user and return true if it is question
+    * and false if not
+    *
+    * @param $phrase any statement
+    * @return bool returns true if statement is question
+    */
+    public function isQuestion($phrase)
+    {
+        $regex_starts_with_que =
+            "/^(कौन|क्या|कहा|कब|किसका|कैसे)(.*)$/";
+        if (preg_match($regex_starts_with_que, trim($phrase))) {
+            return true;
+        }
+        return false;
+    }
+    public static function getQuestionMarker()
+    {
+        return self::$question_marker;
+    }
+    public static function questionParser($question)
+    {
+        $tagged_question = self::tagTokenizePartOfSpeech($question);
+        $generated_questions = [];
+        if (isset($tagged_question[0])) {
+            $token = $tagged_question[0]['token'];
+            if ($token == "कौन") {
+                $generated_questions = self::parseWhoQuestion(
+                   $tagged_question, 1);
+            }
+        }
+        return $generated_questions;
+    }
+}
\ No newline at end of file
diff --git a/src/locale/hi/resources/lexicon.txt.gz b/src/locale/hi/resources/lexicon.txt.gz
index 5d7c3ba79dca9df5a33541c6a74cb8192e6cc93d..4f2e2010103459b68bb15a9725df198eda332dd7 100755
GIT binary patch
diff --git a/src/models/ProfileModel.php b/src/models/ProfileModel.php
index 276dfb2a..19682395 100755
--- a/src/models/ProfileModel.php
+++ b/src/models/ProfileModel.php
@@ -242,8 +242,8 @@ class ProfileModel extends Model
             "INTENT" => "CREATE TABLE INTENT(INTENT_ID INTEGER PRIMARY KEY
                 $auto_increment, INTENT VARCHAR(" . C\MAX_DESCRIPTION_LEN . "),
                 USER_ID INTEGER)",
-            "INTENT_EXPRESSION_ENTITY" => "CREATE TABLE INTENT_EXPRESSION_ENTITY(ID INTEGER
-                PRIMARY KEY $auto_increment,
+            "INTENT_EXPRESSION_ENTITY" => "CREATE TABLE INTENT_EXPRESSION_ENTITY
+                (ID INTEGER PRIMARY KEY $auto_increment,
                 INTENT_ID INTEGER, EXPRESSION_ID INTEGER, ENTITY_ID INTEGER,
                 ENTITY_VALUE VARCHAR(" . C\MAX_DESCRIPTION_LEN ."),
                 PRIMARY KEY(INTENT_ID,EXPRESSION_ID,ENTITY_ID))",
@@ -415,6 +415,8 @@ class ProfileModel extends Model
             "VERSION" => "CREATE TABLE VERSION(ID INTEGER PRIMARY KEY)",
             "WORDS"=> "CREATE TABLE WORDS (WORD_ID INT, WORD VARCHAR)",
             "WORDS_TEMP"=> "CREATE TABLE WORDS_TEMP (WORD_ID INT)",
+            "LEXICON" => "CREATE TABLE LEXICON(WORD VARCHAR, LOCALE VARCHAR,
+                POS VARCHAR, PRIMARY KEY(WORD, LOCALE))"
             ];
     }
     /**
@@ -908,4 +910,4 @@ EOT;
         }
         return $match;
     }
-}
+}
\ No newline at end of file
-- 
2.14.1.windows.1


From 7f6f2a2dc91e023c62f86741733c92b86ce51abd Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Mon, 25 Sep 2017 23:54:32 -0700
Subject: [PATCH 2/4] 1. Version Function Corrected 2.Hindi Triplet Extractor
 Code Improved

---
 src/library/VersionFunctions.php      |   2 +-
 src/locale/hi/resources/Tokenizer.php | 344 +++++++++++++++-------------------
 2 files changed, 156 insertions(+), 190 deletions(-)

diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index 8632e53e..3f00415c 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1517,5 +1517,5 @@ function upgradeDatabaseVersion53(&$db)
 function upgradeDatabaseVersion54(&$db)
 {
     $db->execute("CREATE TABLE LEXICON(WORD VARCHAR, LOCALE VARCHAR,
-                POS VARCHAR, PRIMARY KEY(WORD, LOCALE))";
+                POS VARCHAR, PRIMARY KEY(WORD, LOCALE))");
 }
\ No newline at end of file
diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php
index 6133a7a2..e9cb8b1c 100755
--- a/src/locale/hi/resources/Tokenizer.php
+++ b/src/locale/hi/resources/Tokenizer.php
@@ -28,6 +28,7 @@
  */
 namespace seekquarry\yioop\locale\hi\resources;
 
+use seekquarry\yioop\models as M;
 /**
  * Hindi specific tokenization code. In particular, it has a stemmer,
  * The stemmer is my stab at porting Ljiljana Dolamic (University of Neuchatel,
@@ -42,6 +43,16 @@ namespace seekquarry\yioop\locale\hi\resources;
  */
 class Tokenizer
 {
+    public static $verb_phrases = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ"];
+
+    public static $noun_phrases = ["NN", "NNS", "NNP", "NNPS", "PRP"];
+
+    public static $adjective_phrases = ["JJ", "JJR", "JJS"];
+
+    public static $postpositional_phrases = ["inj", "PREP", "proNN", "CONJ",
+        "INT", "particle", "case", "PSP"];
+
+    public static $question_marker = "qqq";
     /**
      * Words we don't want to be stemmed
      * @var array
@@ -167,43 +178,41 @@ class Tokenizer
      */
     public static function tagTokenizePartofSpeech($text)
     {
-        static $dictionary = [];
-        if (empty($dictionary)) {
-            $serial_lex_file = C\LOCALE_DIR .
-                "/hi/resources/serial_lexicon.txt";
-            if (file_exists($serial_lex_file)) {
-                $dictionary = unserialize(file_get_contents($serial_lex_file));
-            } else {
-                $lines = gzfile(C\LOCALE_DIR . "/hi/resources/lexicon.txt.gz");
-                foreach ($lines as $line) {
-                    $tags = explode(',', trim($line, ' '));
-                    $dictionary[array_shift($tags)] = $tags;
-                }
-                $dictionary_string = serialize($dictionary);
-                file_put_contents($serial_lex_file, $dictionary_string);
-                chmod(0777, $serial_lex_file);
-            }
-        }
-        preg_match_all("/[\w\d]+/", $text, $matches);
-        $tokens = explode(' ', $text);
+        $tokens = preg_split("/[\s]+/", $text);
         $result = [];
         $tag_list = [];
         $i = 0;
-
+        $model = new M\Model();
         foreach ($tokens as $token)
         {
             //Tag the tokens as found in the Lexicon
+            $token = trim($token);
             $current = ['token' => $token, 'tag' => 'UNKNOWN'];
+            $word = $current['token'];
+            $sql = "SELECT * FROM LEXICON WHERE WORD = '{$word}'
+                AND LOCALE = 'hi'";
+            $queryResult = @$model->db->execute($sql);
+            if ($queryResult !== false) {
+                $row = $model->db->fetchArray($queryResult);
+                $current['tag'] = $row['POS'];
+            }
+
             if (is_numeric($token)) {
                 $current['tag'] = "Q";
-            } elseif (!empty($dictionary[$token])) {
-                $tag_list = $dictionary[$token];
-                $current['tag'] = $tag_list[0];
+            } else if (strcmp($token,"है") == 0 ||
+                        strcmp($token, "हैं") == 0) {
+                $current['tag'] = "VB";
             }
+
+            if (!isset($current['tag'])) {
+                $current['tag'] = "UNKNOWN";
+            }
+
             $result[$i] = $current;
             $i++;
         }
-        return tagUnknownWords($result);
+
+        return self::tagUnknownWords($result);
     }
     /**
      *    This method tags the remaining words from the text.
@@ -211,7 +220,6 @@ class Tokenizer
     public static function tagUnknownWords($partiallyTaggedText)
     {
         $result = $partiallyTaggedText;
-        $nouns = ['NN','NNP','NNS'];
         $verbs = ['VBZ','VBD','VBN'];
         $length = count($result);
         $previous = $result[0];
@@ -230,46 +238,46 @@ class Tokenizer
                     $current['tag'] = 'NN';
                     $result[$i] = $current;
                 }
-                //RULE 2: If the current word is a verb then the previous word 
+                //RULE 2: If the current word is a verb then the previous word
                 //is likely to be a noun
-                elseif (in_array($current['tag'], $verbs)) {
+                if (in_array($current['tag'], $verbs)) {
                     $previous['tag'] = 'NN';
                     $result[$i] = $previous;
                 }
                 //PRONOUN IDENTIFICATION
                 //RULE 3: If the previous word is unknown and cuurent word is a
                 //noun then the previous word is most likely to be a pronoun
-                elseif ($previous['tag'] == 'UNKNOWN' &&
+                if ($previous['tag'] == 'UNKNOWN' &&
                     $current['tag'] == 'NN') {
                     $previous['tag'] = 'PRP';
-                    $result[$i] = $previous;
+                    $result[$i-1] = $previous;
                 }
                 //VERB IDENTIFICATION
                 //RULE 4: If the current word is tagged as Auxilary verb  and
                 //previous word is tagged as Unknown then most likely that the
                 //previous word is a verb
-                elseif ($current['tag'] == 'VAUX' &&
+                if ($current['tag'] == 'VAUX' &&
                     $previous['tag'] == 'UNKNOWN') {
                     $previous['tag'] = 'VB';
-                    $result[$i] = $previous;
+                    $result[$i-1] = $previous;
                 }
                 //ADJECTIVE IDENTIFIATION
                 //RULE 5: if the currennt word ends with 'तम' or 'इक' or 'िक'
                 //or 'तर' then the word is an adjective
-                elseif (mb_substr($current['token'], -2, 2) == "इक" ||
+                if (mb_substr($current['token'], -2, 2) == "इक" ||
                        mb_substr($current['token'], -2, 2) == "िक" ||
                        mb_substr($current['token'], -2, 2) == "तर"  ||
                        mb_substr($current['token'], -2, 2) == "तम") {
                     $current['tag'] = 'AJ';
                     $result[$i] = $current;
+                }
+
+                if ($current['tag'] == "UNKNOWN") {
+                    $current['tag'] = 'NN';
+                    $result[$i] = $current;
                 } else {
-                    if ($current['tag'] == "UNKNOWN") {
-                        $current['tag'] = 'NN';
-                        $result[$i] = $current;
-                    } else {
-                        $previous['tag'] = 'NN';
-                        $result[$i-1] = $previous;
-                    }
+                    $previous['tag'] = 'NN';
+                    $result[$i-1] = $previous;
                 }
             }
             $previous = $current;
@@ -288,7 +296,7 @@ class Tokenizer
     public static function taggedPartOfSpeechTokensToString($tagged_tokens,
         $with_tokens = true)
     {
-        $tagged_phrase = [];
+        $tagged_phrase = "";
         $with_tokens = $with_tokens;
         $simplified_parts_of_speech = [
           "NNS" => "NN", "NNP" => "NN", "NNPS" => "NN","WP" => "NN",
@@ -296,7 +304,8 @@ class Tokenizer
           "VBZ" => "VB",
           "JJ" => "AJ", "JJR" => "AJ", "JJS" => "AJ",
           "RB" => "AV", "RBR" => "AV", "RBS" => "AV", "WRB" => "AV",
-          "case" => "CONJ",
+          "inj" => "IN", "case" => "IN", "proNN" => "IN", "particle" => "IN",
+          "PREP" => "IN", "IN" => "IN", "PSP" => "IN",
           "direct_DT" => "DT",
        ];
         foreach ($tagged_tokens as $t) {
@@ -356,13 +365,39 @@ class Tokenizer
         $index = 1)
     {
         $cur_node = $tree['cur_node'];
-        $prep_noun_string = "";
         if (isset($tagged_phrase[$cur_node]['tag']) &&
-            trim($tagged_phrase[$cur_node]['tag'] == "inj")) {
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$postpositional_phrases)) {
+            $preposition_string = "";
+            while (isset($tagged_phrase[$cur_node]['tag']) &&
+                in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$postpositional_phrases)) {
+                $preposition_string .= " ". $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            if (!empty($preposition_string)) {
+                $tree["IN_$index"] = $preposition_string;
+            }
+            if (isset($tagged_phrase[$cur_node]['tag']) &&
+                trim($tagged_phrase[$cur_node]['tag']) == "DT") {
+                $tree['DT_$index'] = $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            $adjective_string = "";
+            while (isset($tagged_phrase[$cur_node]['tag']) &&
+                in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$adjective_phrases)) {
+                $adjective_string .= " " . $tagged_phrase[$cur_node]['token'];
+                $cur_node++;
+            }
+            if (!empty($adjective_string)) {
+                $tree["JJ_$index"] = $adjective_string;
+            }
+            $prep_noun_string = "";
             while (isset($tagged_phrase[$cur_node]['tag']) &&
                 in_array(trim($tagged_phrase[$cur_node]['tag']),
                 self::$noun_phrases)) {
-                $prep_noun_string .= " " . $tagged_phrase[$cur_node]['token'];
+                $prep_noun_string .= " ". $tagged_phrase[$cur_node]['token'];
                 $cur_node++;
             }
             if ($prep_noun_string) {
@@ -430,6 +465,12 @@ class Tokenizer
     public static function extractVerb($tagged_phrase, $tree)
     {
         $cur_node = $tree['cur_node'];
+        // skip stuff before verb (intensifiers and adverbs)
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            !in_array(trim($tagged_phrase[$cur_node]['tag']),
+            self::$verb_phrases)) {
+            $cur_node++;
+        }
         $verb_string = "";
         while (isset($tagged_phrase[$cur_node]['tag']) &&
             in_array(trim($tagged_phrase[$cur_node]['tag']),
@@ -461,7 +502,20 @@ class Tokenizer
     {
         $cur_node = $tree['cur_node'];
         $tree_vb = self::extractVerb($tagged_phrase, ['cur_node' => $cur_node]);
+        if ($tree_vb['cur_node'] == $cur_node) {
+            return $tree;
+        }
         $cur_node = $tree_vb['cur_node'];
+        $preposition_string = "";
+        while (isset($tagged_phrase[$cur_node]['tag']) &&
+            in_array(trim($tagged_phrase[$cur_node]['tag']),
+                self::$postpositional_phrases)) {
+            $preposition_string .= " ". $tagged_phrase[$cur_node]['token'];
+            $cur_node++;
+        }
+        if (!empty($preposition_string)) {
+            $tree_vb["IN"] = $preposition_string;
+        }
         $tree_np = self::extractNounPhrase($tagged_phrase,
             ['cur_node' => $cur_node]);
         $tree_new = [];
@@ -541,6 +595,10 @@ class Tokenizer
         unset($tree_vp['cur_node']);
         $tree['NP'] = $tree_np['NP'];
         $tree['VP'] = $tree_vp['VP'];
+
+        echo "<pre>";
+        print_r($tree);
+        echo "</pre>";
         return $tree;
     }
      /**
@@ -559,22 +617,19 @@ class Tokenizer
         $triplets_list = [];
         $question_list = [];
         $question_answer_list = [];
-        $triplet_types = ['CONCISE', 'RAW'];
         foreach ($word_and_phrase_list as $word_and_phrase => $position_list) {
             $tagged_phrase = self::tagTokenizePartOfSpeech($word_and_phrase);
             $parse_tree = self::generatePhraseParseTree($tagged_phrase);
             $triplets = self::extractTripletsParseTree($parse_tree);
             $extracted_triplets = self::rearrangeTripletsByType($triplets);
-            foreach ($triplet_types as $type) {
-                if (!empty($extracted_triplets[$type])) {
-                    $triplets = $extracted_triplets[$type];
-                    $questions = $triplets['QUESTION_LIST'];
-                    foreach ($questions as $question) {
-                        $question_list[$question] = $position_list;
-                    }
-                    $question_answer_list = array_merge($question_answer_list,
-                        $triplets['QUESTION_ANSWER_LIST']);
+            if (!empty($extracted_triplets)) {
+                $triplets = $extracted_triplets;
+                $questions = $triplets['QUESTION_LIST'];
+                foreach ($questions as $question) {
+                    $question_list[$question] = $position_list;
                 }
+                $question_answer_list = array_merge($question_answer_list,
+                    $triplets['QUESTION_ANSWER_LIST']);
             }
         }
         $out_triplets['QUESTION_LIST'] = $question_list;
@@ -594,30 +649,23 @@ class Tokenizer
     }
     public static function extractSubjectParseTree($tree)
     {
-        $subject = [];
+        $subject = "";
         if (!empty($tree['NP'])) {
-            $subject['CONCISE'] = self::extractDeepestSpeechPartPhrase(
-                $tree['NP'], "NN");
             $raw_subject = "";
             $it = new \RecursiveIteratorIterator(
                 new \RecursiveArrayIterator($tree['NP']));
             foreach ($it as $v) {
                 $raw_subject .= $v . " ";
             }
-            $subject['RAW'] = $raw_subject;
-        } else {
-            $subject['CONCISE'] = "";
-            $subject['RAW'] = "";
+            $subject= $raw_subject;
         }
         return $subject;
     }
-    public static function extractComplementParseTree($tree)
+    public static function extractPredicateParseTree($tree)
     {
         $predicate = [];
         if (!empty($tree['VP'])) {
             $tree_vp = $tree['VP'];
-            $predicate['CONCISE'] = self::extractDeepestSpeechPartPhrase(
-                $tree_vp, "VB");
             $raw_predicate = "";
             if (!empty($tree_vp['VB'])) {
                 $tree_vb = $tree_vp['VB'];
@@ -626,37 +674,26 @@ class Tokenizer
                 foreach ($it as $v) {
                     $raw_predicate .= $v . " ";
                 }
-                $predicate['RAW'] = $raw_predicate;
+                $predicate = $raw_predicate;
             }
-        } else {
-            $predicate['CONCISE'] = "";
-            $predicate['RAW'] = "";
         }
         return $predicate;
     }
     public static function extractObjectParseTree($tree)
     {
-        $object = [];
+        $object = "";
         if (!empty($tree['VP'])) {
             $tree_vp = $tree['VP'];
             if (!empty($tree_vp['NP'])) {
                 $nb = $tree_vp['NP'];
-                $object['CONCISE'] = self::extractDeepestSpeechPartPhrase($nb,
-                    "NN");
                 $raw_object = "";
                 $it = new \RecursiveIteratorIterator(
                     new \RecursiveArrayIterator($nb));
                 foreach ($it as $v) {
                     $raw_object .= $v . " ";
                 }
-                $object['RAW'] = $raw_object;
-            } else {
-                $object['CONCISE'] = "";
-                $object['RAW'] = "";
+                $object = $raw_object;
             }
-        } else {
-            $object['CONCISE'] = "";
-            $object['RAW'] = "";
         }
         return $object;
     }
@@ -675,57 +712,42 @@ class Tokenizer
         $triplets = [];
         $triplets['subject'] = self::extractSubjectParseTree($parse_tree);
         $triplets['object'] = self::extractObjectParseTree($parse_tree);
-        $triplets['complement'] = self::extractComplementParseTree($parse_tree);
+        $triplets['predicate'] = self::extractPredicateParseTree($parse_tree);
         return $triplets;
     }
     public static function rearrangeTripletsByType($sub_pred_obj_triplets)
     {
         $processed_triplet = [];
-        $processed_triplets['CONCISE'] =
-            self::extractTripletByType($sub_pred_obj_triplets, "CONCISE");
-        $processed_triplets['RAW'] =
-            self::extractTripletByType($sub_pred_obj_triplets, "RAW");
+        $processed_triplets =
+            self::extractTriplet($sub_pred_obj_triplets);
         return $processed_triplets;
     }
-    public static function extractTripletByType($sub_pred_obj_triplets, $type)
+    public static function extractTriplet($sub_pred_obj_triplets)
     {
         $triplets = [];
-        $isSubjectEmpty = empty($sub_pred_obj_triplets['subject'][$type]);
-        $isComplementEmpty = empty($sub_pred_obj_triplets['complement'][$type]);
-        $isObjectEmpty = empty($sub_pred_obj_triplets['object'][$type]);
-        if (!$isSubjectEmpty || !$isComplementEmpty || !$isObjectEmpty) {
+        if (!empty($sub_pred_obj_triplets['subject'])
+            && !empty($sub_pred_obj_triplets['predicate'])
+            && !empty($sub_pred_obj_triplets['object'])) {
             $question_answer_triplets = [];
             $question_marker = self::$question_marker;
-
-            if ($isSubjectEmpty === true)
-                $sub_pred_obj_triplets['subject'][$type] = " ";
-
-            if ($isComplementEmpty === true)
-                $sub_pred_obj_triplets['complement'][$type] = " ";
-
-            if ($isObjectEmpty === true)
-                $sub_pred_obj_triplets['object'][$type] = " ";
-
-            $sentence = [trim($sub_pred_obj_triplets['subject'][$type]),
-                         trim($sub_pred_obj_triplets['complement'][$type]),
-                         trim($sub_pred_obj_triplets['object'][$type])];
+            $sentence = [$sub_pred_obj_triplets['subject'],
+                    $sub_pred_obj_triplets['object'],
+                    $sub_pred_obj_triplets['predicate']];
             $question_triplets = [];
-            for ($j = 0; $j < 2; $j++) {
-                for ($i = 0; $i < 3; $i++) {
-                    $q_sentence = $sentence;
-                    $q_sentence[$i] = $question_marker;
-                    $q_sentence_string = implode(" ", $q_sentence);
-                    $question_triplets[] = $q_sentence_string;
-                    $question_answer_triplets[$q_sentence_string] =
-                        preg_replace('/\s+/u', ' ',$sentence[$i]);
-                }
+            for ($i = 0; $i < 3; $i++) {
+                $q_sentence = $sentence;
+                $q_sentence[$i] = $question_marker;
+                $q_sentence_string = implode(" ", $q_sentence);
+                $question_triplets[] = $q_sentence_string;
+                $question_answer_triplets[$q_sentence_string] =
+                    preg_replace('/\s+/u', ' ',$sentence[$i]);
             }
             $triplets['QUESTION_LIST'] = $question_triplets;
             $triplets['QUESTION_ANSWER_LIST'] = $question_answer_triplets;
         }
         return $triplets;
     }
-        /**
+    /**
      * Takes tagged question string starts with Who
      * and returns question triplet from the question string
      *
@@ -737,79 +759,22 @@ class Tokenizer
     {
         $generated_questions = [];
         $question_marker = self::getQuestionMarker();
+        $triplets = [];
+        $tree_np = self::extractNounPhrase($tagged_question, ["cur_node" => 0]);
+        $triplets['subject'] = self::extractSubjectParseTree($tree_np);
         $tree = ["cur_node" => $index];
         $tree['NP'] = "कौन";
-        $triplets = [];
         $tree_vp = self::extractVerbPhrase($tagged_question, $tree);
-        $triplets['complement'] = self::extractComplementParseTree(
-            $tree_vp);
-        $triplets['object'] = self::extractObjectParseTree(
-            $tree_vp);
-        $triplet_types = ['CONCISE', 'RAW'];
-        foreach ($triplet_types as $type) {
-            if (!empty($triplets['object'][$type])
-                && !empty($triplets['complement'][$type])) {
-                $generated_questions[$type][] =
-                    trim($triplets['object'][$type]) .
-                    " " . trim($triplets['complement'][$type]) . " " .
-                    $question_marker;
-                $generated_questions[$type][] = $question_marker .
-                    " " . trim($triplets['complement'][$type]) .
-                    " " . trim($triplets['object'][$type]);
-            }
-        }
-        return $generated_questions;
-    }
-    /**
-     * Takes tagged question string starts with Wh+ except Who
-     * and returns question triplet from the question string
-     * Unlike the WHO case, here we assume there is an auxliary verb
-     * followed by a noun phrase then the rest of the verb phrase. For example,
-     * Where is soccer played?
-     *
-     * @param string $tagged_question part-of-speech tagged question
-     * @param $index current index in statement
-     * @return array parsed triplet suitable for query look-up
-     */
-    public static function parseWHPlusQuestion($tagged_question, $index)
-    {
-        $generated_questions = [];
-        $aux_verb = "";
-        $question_marker = self::getQuestionMarker();
-        while (isset($tagged_question[$index]) &&
-            in_array(trim($tagged_question[$index]['tag']),
-            self::$verb_phrases)) {
-            $token = trim($tagged_question[$index]['token']);
-            $aux_verb .= " " . $token;
-            $index++;
-        }
-        $tree = ["cur_node" => $index];
-        $tree['NP'] = $tagged_question;
-        $triplets = [];
-        $tree_np = self::extractNounPhrase($tagged_question, $tree);
-        $triplets['subject'] = self::extractSubjectParseTree($tree_np);
-        $tree_vp = self::extractVerbPhrase($tagged_question, $tree_np);
         $triplets['predicate'] = self::extractPredicateParseTree($tree_vp);
-        if (!empty($aux_verb)) {
-            if (!isset($triplets['complement']['RAW'])) {
-                $triplets['complement']['RAW'] = "";
-            }
-            $triplets['predicate']['RAW'] = trim($aux_verb) .
-                " " . $triplets['complement']['RAW'];
-        }
-        $triplet_types = ['CONCISE', 'RAW'];
-        foreach ($triplet_types as $type) {
-            if (!empty($triplets['subject'][$type])&&
-                !empty($triplets['complement'][$type])) {
-                $generated_questions[$type][] =
-                    trim($triplets['subject'][$type]) .
-                    " " . trim($triplets['complement'][$type]) .
-                    " " . $question_marker;
-                $generated_questions[$type][] = $question_marker.
-                    " " . trim($triplets['complement'][$type]) .
-                    " " . trim($triplets['subject'][$type]);
-            }
-        }
+        $triplets['object'] = self::extractObjectParseTree($tree_vp);
+        $generated_questions[] =  trim($triplets['subject']) .
+                    " " . trim($triplets['object']) .
+                    " " . trim($triplets['predicate']) . " " .
+                    $question_marker;
+        $generated_questions[] = trim($triplets['subject']) .
+                    " " . trim($triplets['object']) .
+                    " " . $question_marker .
+                    " " . trim($triplets['predicate']);
         return $generated_questions;
     }
     /**
@@ -821,9 +786,9 @@ class Tokenizer
     */
     public function isQuestion($phrase)
     {
-        $regex_starts_with_que =
-            "/^(कौन|क्या|कहा|कब|किसका|कैसे)(.*)$/";
-        if (preg_match($regex_starts_with_que, trim($phrase))) {
+        $who_question = "कौन";
+        $phrase = trim($phrase);
+        if (mb_strpos($phrase, $who_question) !== false) {
             return true;
         }
         return false;
@@ -835,14 +800,15 @@ class Tokenizer
     public static function questionParser($question)
     {
         $tagged_question = self::tagTokenizePartOfSpeech($question);
-        $generated_questions = [];
-        if (isset($tagged_question[0])) {
-            $token = $tagged_question[0]['token'];
-            if ($token == "कौन") {
-                $generated_questions = self::parseWhoQuestion(
-                   $tagged_question, 1);
+        $generated_questions = "";
+        $index = -1;
+        foreach ($tagged_question as $key => $value) {
+            if (strcmp("कौन", $value['token']) == 0) {
+                $index = $key;
+                break;
             }
         }
+        $generated_questions = self::parseWhoQuestion($tagged_question, $index);
         return $generated_questions;
     }
 }
\ No newline at end of file
-- 
2.14.1.windows.1


From a606434ac2ed09e5fe321c0fb4037aed67861f5f Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Tue, 17 Oct 2017 09:38:34 -0700
Subject: [PATCH 3/4] Adding a conditionale special character filter for hi
 language

---
 src/library/PhraseParser.php | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/library/PhraseParser.php b/src/library/PhraseParser.php
index ca8b23b5..10392808 100755
--- a/src/library/PhraseParser.php
+++ b/src/library/PhraseParser.php
@@ -361,7 +361,11 @@ class PhraseParser
             "tagTokenizePartOfSpeech") &&
             !isset(self::$programming_language_map[$lang])) {
             $string = mb_strtolower($string);
-            $sentences = preg_split("/(\n\n+)|\.|\!|\?|。/u", $string);
+            if ($lang == "hi") {
+                $sentences = preg_split('/(|)\p{P}/u', $string);
+            } else {
+                $sentences = preg_split("/(\n\n+)|\.|\!|\?|。/u", $string);
+            }
             $pos = 0;
             $sentences_pos = [];
             $sentences = array_filter($sentences);
-- 
2.14.1.windows.1


From 916c7a6131e37809f3a7b335b619df87b9ddc6a0 Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Tue, 17 Oct 2017 11:25:41 -0700
Subject: [PATCH 4/4] Fixing the hi Tokenizer

---
 src/locale/hi/resources/Tokenizer.php | 77 ++++++++++++++++++-----------------
 1 file changed, 39 insertions(+), 38 deletions(-)

diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php
index e9cb8b1c..94fddc99 100755
--- a/src/locale/hi/resources/Tokenizer.php
+++ b/src/locale/hi/resources/Tokenizer.php
@@ -49,7 +49,7 @@ class Tokenizer
 
     public static $adjective_phrases = ["JJ", "JJR", "JJS"];
 
-    public static $postpositional_phrases = ["inj", "PREP", "proNN", "CONJ",
+    public static $postpositional_phrases = ["inj", "PREP", "proNN", "CONJ", 
         "INT", "particle", "case", "PSP"];
 
     public static $question_marker = "qqq";
@@ -183,23 +183,23 @@ class Tokenizer
         $tag_list = [];
         $i = 0;
         $model = new M\Model();
-        foreach ($tokens as $token)
+        foreach ($tokens as $token) 
         {
             //Tag the tokens as found in the Lexicon
             $token = trim($token);
             $current = ['token' => $token, 'tag' => 'UNKNOWN'];
             $word = $current['token'];
-            $sql = "SELECT * FROM LEXICON WHERE WORD = '{$word}'
+            $sql = "SELECT * FROM LEXICON WHERE WORD = '{$word}' 
                 AND LOCALE = 'hi'";
-            $queryResult = @$model->db->execute($sql);
+            $queryResult = @$model->db->execute($sql);     
             if ($queryResult !== false) {
                 $row = $model->db->fetchArray($queryResult);
                 $current['tag'] = $row['POS'];
             }
 
             if (is_numeric($token)) {
-                $current['tag'] = "Q";
-            } else if (strcmp($token,"है") == 0 ||
+                $current['tag'] = "NN";
+            } else if (strcmp($token,"है") == 0 || 
                         strcmp($token, "हैं") == 0) {
                 $current['tag'] = "VB";
             }
@@ -212,12 +212,13 @@ class Tokenizer
             $i++;
         }
 
+        print_r($result);
         return self::tagUnknownWords($result);
     }
     /**
      *    This method tags the remaining words from the text.
     */
-    public static function tagUnknownWords($partiallyTaggedText)
+    public static function tagUnknownWords($partiallyTaggedText) 
     {
         $result = $partiallyTaggedText;
         $verbs = ['VBZ','VBD','VBN'];
@@ -228,9 +229,9 @@ class Tokenizer
             $current = $result[$i];
             $current['token'] = trim($current['token']);
             $current['tag'] = trim($current['tag']);
-            if ($current['tag'] == "UNKNOWN" || $previous['tag'] == "UNKNOWN")
+            if ($current['tag'] == "UNKNOWN" || $previous['tag'] == "UNKNOWN") 
             {
-                //RULE 1: If the previous word tagged is a Adjective / Pronoun
+                //RULE 1: If the previous word tagged is a Adjective / Pronoun 
                 // Postposition then the current word is likely to be a noun
                 if ($previous['tag'] == 'JJ'     ||
                     $previous['tag'] == 'PRO_NN' ||
@@ -238,8 +239,8 @@ class Tokenizer
                     $current['tag'] = 'NN';
                     $result[$i] = $current;
                 }
-                //RULE 2: If the current word is a verb then the previous word
-                //is likely to be a noun
+                //RULE 2: If the current word is a verb then the previous word is 
+                //likely to be a noun
                 if (in_array($current['tag'], $verbs)) {
                     $previous['tag'] = 'NN';
                     $result[$i] = $previous;
@@ -253,31 +254,33 @@ class Tokenizer
                     $result[$i-1] = $previous;
                 }
                 //VERB IDENTIFICATION
-                //RULE 4: If the current word is tagged as Auxilary verb  and
-                //previous word is tagged as Unknown then most likely that the
+                //RULE 4: If the current word is tagged as Auxilary verb  and 
+                //previous word is tagged as Unknown then most likely that the 
                 //previous word is a verb
                 if ($current['tag'] == 'VAUX' &&
                     $previous['tag'] == 'UNKNOWN') {
                     $previous['tag'] = 'VB';
                     $result[$i-1] = $previous;
-                }
+                } 
                 //ADJECTIVE IDENTIFIATION
-                //RULE 5: if the currennt word ends with 'तम' or 'इक' or 'िक'
+                //RULE 5: if the currennt word ends with 'तम' or 'इक' or 'िक' 
                 //or 'तर' then the word is an adjective
-                if (mb_substr($current['token'], -2, 2) == "इक" ||
+                if(mb_substr($current['token'], -2, 2) == "इक" || 
                        mb_substr($current['token'], -2, 2) == "िक" ||
                        mb_substr($current['token'], -2, 2) == "तर"  ||
                        mb_substr($current['token'], -2, 2) == "तम") {
                     $current['tag'] = 'AJ';
                     $result[$i] = $current;
-                }
+                } 
 
                 if ($current['tag'] == "UNKNOWN") {
                     $current['tag'] = 'NN';
                     $result[$i] = $current;
-                } else {
+                } 
+
+                if ($previous['tag'] == "UNKNOWN"){
                     $previous['tag'] = 'NN';
-                    $result[$i-1] = $previous;
+                    $result[$i-1] = $previous; 
                 }
             }
             $previous = $current;
@@ -304,8 +307,8 @@ class Tokenizer
           "VBZ" => "VB",
           "JJ" => "AJ", "JJR" => "AJ", "JJS" => "AJ",
           "RB" => "AV", "RBR" => "AV", "RBS" => "AV", "WRB" => "AV",
-          "inj" => "IN", "case" => "IN", "proNN" => "IN", "particle" => "IN",
-          "PREP" => "IN", "IN" => "IN", "PSP" => "IN",
+          "inj" => "IN", "case" => "IN", "proNN" => "IN", "particle" => "IN", 
+          "PREP" => "IN", "IN" => "IN", "PSP" => "IN",   
           "direct_DT" => "DT",
        ];
         foreach ($tagged_tokens as $t) {
@@ -361,7 +364,7 @@ class Tokenizer
      * @return array has fields
      *      "cur_node" index of how far we parsed $tagged_phrase
      */
-    public static function extractPostposition($tagged_phrase, $tree,
+    public static function extractPostposition($tagged_phrase, $tree, 
         $index = 1)
     {
         $cur_node = $tree['cur_node'];
@@ -426,11 +429,11 @@ class Tokenizer
     public static function extractNounPhrase($tagged_phrase, $tree)
     {
         $cur_node = $tree['cur_node'];
-        $tree_jj = self::extractAdjective($tagged_phrase,
+        $tree_jj = self::extractAdjective($tagged_phrase, 
             ['cur_node' => $tree['cur_node']]);
-         $tree_nn =self::extractNoun($tagged_phrase,
+         $tree_nn =self::extractNoun($tagged_phrase, 
             ['cur_node' => $tree_jj['cur_node']]);
-        $tree_post = self::extractPostposition($tagged_phrase,
+        $tree_post = self::extractPostposition($tagged_phrase, 
             ['cur_node' => $tree_nn['cur_node']]);
         if ($tree_nn['cur_node'] == $cur_node) {
             $tree['NP'] = "";
@@ -508,7 +511,7 @@ class Tokenizer
         $cur_node = $tree_vb['cur_node'];
         $preposition_string = "";
         while (isset($tagged_phrase[$cur_node]['tag']) &&
-            in_array(trim($tagged_phrase[$cur_node]['tag']),
+            in_array(trim($tagged_phrase[$cur_node]['tag']), 
                 self::$postpositional_phrases)) {
             $preposition_string .= " ". $tagged_phrase[$cur_node]['token'];
             $cur_node++;
@@ -577,7 +580,7 @@ class Tokenizer
      *      $tree["cur_node"] index of how far we parsed our$tagged_phrase
      *      $tree["NP"] contains a subtree for a noun phrase
      *      $tree["VP"] contains a subtree for a verb phrase
-     */
+     */ 
     public static function generatePhraseParseTree($tagged_phrase)
     {
         $tree = [];
@@ -595,10 +598,6 @@ class Tokenizer
         unset($tree_vp['cur_node']);
         $tree['NP'] = $tree_np['NP'];
         $tree['VP'] = $tree_vp['VP'];
-
-        echo "<pre>";
-        print_r($tree);
-        echo "</pre>";
         return $tree;
     }
      /**
@@ -658,7 +657,7 @@ class Tokenizer
                 $raw_subject .= $v . " ";
             }
             $subject= $raw_subject;
-        }
+        } 
         return $subject;
     }
     public static function extractPredicateParseTree($tree)
@@ -731,10 +730,10 @@ class Tokenizer
             $question_answer_triplets = [];
             $question_marker = self::$question_marker;
             $sentence = [$sub_pred_obj_triplets['subject'],
-                    $sub_pred_obj_triplets['object'],
-                    $sub_pred_obj_triplets['predicate']];
+                    $sub_pred_obj_triplets['predicate'],
+                    $sub_pred_obj_triplets['object']];
             $question_triplets = [];
-            for ($i = 0; $i < 3; $i++) {
+            for ($i = 0; $i < 3; $i++) { 
                 $q_sentence = $sentence;
                 $q_sentence[$i] = $question_marker;
                 $q_sentence_string = implode(" ", $q_sentence);
@@ -767,11 +766,13 @@ class Tokenizer
         $tree_vp = self::extractVerbPhrase($tagged_question, $tree);
         $triplets['predicate'] = self::extractPredicateParseTree($tree_vp);
         $triplets['object'] = self::extractObjectParseTree($tree_vp);
-        $generated_questions[] =  trim($triplets['subject']) .
+
+        print_r($triplets);
+        $generated_questions[] =  trim($triplets['subject']) .  
                     " " . trim($triplets['object']) .
                     " " . trim($triplets['predicate']) . " " .
                     $question_marker;
-        $generated_questions[] = trim($triplets['subject']) .
+        $generated_questions[] = trim($triplets['subject']) .  
                     " " . trim($triplets['object']) .
                     " " . $question_marker .
                     " " . trim($triplets['predicate']);
@@ -811,4 +812,4 @@ class Tokenizer
         $generated_questions = self::parseWhoQuestion($tagged_question, $index);
         return $generated_questions;
     }
-}
\ No newline at end of file
+}
-- 
2.14.1.windows.1

