_binary From 927e1c583df00ea08eb92a8cde739695c06c58e0 Mon Sep 17 00:00:00 2001
From: SalilShenoy <salilshenoy@gmail.com>
Date: Wed, 15 Nov 2017 12:33:12 -0800
Subject: [PATCH] Tweaks to a previous patch on QA Module

---
 src/configs/Config.php                |   2 +
 src/configs/Createdb.php              |  19 ++---
 src/library/VersionFunctions.php      |  46 +++++++++++
 src/locale/hi/resources/Tokenizer.php | 139 +++++++++++++++++++---------------
 4 files changed, 132 insertions(+), 74 deletions(-)

diff --git a/src/configs/Config.php b/src/configs/Config.php
index 0a010845..d42a5e7b 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -976,3 +976,5 @@ nsconddefine('AD_LOGO','resources/adv-logo.png');
 nsconddefine('SENTENCE_COMPRESSION_ENABLED', false);
 /** Define cipher to be used in AES */
 nsconddefine('AES_256_CBC', 'aes-256-cbc');
+/** The number of rows to be used in bulk insert from Lexicon */
+nsconddefine('NUM_LEX_BULK_INSERTS',100000);
diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php
index efd9bf31..245836e6 100755
--- a/src/configs/Createdb.php
+++ b/src/configs/Createdb.php
@@ -250,12 +250,12 @@ foreach ($locales as $locale) {
 }
 //Insert the lexicon for all locales in the database
 foreach ($locales as $locale) {
-    $folderName = $locale[0];
+    $folder_name = $locale[0];
     if (strstr($locale[0], "-")) {
-        $localeName = explode("-", $locale[0]);
-        $folderName = $localeName[0] . "_" . $localeName[1];
+        $locale_name = explode("-", $locale[0]);
+        $folder_name = $locale_name[0] . "_" . $locale_name[1];
     }
-    $lexicon_file = C\LOCALE_DIR . "/" . $folderName .
+    $lexicon_file = C\LOCALE_DIR . "/" . $folder_name .
         "/resources/lexicon.txt.gz";
     if (file_exists($lexicon_file)) {
         $lines = gzfile($lexicon_file);
@@ -264,21 +264,16 @@ foreach ($locales as $locale) {
         foreach ($lines as $line) {
             $line = trim($line, " ");
             $line = explode(" ", $line);
-            if (empty($line[0]) || empty($line[1]))
-                continue;
             $insert_values .= '(\'' . trim($line[0]) . '\',\'' . $locale[0] .
                 '\',\'' . trim($line[1]) . '\'),';
             $count++;
-            if ($count >= 10000) {
+            if ($count >= C\NUM_LEX_BULK_INSERTS) {
                 $insert_values = rtrim($insert_values, ',');
-                $query = 'INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
-                    {$insert_values}';
+                $query = "INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                    {$insert_values}";
                 $db->exec($query);
                 $insert_values = "";
                 $count = 0;
-                if ($db->affectedRows() == 0) {
-                    continue;
-                }
             }
         }
         if ($count > 0) {
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index 6131a945..67dc2a03 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1566,4 +1566,50 @@ function upgradeDatabaseVersion55(&$db)
 {
     $db->execute("CREATE TABLE LEXICON(WORD VARCHAR, LOCALE VARCHAR,
                 POS VARCHAR, PRIMARY KEY(WORD, LOCALE))");
+    // Retrieve the locales added to the Locale table
+    $sql = "SELECT LOCALE_TAG from LOCALE";
+    $result = $db->execute($sql);
+    if ($result) {
+        $locales = $db->fetchArray($result);
+    }
+    /*
+     * Go through the locales, check of there is a lexicon,
+     * if present then add it to the Lexicon database.
+     * as (term, part_of_speech, locale)
+     */
+    foreach ($locales as $locale) {
+        $folder_name = $locale;
+        if (strstr($locale, "-")) {
+            $locale_name = explode("-", $locale);
+            $folder_name = $locale_name . "_" . $locale_name;
+        }
+        $lexicon_file = C\LOCALE_DIR . "/" . $folder_name .
+            "/resources/lexicon.txt.gz";
+        if (file_exists($lexicon_file)) {
+            $lines = gzfile($lexicon_file);
+            $insert_values = "";
+            $count = 0;
+            foreach ($lines as $line) {
+                $line = trim($line, " ");
+                $line = explode(" ", $line);
+                $insert_values .= '(' . trim($line[0]) . ',' . $locale[0] .
+                    ',' . trim($line[1]) . '),';
+                $count++;
+                if ($count >= C\NUM_LEX_BULK_INSERTS) {
+                    $insert_values = rtrim($insert_values, ',');
+                    $query = "INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                        {$insert_values}";
+                    $db->exec($query);
+                    $insert_values = "";
+                    $count = 0;
+                }
+            }
+            if ($count > 0) {
+                $insert_values = rtrim($insert_values, ',');
+                $query = "INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                    {$insert_values}";
+                $db->exec($query);
+            }
+        }
+    }
 }
diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php
index f0df8568..37b69ccc 100755
--- a/src/locale/hi/resources/Tokenizer.php
+++ b/src/locale/hi/resources/Tokenizer.php
@@ -47,12 +47,13 @@ class Tokenizer
      * List of verb-like parts of speech that might appear in lexicon
      * @array
      */
-    public static $verb_phrases = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ"];
+    public static $verb_phrases = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ",
+        "RB"];
     /**
      * List of noun-like parts of speech that might appear in lexicon
      * @array
      */
-    public static $noun_phrases = ["NN", "NNS", "NNP", "NNPS", "PRP"];
+    public static $noun_phrases = ["NN", "NNS", "NNP", "NNPS", "DT"];
     /**
      * List of adjective-like parts of speech that might appear in lexicon
      * @array
@@ -62,8 +63,13 @@ class Tokenizer
      * List of postpositional-like parts of speech that might appear in lexicon
      * @array
      */
-    public static $postpositional_phrases = ["inj", "PREP", "proNN", "CONJ", 
-        "INT", "particle", "case", "PSP"];
+    public static $postpositional_phrases = ["IN", "inj", "PREP", "proNN",
+        "CONJ", "INT", "particle", "case", "PSP", "direct_DT", "PRP"];
+    /*
+     * List of questions in Hindi
+     */
+    public static $questions = ["क्या", "कब", "कहा", "क्यों", "कौन", "जिसे",
+            "जिसका", "कहाँ", "कहां"];
     /**
      * Any unique identifier corresponding to the component of a triplet which
      * can be answered using a question answer list
@@ -195,7 +201,7 @@ class Tokenizer
      */
     public static function tagTokenizePartofSpeech($text)
     {
-        $tokens = preg_split("/[\s]+/", $text);
+        $tokens = preg_split("/\s+/u", $text);
         $result = [];
         $tag_list = [];
         $i = 0;
@@ -246,40 +252,51 @@ class Tokenizer
             $current['tag'] = trim($current['tag']);
             if ($current['tag'] == "UNKNOWN" || $previous['tag'] == "UNKNOWN") 
             {
-                //RULE 1: If the previous word tagged is a Adjective / Pronoun 
-                // Postposition then the current word is likely to be a noun
+                /**
+                 * RULE 1: If the previous word tagged is a Adjective Pronoun
+                 * Postposition then the current word is likely to be a noun
+                 */
                 if ($previous['tag'] == 'JJ'     ||
                     $previous['tag'] == 'PRO_NN' ||
                     $previous['tag'] == 'POST_POS') {
                     $current['tag'] = 'NN';
                     $result[$i] = $current;
                 }
-                //RULE 2: If the current word is a verb then the previous word is 
-                //likely to be a noun
+                /**
+                 * RULE 2: If the current word is a verb then the previous
+                 * word is likely to be a noun
+                 */
                 if (in_array($current['tag'], $verbs)) {
                     $previous['tag'] = 'NN';
-                    $result[$i] = $previous;
+                    $result[$i-1] = $previous;
                 }
-                //PRONOUN IDENTIFICATION
-                //RULE 3: If the previous word is unknown and cuurent word is a
-                //noun then the previous word is most likely to be a pronoun
+                /**
+                 * PRONOUN IDENTIFICATION
+                 * RULE 3: If the previous word is unknown and cuurent word
+                 * is a noun then the previous word is most likely to be a
+                 * pronoun
+                 */
                 if ($previous['tag'] == 'UNKNOWN' &&
                     $current['tag'] == 'NN') {
                     $previous['tag'] = 'PRP';
                     $result[$i-1] = $previous;
                 }
-                //VERB IDENTIFICATION
-                //RULE 4: If the current word is tagged as Auxilary verb  and 
-                //previous word is tagged as Unknown then most likely that the 
-                //previous word is a verb
+                /**
+                 * VERB IDENTIFICATION
+                 * RULE 4: If the current word is tagged as Auxilary verb and
+                 * previous word is tagged as Unknown then most likely that
+                 * the previous word is a verb
+                 */
                 if ($current['tag'] == 'VAUX' &&
                     $previous['tag'] == 'UNKNOWN') {
                     $previous['tag'] = 'VB';
                     $result[$i-1] = $previous;
-                } 
-                //ADJECTIVE IDENTIFIATION
-                //RULE 5: if the currennt word ends with 'तम' or 'इक' or 'िक' 
-                //or 'तर' then the word is an adjective
+                }
+                /**
+                 * ADJECTIVE IDENTIFIATION
+                 * RULE 5: if the current word ends with 'तम' or 'इक' or 'िक'
+                 * or 'तर' then the word is an adjective
+                 */
                 if(mb_substr($current['token'], -2, 2) == "इक" || 
                        mb_substr($current['token'], -2, 2) == "िक" ||
                        mb_substr($current['token'], -2, 2) == "तर"  ||
@@ -630,7 +647,10 @@ class Tokenizer
         $question_answer_list = [];
         $triplet_types = ['CONCISE', 'RAW'];
         foreach ($word_and_phrase_list as $word_and_phrase => $position_list) {
-            $tagged_phrase = self::tagTokenizePartOfSpeech($word_and_phrase);
+            $sentence = $word_and_phrase;
+            $sentence = preg_replace("/\s+/u", " ", $word_and_phrase);
+            $sentence = trim($sentence);
+            $tagged_phrase = self::tagTokenizePartOfSpeech($sentence);
             $parse_tree = self::generatePhraseParseTree($tagged_phrase);
             $triplets = self::extractTripletsParseTree($parse_tree);
             $extracted_triplets = self::rearrangeTripletsByType($triplets);
@@ -802,9 +822,9 @@ class Tokenizer
     }
     /**
      * Takes a triplets array with subject, predicate, object fields with
-     * CONCISE, RAW subfields and produces a triplits with $type subfield (where
-     * $type is one of CONCISE and RAW) and with subject, predicate, object,
-     * and QUESTION_ANSWER_LIST subfields
+     * CONCISE, RAW subfields and produces triplets with $type subfield
+     * where $type is one of CONCISE and RAW and with subject, predicate,
+     * object and QUESTION_ANSWER_LIST subfields
      *
      * @param array $sub_pred_obj_triplets  in format described above
      * @param string $type either CONCISE or RAW
@@ -819,17 +839,20 @@ class Tokenizer
             $question_answer_triplets = [];
             $question_marker = self::$question_marker;
             $sentence = [$sub_pred_obj_triplets['subject'][$type],
-                    $sub_pred_obj_triplets['predicate'][$type],
-                    $sub_pred_obj_triplets['object'][$type]];
+                    $sub_pred_obj_triplets['object'][$type],
+                    $sub_pred_obj_triplets['predicate'][$type]];
             $question_triplets = [];
             for ($j = 0; $j < 2; $j++) {
                 for ($i = 0; $i < 3; $i++) {
-                    $q_sentence = $sentence;
-                    $q_sentence[$i] = $question_marker;
-                    $q_sentence_string = implode(" ", $q_sentence);
-                    $question_triplets[] = $q_sentence_string;
-                    $question_answer_triplets[$q_sentence_string] =
-                        preg_replace('/\s+/u', ' ',$sentence[$i]);
+                    $question = $sentence;
+                    $question[$i] = $question_marker;
+                    $question_string = implode(" ", $question);
+                    $question_string = trim($question_string);
+                    $question_string = preg_replace("/\s+/u", " ",
+                        $question_string);
+                    $question_triplets[] = $question_string;
+                    $question_answer_triplets[$question_string] =
+                        preg_replace("/\s+/u", " ", $sentence[$i]);
                 }
             }
             $triplets['QUESTION_LIST'] = $question_triplets;
@@ -845,33 +868,26 @@ class Tokenizer
      * @param int $index current index in statement
      * @return array parsed triplet
      */
-    public static function parseWhoQuestion($tagged_question, $index)
+    public static function parseQuestion($tagged_question, $index)
     {
-        $start_pos = 0;
-        if ($index == 0)
-            $start_pos = $index + 1;
         $generated_questions = [];
-        $question_marker = self::getQuestionMarker();
+        $question_marker = trim(self::getQuestionMarker());
         $triplets = [];
-        $tree_np = self::extractNounPhrase($tagged_question, ["cur_node" => 
-            $start_pos]);
+        $tree_np = self::extractNounPhrase($tagged_question, 
+            ["cur_node" => 0]);
         $triplets['subject'] = self::extractSubjectParseTree($tree_np);
-        $tree = ["cur_node" => $index];
-        $tree['NP'] = $tagged_question[$index]['token'];
-        $tree_vp = self::extractVerbPhrase($tagged_question, $tree);
+        $tree_vp = self::extractVerbPhrase($tagged_question,
+            ['cur_node' => $index+1]);
         $triplets['predicate'] = self::extractPredicateParseTree($tree_vp);
         $triplet_types = ['CONCISE', 'RAW'];
         foreach ($triplet_types as $type) {
             if (!empty($triplets['subject'][$type])
                 && !empty($triplets['predicate'][$type])) {
-                $generated_questions[$type][] = 
-                    trim($triplets['subject'][$type]) .
-                    " " . trim($triplets['predicate'][$type]) . " " .
-                    $question_marker;
-                $generated_questions[$type][] = 
-                    trim($triplets['subject'][$type]) .
+                $question = trim (trim($triplets['subject'][$type]) .
                     " " . $question_marker .
-                    " " . trim($triplets['predicate'][$type]);
+                    " " . trim($triplets['predicate'][$type]));
+                $question = preg_replace("/\s+/u", " ", $question);
+                $generated_questions[$type][] = $question;
             }
         }
         return $generated_questions;
@@ -885,10 +901,11 @@ class Tokenizer
      */
     public function isQuestion($phrase)
     {
-        $who_question = "कौन";
         $phrase = trim($phrase);
-        if (mb_strpos($phrase, $who_question) !== false) {
-            return true;
+        for ($i = 0; $i < count(self::$questions); $i++) {
+            if (mb_strpos($phrase, trim(self::$questions[$i])) !== false) {
+                return true;
+            }
         }
         return false;
     }
@@ -902,27 +919,25 @@ class Tokenizer
         return self::$question_marker;
     }
     /**
-     * Takes WH questions and returns the triplet from the question
+     * Takes questions and returns the triplet from the question
      *
      * @param string $question question to parse
      * @return array question triplet
      */
     public static function questionParser($question)
     {
-        /*
-         * Array of 'wh' questions: What, When, Where, Why, Who, Which, Whom, 
-         * Whose
-         */
-        $wh_questions = array( "क्या", "कब", "कहा", "क्यों", "कौन", "जिसे",
-            "जिसका", "कहाँ");
+        $question = trim($question);
+        $question = preg_replace("/\s+/u", " ", $question);
         $tagged_question = self::tagTokenizePartOfSpeech($question);
         $index = -1;
         foreach ($tagged_question as $i => $term_pos) {
-            if (in_array($term_pos['token'], $wh_questions)) {
+            if (in_array($term_pos['token'], self::$questions)) {
                 $index = $i;
+                $term_pos['tag'] = "p_wh";
+                $tagged_question[$i] = $term_pos;
                 break;
             }
         }
-        return self::parseWhoQuestion($tagged_question, $index);
+        return self::parseQuestion($tagged_question, $index);
     }
 }
-- 
2.15.0.windows.1

