_binary From fe3aa8c1520fd92d1e351c4c1d6aee331f5101a1 Mon Sep 17 00:00:00 2001
From: Salil Shenoy <salilshenoy@gmail.com>
Date: Tue, 2 May 2017 20:28:34 -0700
Subject: [PATCH] The changes here store the lexicon in the database used by
 Yioop

---
 src/configs/Config.php                 |   4 +--
 src/configs/Createdb.php               |  49 ++++++++++++++++++++++++++++++++-
 src/library/VersionFunctions.php       |   9 ++++++
 src/locale/hi/resources/lexicon.txt.gz | Bin 200022 -> 1791332 bytes
 src/models/ProfileModel.php            |   4 ++-
 5 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/src/configs/Config.php b/src/configs/Config.php
index d9feae8..b202d67 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -112,7 +112,7 @@ function nsconddefine($constant, $value)
  * Version number for upgrade function
  * @var int
  */
-nsdefine('YIOOP_VERSION', 50);
+nsdefine('YIOOP_VERSION', 51);
 /**
  * Minimum Version fo Yioop for which keyword ad script
  * still works with this version
@@ -954,4 +954,4 @@ nsconddefine('AD_LOGO','resources/adv-logo.png');
 /** sentence compression enabled or not*/
 nsconddefine('SENTENCE_COMPRESSION_ENABLED', false);
 /** Define cipher to be used in AES */
-nsconddefine('AES_256_CBC', 'aes-256-cbc');
+nsconddefine('AES_256_CBC', 'aes-256-cbc');
\ No newline at end of file
diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php
index de76201..751bf31 100755
--- a/src/configs/Createdb.php
+++ b/src/configs/Createdb.php
@@ -568,10 +568,57 @@ if (stristr(DB_HOST, "pgsql") !== false) {
     }
 }
 
+//Insert the lexicon for all locales in the database
+foreach ($locales as $locale) {
+    $folderName = $locale[0];
+    if (strstr($locale[0], "-")) {
+        $localeName = explode("-", $locale[0]);
+        $folderName = $localeName[0] . "_" . $localeName[1];
+    }
+    $lexFile = LOCALE_DIR . "/" . $folderName . "/resources/lexicon.txt.gz";
+    if (file_exists($lexFile)) {
+        print "\n" . $lexFile;
+        $lines = gzfile($lexFile);
+        $insertValues = "";
+        $count = 0;
+        $timePre = microtime(true);
+        foreach ($lines as $line) {
+            $line = trim($line, " ");
+            $line = explode(" ", $line);
+            if (empty($line[0]) || empty($line[1])) {
+                continue;
+            }
+            $insertValues .= "(\"" . trim($line[0]) . "\",\"" . $locale[0] .
+                "\",\"" . trim($line[1]) . "\"),";
+            $count++;
+            if ($count >= 10000) {
+                $insertValues = rtrim($insertValues, ",");
+                $query = "INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                    {$insertValues}";
+                $db->exec($query);
+                $insertValues = "";
+                $count = 0;
+            }
+        }
+        if ($count > 0) {
+            $insert_values = rtrim($insertValues, ",");
+            $query = "INSERT INTO LEXICON (WORD, LOCALE, POS) VALUES
+                {$insertValues}";
+            $db->exec($query);
+            $insertValues = "";
+            $count = 0;
+        }
+        $timePost = microtime(true);
+        $execTime = $timePost - $timePre;
+        print "\n Lexicon : " . $lexFile;
+        print "\n Time to insert : " . $execTime;
+    }
+}
+
 $db->disconnect();
 $private_db->disconnect();
 if (in_array(DBMS, ['sqlite','sqlite3'])){
     chmod(CRAWL_DIR."/data/".DB_NAME.".db", 0666);
     chmod(CRAWL_DIR."/data/".PRIVATE_DB_NAME.".db", 0666);
 }
-echo "Create DB succeeded\n";
+echo "Create DB succeeded\n";
\ No newline at end of file
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index dce2a3f..8c602d3 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1392,3 +1392,12 @@ function upgradeDatabaseVersion50(&$db)
         SELECT GROUP_ID, GROUP_NAME, CREATED_TIME, OWNER_ID, REGISTER_TYPE,
         MEMBER_ACCESS, VOTE_ACCESS, POST_LIFETIME FROM GROUPS_OLD");
 }
+/**
+ * Upgrades a Version 50 version of the Yioop! database to a Version 51 version
+ * @param object $db datasource to use to upgrade.
+ */
+function upgradeDatabaseVersion51(&$db)
+{
+    $db->execute("CREATE TABLE LEXICON(WORD VARCHAR, LOCALE VARCHAR,
+                POS VARCHAR, PRIMARY KEY(WORD, LOCALE) ON CONFLICT IGNORE)";
+}
\ No newline at end of file
diff --git a/src/locale/hi/resources/lexicon.txt.gz b/src/locale/hi/resources/lexicon.txt.gz
index 5d7c3ba79dca9df5a33541c6a74cb8192e6cc93d..4f2e2010103459b68bb15a9725df198eda332dd7 100755
GIT binary patch
diff --git a/src/models/ProfileModel.php b/src/models/ProfileModel.php
index 24159a4..216eb47 100755
--- a/src/models/ProfileModel.php
+++ b/src/models/ProfileModel.php
@@ -332,6 +332,8 @@ class ProfileModel extends Model
                 ACCESS_COUNT INTEGER,
                 PRIMARY KEY(ADDRESS, PAGE_NAME))",
             "VERSION" => "CREATE TABLE VERSION(ID INTEGER PRIMARY KEY)",
+            "LEXICON" => "CREATE TABLE LEXICON(WORD VARCHAR, LOCALE VARCHAR,
+                POS VARCHAR, PRIMARY KEY(WORD, LOCALE) ON CONFLICT IGNORE)",
             ];
     }
     /**
@@ -825,4 +827,4 @@ EOT;
         }
         return $match;
     }
-}
+}
\ No newline at end of file
-- 
2.10.0.windows.1

