_binary From 30b55662f5bad3437655f517a81d93fee87ec058 Mon Sep 17 00:00:00 2001
From: Sarika Padmashali <padmashalisarika@gmail.com>
Date: Wed, 17 May 2017 19:42:39 -0700
Subject: [PATCH] Recommendation System Patch

---
 src/configs/Config.php                             |    4 +-
 src/configs/Createdb.php                           |    3 +-
 .../components/AccountaccessComponent.php          |    9 +
 src/library/VersionFunctions.php                   |   62 +
 src/library/media_jobs/RecommendationJob.php       | 1481 ++++++++++++++++++++
 src/models/ProfileModel.php                        |   62 +
 src/models/UserModel.php                           |  130 ++
 src/views/elements/ManageaccountElement.php        |   37 +
 tests/RecommendationTest.php                       |   83 ++
 9 files changed, 1868 insertions(+), 3 deletions(-)
 create mode 100644 src/library/media_jobs/RecommendationJob.php
 create mode 100644 tests/RecommendationTest.php

diff --git a/src/configs/Config.php b/src/configs/Config.php
index a408067a..af2f0a1a 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -112,7 +112,7 @@ function nsconddefine($constant, $value)
  * Version number for upgrade function
  * @var int
  */
-nsdefine('YIOOP_VERSION', 52);
+nsdefine('YIOOP_VERSION', 53);
 /**
  * Minimum Version fo Yioop for which keyword ad script
  * still works with this version
@@ -249,7 +249,7 @@ if (MAINTENANCE_MODE && $_SERVER["SERVER_ADDR"] != $_SERVER["REMOTE_ADDR"]) {
 nsdefine('DEFAULT_WORK_DIRECTORY', PARENT_DIR . "/work_directory");
 
 if (!nsdefined('WORK_DIRECTORY')) {
-/*+++ The next block of code is machine edited, change at
+/*+++ The next block of code is machine edited, change at 
 your own risk, please use configure web page instead +++*/
 nsdefine('WORK_DIRECTORY', DEFAULT_WORK_DIRECTORY);
 /*++++++*/
diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php
index b2a6c084..bc91d867 100755
--- a/src/configs/Createdb.php
+++ b/src/configs/Createdb.php
@@ -575,11 +575,12 @@ if (stristr(DB_HOST, "pgsql") !== false) {
         $db->execute($sql);
     }
 }
-
 $db->disconnect();
 $private_db->disconnect();
 if (in_array(DBMS, ['sqlite','sqlite3'])){
     chmod(CRAWL_DIR."/data/".DB_NAME.".db", 0666);
     chmod(CRAWL_DIR."/data/".PRIVATE_DB_NAME.".db", 0666);
 }
+$db->execute("INSERT INTO AVERAGE_RATING VALUES (0)");
+$db->execute("INSERT INTO LIVE_TABLE VALUES (1,1,1)");
 echo "Create DB succeeded\n";
diff --git a/src/controllers/components/AccountaccessComponent.php b/src/controllers/components/AccountaccessComponent.php
index 24e8945b..48c6c52f 100644
--- a/src/controllers/components/AccountaccessComponent.php
+++ b/src/controllers/components/AccountaccessComponent.php
@@ -94,6 +94,15 @@ class AccountaccessComponent extends Component
             $data['yioop_bot_configuration'] = false;
         }
         $user_id = $_SESSION['USER_ID'];
+        /**
+        * Preparing data for recommending users with threads and groups
+        **/
+        $recommendations_threads = $user_model->getRecommendationsThreads($user_id);
+        $recommendations_groups = $user_model->getRecommendationsGroups($user_id);
+        $recommendations_trending = $user_model->getRecommendationsTrendingGroups($user_id);
+        $data['RECOMMENDATIONS_THREADS']= $recommendations_threads;
+        $data['RECOMMENDATIONS_GROUPS']= $recommendations_groups;
+        $data['RECOMMENDATIONS_TRENDING'] = $recommendations_trending;
         $username = $signin_model->getUserName($user_id);
         $data["USER"] = $user_model->getUser($username);
         $data["CRAWL_MANAGER"] = false;
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index f50b3844..ef96b677 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1449,3 +1449,65 @@ function upgradeDatabaseVersion52(&$db)
         ITEM_TYPE INTEGER, UPDATE_PERIOD INTEGER, SUM INTEGER DEFAULT -1,
         FUZZY_NUM_VIEWS INTEGER DEFAULT -1");
 }
+/**
+ * Upgrades a Version 52 version of the Yioop! database to a Version 53 version
+ * @param object $db datasource to use to upgrade.
+ */
+function upgradeDatabaseVersion53(&$db)
+{
+    $db->execute("CREATE TABLE AVERAGE_RATING (AVG_RATING INT)");
+    $db->execute("INSERT INTO AVERAGE_RATING VALUES (0)");
+    $db->execute("CREATE TABLE IDF (WORD_ID INT, ITEM_IMP FLOAT,
+        USER_IMP FLOAT)");
+    $db->execute("CREATE TABLE IDF_TEMP (WORD_ID INT, ITEM_IMP
+        FLOAT, USER_IMP FLOAT)");
+    $db->execute("CREATE TABLE ITEM_BIAS(ITEM_ID INT,
+        ITEM_BIAS FLOAT)");
+    $db->execute("CREATE TABLE ITEM_WORD_FREQUENCY (ITEM_ID INT,
+        WORD_ID INT, FREQUENCY INT, LOGFREQUENCY FLOAT)");
+    $db->execute("CREATE TABLE ITEM_WORD_WEIGHTS(WORD_ID INT,
+        ITEM_ID INT ,WEIGHT FLOAT)");
+    $db->execute("CREATE TABLE LIVE_TABLE (TRENDING INT,
+        THREADS INT, GROUPS INT)");
+    $db->execute("INSERT INTO LIVE_TABLE VALUES (1,1,1)");
+    $db->execute("CREATE TABLE PREDICTED_TABLE(USER_ID INT,
+        ITEM_ID INT, USER_BIAS FLOAT, ITEM_BIAS FLOAT, RATING FLOAT)");
+    $db->execute("CREATE TABLE RECOMMENDATION_LIST_GROUPS_1
+        (USER_ID INT, GROUP_1 INT, GROUP_2
+        INT, GROUP_3 INT, RECOMMEND_TYPE VARCHAR)");
+    $db->execute("CREATE TABLE RECOMMENDATION_LIST_GROUPS_2 
+        (USER_ID INT, GROUP_1 INT, GROUP_2
+        INT, GROUP_3 INT, RECOMMEND_TYPE VARCHAR)");
+    $db->execute("CREATE TABLE RECOMMENDATION_LIST_THREADS_1 
+        (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+        RECOMMEND_TYPE VARCHAR)");
+    $db->execute("CREATE TABLE RECOMMENDATION_LIST_THREADS_2 
+        (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+        RECOMMEND_TYPE VARCHAR)");
+    $db->execute("CREATE TABLE RECOMMENDATION_TRENDING_THREADS_1 
+        (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+        RECOMMEND_TYPE VARCHAR)");
+    $db->execute("CREATE TABLE RECOMMENDATION_TRENDING_THREADS_2 
+        (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+        RECOMMEND_TYPE VARCHAR)");
+    $db->execute("CREATE TABLE SIMILARITY 
+        (USER_ID INT, ITEM_ID INT, COSINE_SIMILARITY FLOAT)");
+    $db->execute("CREATE TABLE USER_BIAS 
+        (USER_ID INT,USER_BIAS FLOAT)");
+    $db->execute("CREATE TABLE USER_WORD_FREQUENCY 
+        (FREQUENCY INT, USER_ID INT, WORD_ID INT, 
+        LOGFREQUENCY FLOAT)");
+    $db->execute("CREATE TABLE USER_WORD_FREQUENCY_TEMP 
+        (FREQUENCY INT, USER_ID INT, WORD_ID INT, LOGFREQUENCY FLOAT)");
+    $db->execute("CREATE TABLE USER_WORD_WEIGHTS 
+        (WORD_ID INT, USER_ID INT, WEIGHT FLOAT)");
+    $db->execute("CREATE TABLE USER_WORD_WEIGHTS_TEMP 
+        (WORD_ID INT, USER_ID INT, WEIGHT FLOAT)");
+    $db->execute("CREATE TABLE WORDS 
+        (WORD_ID INT, WORD VARCHAR)");
+    $db->execute("CREATE TABLE WORDS_TEMP (WORD_ID INT)");
+    $db->execute("CREATE TABLE SIMILARITY_TEMP 
+        (USER_ID INT, ITEM_ID INT, COSINE_SIMILARITY FLOAT)");
+    $db->execute("CREATE TABLE USER_THREAD_GROUP 
+                (USER_ID INT, ITEM_ID INT, GROUP_ID INT)");
+}
\ No newline at end of file
diff --git a/src/library/media_jobs/RecommendationJob.php b/src/library/media_jobs/RecommendationJob.php
new file mode 100644
index 00000000..712f80a6
--- /dev/null
+++ b/src/library/media_jobs/RecommendationJob.php
@@ -0,0 +1,1481 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2017  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Sarika Padmashali padmashalisarika@gmail.com
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2017
+ * @filesource
+ */
+namespace seekquarry\yioop\library\media_jobs;
+
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+use seekquarry\yioop\library\CrawlConstants;
+use seekquarry\yioop\library\processors\PageProcessor;
+use seekquarry\yioop\models\ImpressionModel;
+use seekquarry\yioop\models\MachineModel;
+use seekquarry\yioop\models\PhraseModel;
+
+/**
+ * Recommendation Job recommends the trending threads as well 
+ * as threads and groups which are relevant based on the 
+ * users viewing history
+ */
+class RecommendationJob extends MediaJob
+{
+    /**
+     * Time in current epoch when analytics last updated
+     * @var int
+     */
+    public $update_time;
+    /**
+     * Keeps the track of the last word added in the dictionary
+     * @var int
+     */
+    public $word_count;
+    /**
+     * Used to get statistics from DBMS about wiki and thread views
+     *
+     * @var object
+     */
+    public $impression_model;
+    /**
+     *  Used to get crawl statistics about the number of various HTTP response
+     * requests seen during a crawl
+     *
+     * @var object
+     */
+    public $phrase_model;
+    /**
+     * Used to determine which queue servers are available and which might
+     * have information about a crawl
+     *
+     * @var object
+     */
+    public $machine_model;
+    /**
+     * For size and time distrbutions the number of times the miminal
+     * recorded interval (DOWNLOAD_SIZE_INTERVAL for size) to check for
+     * pages with that size/download time
+     */
+    const NUM_TIMES_INTERVAL = 50;
+    /**
+     * While computing the statistics page, number of seconds until a
+     * page refresh and save of progress so far occurs
+     */
+    const STATISTIC_REFRESH_RATE = C\ANALYTICS_UPDATE_INTERVAL/2;
+    /**
+     * Initializes the time of last analytics update
+     */
+    /**
+     * The number of time a job should run in incremental mode
+     */
+    const NUMBER_RUNS = 2;
+    public function init()
+    {
+        $this->update_time = 0;
+        $this->live_table = 0;
+        $this->last_updated = 0;
+        $this->word_count = 0;
+        $this->new_word_start = 0;
+        $this->number_job_incremental_mode = 2;
+        $this->name_server_does_client_tasks = true;
+        $this->name_server_does_client_tasks_only = true;
+        $this->impression_model = new ImpressionModel();
+        $this->phrase_model = new PhraseModel();
+        $this->machine_model = new MachineModel();
+        $db_class = C\NS_DATASOURCES . ucfirst(C\DBMS). "Manager";
+        $this->db = new $db_class();
+        $this->db->connect();
+        PageProcessor::initializeIndexedFileTypes();
+    }
+    /**
+     * Only update if its been more than an hour since the last update
+     *
+     * @return bool whether its been an hour since the last update
+     */
+    public function checkPrerequisites()
+    {
+        $time = time();
+        $delta = $time - $this->update_time;
+        if ($delta > 10) {
+            $this->update_time = $time;
+            L\crawlLog("Performing the Recommendation Media Job");
+            return true;
+        }
+        L\crawlLog("Time since last update not exceeded, skipping".
+            " Recommendation MediaJob $delta");
+        return false;
+    }
+    /**
+     * For now analytics update is only done on name server as Yioop
+     * currently only supports one DBMS at a time.
+     */
+    public function nondistributedTasks()
+    {
+        $db = $this->db;
+        $live = $this->live_table;
+        $updating = 1;
+        if ($live == 1) {
+            $updating = 2;
+        }
+        $recommendation_table_threads = "RECOMMENDATION_".
+            "LIST_THREADS_".$updating;
+        $recommendation_table_groups = "RECOMMENDATION_".
+            "LIST_GROUPS_".$updating;
+        $recommendation_trending_threads_table = "RECOMMENDATION_".
+            "TRENDING_THREADS_".$updating;    
+        L\crawlLog("tables: $recommendation_table_threads, ".
+            "$recommendation_table_groups");
+        $count = $this->word_count;
+        $trending_job = $this->trendingJob($db,$updating);
+        if ($this->number_job_incremental_mode == self::NUMBER_RUNS) {
+            $this->batchJob($db, $recommendation_table_threads, 
+                $recommendation_table_groups, $updating);
+
+        } else {
+            $this->incrementalJob($db, $recommendation_table_threads, 
+                $recommendation_table_groups, $updating);
+        }
+        $resolving_cold_start = $this->resolvingColdStart($db, 
+            $recommendation_table_threads, $recommendation_table_groups, 
+            $recommendation_trending_threads_table);
+    }
+    /**
+     * Function which implements the batch job
+     * @param database connection $db
+     */
+    public function resolvingColdStart($db, $recommendation_table_threads,
+        $recommendation_table_groups, $recommendation_trending_threads_table)
+    {
+        $sql_popular_threads = "SELECT COUNT(*) AS POPULAR, GROUP_1 ".
+            "AS THREADS FROM $recommendation_table_threads ".
+            "GROUP BY GROUP_1 ORDER BY POPULAR DESC";
+        $execute_popular_threads = $db->execute($sql_popular_threads);
+        $count = 1;
+        $popular_threads_1 = null;
+        $popular_threads_2 = null;
+        $popular_threads_3 = null;
+        while($count <= 3 AND ($item = $db->fetchArray(
+            $execute_popular_threads))) {
+            if($count == 1) {
+                $popular_threads_1 = $item['THREADS'];
+            }
+            if($count == 2) {
+                $popular_threads_2 = $item['THREADS'];
+            }
+            if($count == 3) {
+                $popular_threads_3 = $item['THREADS'];
+            }
+            $count = $count + 1;
+        }
+        $sql_popular_threads1 = "SELECT COUNT(*) AS POPULAR, GROUP_1 AS".
+            " THREADS  FROM $recommendation_table_groups ".
+            "GROUP BY GROUP_1 ORDER BY POPULAR DESC";
+        $execute_popular_groups = $db->execute($sql_popular_threads1);
+        $count1 = 1;
+        $popular_groups_1 = null;
+        $popular_groups_2 = null;
+        $popular_groups_3 = null;
+        while($count1 <= 3 AND $item2 = $db->fetchArray(
+            $execute_popular_groups)) {
+            if($count1 == 1) {
+                $popular_groups_1 = $item2['THREADS'];
+            }
+            if($count1 == 2) {
+                $popular_groups_2 = $item2['THREADS'];
+            }
+            if($count1 == 3) {
+                $popular_groups_3 = $item2['THREADS'];
+            }
+            $count1 = $count1 + 1;
+        }
+        $sql_popular_trendings = "SELECT COUNT(*) AS POPULAR, GROUP_1 AS".
+            " THREADS  FROM $recommendation_trending_threads_table ".
+            "GROUP BY GROUP_1 ORDER BY POPULAR DESC";
+        $execute_popular_trending = $db->execute($sql_popular_trendings);
+        $count2 = 1;
+        $popular_trending_1 = null;
+        $popular_trending_2 = null;
+        $popular_trending_3 = null;
+        while($count2 <= 3 AND $item3 = $db->fetchArray(
+            $execute_popular_trending)) {
+            if($count2 == 1) {
+                $popular_trending_1 = $item3['THREADS'];
+            }
+            if($count2 == 2) {
+                $popular_trending_2 = $item3['THREADS'];
+            }
+            if($count2 == 3) {
+                $popular_trending_3 = $item3['THREADS'];
+            }
+            $count2 = $count2 + 1;
+        }
+        $cold_start_users = "SELECT USER_ID AS USER_ID ".
+            "FROM USERS WHERE USER_ID NOT IN ".
+            "(SELECT USER_ID FROM $recommendation_table_threads)";
+        $execute_cold_users = $db->execute($cold_start_users);
+        $sql_thread = "INSERT INTO $recommendation_table_threads 
+            VALUES(?, ?, ?, ?, ?)";
+        while($row = $db->fetchArray($execute_cold_users)) {
+            $user = $row['USER_ID'];
+            $execute_recommend = $db->execute($sql_thread, 
+                [$user, $popular_threads_1, $popular_threads_2, 
+                $popular_threads_3, 'THREAD']);
+        }
+        $cold_start_users1 = "SELECT USER_ID AS USER_ID FROM USERS ".
+            "WHERE USER_ID NOT IN (SELECT USER_ID ".
+            "FROM $recommendation_table_groups)";
+        $execute_cold_users1 = $db->execute($cold_start_users1);
+        $sql_thread1 = "INSERT INTO $recommendation_table_groups 
+            VALUES(?, ?, ?, ?, ?)";
+        while($row = $db->fetchArray($execute_cold_users1)) {
+            $user = $row['USER_ID'];
+            $execute_recommend1 = $db->execute($sql_thread1, 
+                [$user, $popular_groups_1, $popular_groups_2, 
+                $popular_groups_3, 'GROUP']);
+        }
+        $cold_start_users2 = "SELECT USER_ID AS USER_ID FROM ".
+            "USERS WHERE USER_ID NOT IN (SELECT USER_ID ".
+            "FROM $recommendation_trending_threads_table)";
+        $execute_cold_users2 = $db->execute($cold_start_users2);
+        $sql_thread2 = "INSERT INTO $recommendation_trending_threads_table 
+            VALUES(?, ?, ?, ?, ?)";
+        while($row = $db->fetchArray($execute_cold_users2)) {
+            $user = $row['USER_ID'];
+            $execute_recommend2 = $db->execute($sql_thread2, 
+                [$user, $popular_trending_1, $popular_trending_2, 
+                $popular_trending_3, 'THREAD']);
+        }
+    }
+    /**
+     * Function which implements the batch job
+     * @param database connection $db
+     */
+    public function batchJob($db, $recommendation_table_threads, 
+        $recommendation_table_groups, $updating)
+    {
+        $this->number_job_incremental_mode =
+                $this->number_job_incremental_mode - 1;
+        if ($this->number_job_incremental_mode == 0) {
+            $this->number_job_incremental_mode = 2;
+        }
+        $this->last_updated = $this->update_time;
+        $count = $this->word_count;
+        L\crawlLog("Performing TF-IDF Recommendation Job in batch mode");
+        $delete_records = $this->deleteTablesRecords($db,
+            $recommendation_table_threads, $recommendation_table_groups);
+        /*
+         Extracting all the title and description content of threads
+         from the table GROUP_ITEM
+         Calculating Term Frequency for items and
+         creating a bag of words
+        */
+        $term_frequency_items = $this->termFrequencyItem($db, $count);
+        $number_items = $this->numberItems($db);
+        $term_frequency_items = $this->termFrequencyUser($db);
+        $number_users = $this->numberUsers($db);
+        /*
+         Calculating IDF for users and groups and storing in a table
+        */
+        $calculating_idf = $this->idf($db, $number_items,
+            $number_users);
+        /*
+         Calculating the TF*IDF for items and users
+        */
+        $calculating_tf_idf_users = $this->tfIdfUsers($db);
+        $calculating_tf_idf_items = $this->tfIdfItems($db);
+        $user_part_of_thread = $this->userThreadMap($db);
+        /*
+         Creating the similarity table
+         of users and the threads they might like based on bag of words
+        */
+        $computing_similarity = $this->computeSimilarity($db);
+        $recommending_threads = $this->recommendingThreads($db,
+            $recommendation_table_threads , 'SIMILARITY');
+        $this->last_updated = $this->update_time;
+        $this->word_count = $count; 
+        /*
+         Groups Recommendation
+        */
+        $recommending_groups = $this->recommendingGroups($db,
+            $recommendation_table_groups);
+        /*
+         updating live table for recommendations
+         this is the table from which we will be showing recommendations
+        */
+        $this->live_table = $updating;
+        $update_live_table = $this->updateLiveTable($db, $updating);
+        $this->doTasks([]);      
+    }
+    /**
+     * Function which implements incremental job
+     * @param database connection $db
+     */
+    public function incrementalJob($db, $recommendation_table_threads,
+        $recommendation_table_groups, $updating)
+    {
+        /*
+        Running Job in incremental mode
+        */
+        L\crawlLog("Recommendation job running in incremental mode");
+        $this->number_job_incremental_mode =
+            $this->number_job_incremental_mode - 1;
+        if ($this->number_job_incremental_mode == 0) {
+            $this->number_job_incremental_mode = 2;
+        }
+        $delete_incremental_tables = $this->incrementalDelete($db,
+        $recommendation_table_threads, $recommendation_table_groups);
+        $count = $this->word_count;
+        $last_update = $this->last_updated;
+        $current = $this->update_time;
+        L\crawlLog("Last Updated $last_update and current time ".
+            "$current");
+        /*
+         Number of new users, old users,
+         total users, thread items in the corpus
+        */
+        $number_new_users = $this->countNewUsers($db, $last_update);
+        $number_old_users = $this->countOldUsers($db, $last_update);
+        $number_users = $number_new_users + $number_old_users;
+        $number_items = $this->numberItems($db);
+        /*
+         Extracting the title and description, calculates
+         the term frequency for users and items
+        */
+        $term_frequency_items_incremental =
+            $this->termFrequencyItemIncremental($db,
+                $number_items, $last_update, $count);
+        $term_frequency_items_incremental =
+                $this->termFrequencyUserIncremental($db,
+                        $last_update, $current);
+        $updatedList = $this->updatedWordListIncremental($db);
+        $number_distinct_users = $this->countDistinctUsers($db);
+        /**
+            Calculating IDF for users and groups and storing in a table
+        */
+        $calculating_idf_incremental = $this->idfIncremental($db,
+        $number_items, $number_distinct_users);
+        /*
+            Caluclating the factor of multiplication
+        */
+        $total_users = $number_new_users + $number_old_users;
+        $old_factor =
+            floatval($number_old_users/floatval($total_users));
+        $new_factor =
+            floatval($number_new_users+1.0/floatval($total_users));
+        $calculating_tf_idf_incremental =
+            $this->tfidfusersIncremental($db,
+                $old_factor, $new_factor);
+        /*
+         Creating the similarity table
+         of users and the threads they might like based on bag of words
+        */
+        $calculating_similarity_temp =
+            $this->computeSimilarityIncremental($db);
+        $recommending_threads =
+        $this->recommendingThreads($db,
+                $recommendation_table_threads,
+                'SIMILARITY_TEMP');
+        $this->last_updated = $this->update_time;
+        /*
+         Groups Recommendation
+        */
+        $recommending_groups = $this->recommendingGroups($db,
+            $recommendation_table_groups);
+        /*
+         updating live table for recommendations
+         this is the table from which we will be showing recommendations
+        */
+        $this->live_table = $updating;
+        $update_live_table = $this->updateLiveTable($db, $updating);
+        $this->doTasks([]);
+    }
+    /**
+     * Updating Live table
+     * @param database connection $db
+     * @param int $updating the latest table updated
+     */
+    public function updateLiveTable($db, $updating)
+    {
+        $sql_live ="UPDATE LIVE_TABLE SET THREADS = ? , GROUPS = ? ";
+        $db->execute($sql_live, [$updating, $updating]);
+    }
+    /**
+     * Delete all rows from item_word_frequency table, 
+     * IDF,ITEM_WORD_FREQUENCY,
+     * WORDS, USER_WORD_FREQUENCY, USER_WORD_WEIGHTS, ITEM_WORD_WEIGHTS,
+     * RECOMMENDATION LIST table before inserting
+     * @param database connection $db
+     * @param string $recommendation_threads_table table to be updated
+     * @param string $recommendation_groups_table table to be updated
+     */
+    public function deleteTablesRecords($db, $recommendation_threads_table,
+        $recommendation_groups_table)
+    {
+        $tables = array("IDF", "ITEM_WORD_FREQUENCY", "USER_WORD_FREQUENCY",
+            "USER_WORD_WEIGHTS", "ITEM_WORD_WEIGHTS",
+            $recommendation_threads_table, "WORDS",
+            "SIMILARITY", "USER_THREAD_GROUP", $recommendation_groups_table);
+        foreach ($tables as $value) {
+            $sql_delete ="DELETE FROM $value";
+            $db->execute($sql_delete);
+        }
+    }
+    /**
+     * Computes the number of items
+     * @param database connection $db
+     * @return int number of items
+     */
+    public function numberItems($db)
+    {
+        $sql_number_documents = "SELECT COUNT(*) AS NUMBER FROM ".
+            "GROUP_ITEM WHERE TITLE NOT LIKE '%wiki%'";
+        $result_number_documents = $db->execute($sql_number_documents);
+        $number_items = 0;
+        $row = $db->fetchArray($result_number_documents);
+        $number_items = $row['NUMBER'];
+        return $number_items;
+    }
+    /**
+     * Computes the number of users
+     * @param database connection $db
+     * @return int number of users
+     */
+    public function numberUsers($db)
+    {
+        $sql_number_users = "SELECT COUNT(*) AS NUMBER FROM USERS";
+        $result_number_users = $db->execute($sql_number_users);
+        $number_users = 0;
+        $rows = $db->fetchArray($result_number_users);
+        $number_users = $rows['NUMBER'];
+        return $number_users;
+    }
+    /**
+     * Computes the term frequency for items
+     * @param database connection $db
+     * @param int $count word_id of the last word added in words table
+     */
+    public function termFrequencyItem($db, $count)
+    {
+        $sql_extracting_title_desc = "SELECT ID AS ITEM_ID, TITLE, ".
+            "DESCRIPTION FROM GROUP_ITEM WHERE TITLE NOT LIKE '%wiki%'";
+        $result_set = $db->execute($sql_extracting_title_desc);
+        $sql_insert_word_freq1 = "INSERT INTO ITEM_WORD_FREQUENCY".
+            " VALUES(?, ?, ?, ?)";
+        while ($item = $db->fetchArray($result_set)) {
+            $recommend_id = $item['ITEM_ID'];
+            $record = $item['TITLE']." ".$item['DESCRIPTION'];
+            $word_frequency = $this->wordCount($record);
+            foreach ($word_frequency as $key => $value) {
+                $sql_check_words_exists = "SELECT COUNT(*) AS COUNT,".
+                    " WORD_ID AS WORD_ID FROM WORDS WHERE WORD LIKE '{$key}'";
+                $g = $db->execute($sql_check_words_exists);
+                $exists = $db->fetchArray($g);
+                $t=$exists['COUNT'];
+                $word_id = $exists['WORD_ID'];
+                if ($t == 0) {
+                    $sql_creating_words_list = "INSERT INTO WORDS ".
+                        "VALUES ($count,'$key')";
+                    $insert = $db->execute($sql_creating_words_list);
+                    $logValue = log($value,10) + 1;
+                    $inserted_row = $db->execute($sql_insert_word_freq1,
+                        [$recommend_id,$count,$value,$logValue]);
+                    $count = $count + 1;
+                    $this->word_count = $this->word_count + 1;
+                } else {
+                    $logValue = log($value,10) + 1;
+                    $inserted1 = $db->execute($sql_insert_word_freq1 ,
+                        [$recommend_id,$word_id,$value,$logValue]);
+                }
+            }
+        }
+    }
+    /**
+     * Calculating the words count
+     * @param string string of words
+     * @return array of words and count
+     */
+    public static function wordCount($record)
+    {
+        $array_of_words = explode(" ", $record);
+        $word_frequency = array_count_values($array_of_words);
+        return $word_frequency;
+    }
+    /**
+     * Calculating the term frequency
+     * @param database connection $db
+     */
+    public function termFrequencyUser($db)
+    {
+        $sql_user_word_count = "SELECT COUNT(*) AS FREQUENCY, USER_ID ".
+            "AS UID, WORD_ID AS WID FROM ITEM_WORD_FREQUENCY IWF ".
+            "INNER JOIN ITEM_IMPRESSION II WHERE IWF.ITEM_ID = II.ITEM_ID".
+            " GROUP BY USER_ID, WORD_ID";
+        $result_user_word_count = $db->execute($sql_user_word_count);
+        while($rows_fetched = $db->fetchArray($result_user_word_count)) {
+            $frequency_words = $rows_fetched['FREQUENCY'];
+            $uid = $rows_fetched['UID'];
+            $wid = $rows_fetched['WID'];
+            $logValue = log($frequency_words,10) + 1.0;
+            $sql_insert_word_freq_user= "INSERT INTO ".
+                "USER_WORD_FREQUENCY VALUES(?, ?, ?, ?)";
+            $inserted_row_uwf = $db->execute($sql_insert_word_freq_user,
+                [$frequency_words, $uid, $wid, $logValue]);
+        }
+    }
+    /**
+     * Computes the idf for users and items for newly updated word impressions
+     * @param database connection $db
+     * @param int $number_items number of items
+     * @param int $number_users number of  users
+     */
+    public function idf($db, $number_items, $number_users)
+    {
+        $sql_words = "SELECT WORD W, WORD_ID AS WID FROM WORDS";
+            $result_list = $db->execute($sql_words);
+        $sql_item_with_word ="SELECT COUNT(DISTINCT ITEM_ID)".
+            " AS COUNT_ITEMS_WITH_WORD FROM ITEM_WORD_FREQUENCY ".
+            "WHERE WORD_ID = ? ";
+        $sql_user_with_word ="SELECT COUNT(DISTINCT USER_ID) ".
+            "AS COUNT_USERS_WITH_WORD FROM USER_WORD_FREQUENCY ".
+            "WHERE WORD_ID = ? ";
+        $sql_insert_idf = "INSERT INTO IDF VALUES (?, ?, ?)";
+        while($list = $db->fetchArray($result_list)) {
+            $word = $list['W'];
+            $word_id= $list['WID'];
+            /*
+                Number of groups having the required word
+            */
+            $execute_sql_item_with_word =
+                $db->execute($sql_item_with_word, [$word_id]);
+            $fetched_sql_item_with_word =
+                $db->fetchArray($execute_sql_item_with_word);
+            $number_items_with_word =
+                $fetched_sql_item_with_word['COUNT_ITEMS_WITH_WORD'];
+            $idf_word_item = log($number_items/$number_items_with_word, 10);
+            /*
+                Number of users having the required word
+            */
+                $execute_sql_user_with_word =
+                    $db->execute($sql_user_with_word, [$word_id]);
+                $fetched_sql_user_with_word =
+                    $db->fetchArray($execute_sql_user_with_word);
+                $number_users_with_word =
+                    $fetched_sql_user_with_word['COUNT_USERS_WITH_WORD'];
+                $idf_word_user =
+                    log($number_users/$number_users_with_word, 10);
+            /*
+                Inserting the inverse document frequency value for groups
+            */
+            $execute_insert_idf = $db->execute($sql_insert_idf,
+                [$word_id, $idf_word_item, $idf_word_user]);
+            }
+    }
+    /**
+     * Calculating the TF*IDF for users
+     * @param database connection $db
+     */
+    public function tfIdfUsers($db)
+    {
+        $sql_weights_users = "SELECT IDF.USER_IMP* UWF.LOGFREQUENCY AS ".
+            "WEIGHT, UWF.WORD_ID AS WORD_ID, UWF.USER_ID AS USER_ID FROM ".
+            "IDF JOIN USER_WORD_FREQUENCY AS UWF ".
+            "WHERE IDF.WORD_ID = UWF.WORD_ID";
+        $sql_insert_uww = "INSERT INTO USER_WORD_WEIGHTS ".
+            "VALUES (?, ?, ?)";
+        $execute_weights_users = $db->execute($sql_weights_users);
+        while($list = $db->fetchArray($execute_weights_users)) {
+            $weight = $list['WEIGHT'];
+            $word_id= $list['WORD_ID'];
+            $user_id= $list['USER_ID'];
+            $execute_insert_uww = $db->execute($sql_insert_uww,
+                [$word_id, $user_id, $weight]);
+        }
+    }
+    /**
+     * Calculating the TF*IDF for items
+     * @param database connection $db
+     */
+    public function tfIdfItems($db)
+    {
+        $sql_weights_items = "SELECT IWF.WORD_ID AS WORD_ID, IWF.ITEM_ID ".
+            "AS ITEM_ID, IDF.ITEM_IMP*IWF.LOGFREQUENCY AS ".
+            "WEIGHT FROM IDF INNER JOIN ITEM_WORD_FREQUENCY IWF ".
+            "WHERE IDF.WORD_ID = IWF.WORD_ID";
+        $execute_weights_items = $db->execute($sql_weights_items);
+        $sql_insert_iww = "INSERT INTO ITEM_WORD_WEIGHTS VALUES (?, ?, ?)";
+        while($list = $db->fetchArray($execute_weights_items)) {
+            $weight = $list['WEIGHT'];
+            $word_id= $list['WORD_ID'];
+            $item_id= $list['ITEM_ID'];
+            $execute_insert_iww = $db->execute($sql_insert_iww,
+                [$word_id, $item_id, $weight]);
+            }
+    }
+    /**
+     * Creating the similarity table
+     * of users and the threads they might like based on bag of words
+     * @param database connection $db
+     */
+    public function computeSimilarity($db)
+    {
+        $sql_weights = "SELECT SUM(UWW.WEIGHT*IWW.WEIGHT) AS DOT_PRODUCT, ".
+            "SUM(UWW.WEIGHT) AS USER_MAG, SUM(IWW.WEIGHT) AS ITEM_MAG, * ".
+            "FROM ITEM_WORD_WEIGHTS IWW INNER JOIN ".
+            "USER_WORD_WEIGHTS UWW GROUP BY USER_ID, ITEM_ID";
+        $result_weights = $db->execute($sql_weights);
+        $sql_already_a_member = "SELECT COUNT(UG.USER_ID) AS ".
+            "COUNTER ,GI. ID AS ITEM_ID FROM ".
+            "USER_GROUP UG JOIN GROUP_ITEM GI WHERE ".
+            "UG.GROUP_ID = GI.GROUP_ID AND GI.TITLE ".
+            "NOT LIKE '%wiki%' AND UG.USER_ID = ? AND  ITEM_ID = ? ";
+        $public_join = "SELECT G.GROUP_ID, G.REGISTER_TYPE AS REGISTER FROM ".
+            "GROUPS G JOIN GROUP_ITEM GI WHERE ".
+            "G.GROUP_ID = GI.GROUP_ID AND GI.ID = ? ";
+        $sql_insert_into_recommendation = "INSERT INTO ".
+            "SIMILARITY VALUES (?, ?, ?)";
+        while($rows = $db->fetchArray($result_weights)) {
+            $dot_product= $rows['DOT_PRODUCT'];
+            $user_magnitude = $rows['USER_MAG'];
+            $item_magnitude = $rows['ITEM_MAG'];
+            $item_id = $rows['ITEM_ID'];
+            $user_id = $rows['USER_ID'];
+            $result =
+                $db->execute($sql_already_a_member, [$user_id, $item_id]);
+            $exist = $db->fetchArray($result);
+            $T = $exist['COUNTER'];
+            $access_result = $db->execute($public_join, [$item_id]);
+            $access_fetch = $db->fetchArray($access_result);
+            $accessible = $access_fetch['REGISTER'];
+            if ($T == 0 AND ($accessible == 3 OR $accessible == 4)) {
+                $cosine_similarity = floatval($dot_product)
+                    /floatval($user_magnitude*$item_magnitude);
+                $execute_insert_into_recommendation =
+                    $db->execute($sql_insert_into_recommendation,
+                    [$user_id, $item_id, $cosine_similarity]);
+            }
+        }
+    }
+    /**
+     * Threads Recommendation
+     * @param database connection $db
+     * @param STRING $recommendation_table_threads the table to be updated
+     * @param STRING the name of similarity table to be consider
+     */
+    public function recommendingThreads($db, 
+        $recommendation_table_threads, $similarity_table)
+    {
+        $sql_similarity = "SELECT * FROM $similarity_table ".
+            "GROUP BY USER_ID, ITEM_ID ORDER BY USER_ID ASC,".
+            " COSINE_SIMILARITY DESC";
+        $similar_results = $db->execute($sql_similarity);
+        $first_record = $db->fetchArray($similar_results);
+        $user_id= $first_record['USER_ID'];
+        $item_id = $first_record['ITEM_ID'];
+        $cosine_similarity = $first_record['COSINE_SIMILARITY'];
+        $current_user_id = $user_id;
+        $old_user_id = $user_id;
+        $sql_member = "SELECT COUNT(*) as EXISTS FROM USER_THREAD_GROUP ".
+            "WHERE USER_ID = ? AND ITEM_ID = ? ";
+        $execute_sql_member = $db->execute($sql_member, [$user_id, $item_id]);
+        $existing = $db->fetchArray($execute_sql_member);
+        $member = $existing['EXISTS'];
+        $count_number = 1;
+        $first_record = false;
+        if($member == 0) {
+            $sql_insert_into_recommendation = "INSERT INTO ".
+                "$recommendation_table_threads VALUES ".
+                "($user_id, $item_id, 0, 0, 'THREAD')";
+            $execute_insert_into_recommendation = $db->execute(
+                $sql_insert_into_recommendation);
+            $sql_exists = "SELECT * FROM ".
+                "$recommendation_table_threads WHERE USER_ID = ? ";
+            $count_number = 1;
+            $first_record = true;
+        }
+        while($rows = $db->fetchArray($similar_results)) {
+            $user_id= $rows['USER_ID'];
+            $item_id = $rows['ITEM_ID'];
+            $cosine_similarity = $rows['COSINE_SIMILARITY'];
+            $current_user_id = $user_id;
+            $execute_sql_member = $db->execute($sql_member, [$user_id, $item_id]);
+            $existing = $db->fetchArray($execute_sql_member);
+            $member = $existing['EXISTS'];
+            if($member == 0) {
+                if ($current_user_id != $old_user_id) {
+                    $old_user_id = $current_user_id;
+                    $count_number = 1;
+                }
+                if ($count_number <= 3 and $old_user_id == $current_user_id) {
+                    if ($count_number == 1 and $first_record) {
+                        $count_number =2;
+                        $first_record = false;
+                    }
+                    if ($count_number == 1) {
+                        $col = "GROUP_1";
+                    } else if ($count_number == 2) {
+                        $col = "GROUP_2";
+                    } else if ($count_number == 3) {
+                        $col = "GROUP_3";
+                    }
+                    $execute_exists = $db->execute($sql_exists, [$user_id]);
+                    $does_exist = $db->fetchArray($execute_exists);
+                    $y = $does_exist['USER_ID'];
+                    if (!$y) {
+                        $sql_insert_into_recommendation1 = "INSERT INTO ".
+                            "$recommendation_table_threads ".
+                            "(USER_ID, $col, RECOMMEND_TYPE) VALUES (?, ?, ?)";
+                        $execute_insert_into_recommendation1 = $db->execute(
+                            $sql_insert_into_recommendation1,
+                            [$user_id, $item_id, 'THREAD']);
+                        $count_number = $count_number + 1;
+                        $old_user_id = $user_id;
+                    } else {
+                        $sql_update_into_recommendation = "UPDATE ".
+                            "$recommendation_table_threads SET ".
+                            "$col = ? , USER_ID = ? , RECOMMEND_TYPE = ?".
+                            " WHERE USER_ID = ? ";
+                        $execute_update_into_recommendation = $db->execute(
+                            $sql_update_into_recommendation,
+                            [$item_id, $user_id,'THREAD',$user_id ]);
+                        $count_number = $count_number + 1;
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Creating which threads a user is a part of and storing in table
+     * @param database connection $db
+     */
+    public function userThreadMap($db)
+    {
+        $sql_calculating_user_thread = "SELECT G.USER_ID ".
+            "AS USER_ID, ID AS ITEM_ID, G.GROUP_ID AS GROUP_ID ".
+            "FROM USER_GROUP UG INNER JOIN GROUP_ITEM G ON ".
+            "UG.GROUP_ID = G.GROUP_ID AND UG.USER_ID = G.USER_ID";
+        $execute_sql_calculating_user_thread =
+            $db->execute($sql_calculating_user_thread);
+        $sql_insert = "INSERT INTO USER_THREAD_GROUP VALUES (?, ?, ?)";
+        while($rows = $db->fetchArray($execute_sql_calculating_user_thread)) {
+            $user_id = $rows['USER_ID'];
+            $item_id = $rows['ITEM_ID'];
+            $group_id = $rows['GROUP_ID'];
+            $execute_sql_insert =
+                $db->execute($sql_insert, [$user_id, $item_id, $group_id]);
+        }
+    }
+    /**
+     * Groups Recommendation
+     * @param database connection $db
+     * @param STRING $recommendation_table_groups the table to be updated
+     */
+    public function recommendingGroups($db, $recommendation_table_groups)
+    {
+        $sql_aggregate_groups = "SELECT SUM(S.COSINE_SIMILARITY) AS ".
+            "RATING, S.USER_ID AS USER_ID, G.GROUP_ID AS GROUP_ID FROM ".
+            "GROUP_ITEM G INNER JOIN SIMILARITY S WHERE G.ID = S.ITEM_ID ".
+            "GROUP BY G.GROUP_ID, S.USER_ID ORDER BY S.USER_ID,RATING DESC";
+        $execute_sql_aggregate_groups = $db->execute($sql_aggregate_groups);
+        $first_record = $db->fetchArray($execute_sql_aggregate_groups);
+        $user_id= $first_record['USER_ID'];
+        $group_id = $first_record['GROUP_ID'];
+        $cosine_similarity = $first_record['RATING'];
+        $current_user_id = $user_id;
+        $old_user_id = $user_id;
+        $sql_member = "SELECT COUNT(*) as EXISTS FROM USER_THREAD_GROUP ".
+            "WHERE USER_ID = ? AND GROUP_ID = ? ";
+        $execute_sql_member = $db->execute($sql_member, [$user_id, $group_id]);
+        $existing = $db->fetchArray($execute_sql_member);
+        $member = $existing['EXISTS'];
+        $count_number = 1;
+        $first_record = false;
+        if($member == 0) {
+            $sql_insert_into_recommendation = "INSERT INTO ".
+                "$recommendation_table_groups VALUES (?, ?, ?, ?, ?)";
+            $execute_insert_into_recommendation = $db->execute(
+                $sql_insert_into_recommendation,
+                [$user_id, $group_id, 0, 0, 'GROUP']);
+            $sql_exists = "SELECT * FROM $recommendation_table_groups".
+                " WHERE USER_ID = ? ";
+            $count_number = 1;
+            $first_record = true;
+        }
+        while($rows = $db->fetchArray($execute_sql_aggregate_groups)) {
+            $user_id= $rows['USER_ID'];
+            $item_id = $rows['GROUP_ID'];
+            $cosine_similarity = $rows['RATING'];
+            $current_user_id = $user_id;
+            $execute_sql_member = $db->execute($sql_member, [$user_id, $item_id]);
+            $existing = $db->fetchArray($execute_sql_member);
+            $member = $existing['EXISTS'];
+            if($member == 0) {
+                if ($current_user_id != $old_user_id) {
+                    $old_user_id = $current_user_id;
+                    $count_number = 1;
+                }
+                if ($count_number <= 3 and $old_user_id == $current_user_id) {
+                    if ($count_number == 1 and $first_record) {
+                        $count_number =2;
+                        $first_record = false;
+                    }
+                    if ($count_number == 1) {
+                        $col = "GROUP_1";
+                    } else if ($count_number == 2) {
+                        $col = "GROUP_2";
+                    } else if ($count_number == 3) {
+                        $col = "GROUP_3";
+                    }
+                    $execute_exists = $db->execute($sql_exists, [$user_id]);
+                    $does_exist = $db->fetchArray($execute_exists);
+                    $y = $does_exist['USER_ID'];
+                    if (!$y) {
+                        $sql_insert_into_recommendation1 = "INSERT INTO ".
+                            "$recommendation_table_groups (USER_ID, $col, ".
+                            "RECOMMEND_TYPE) VALUES (?, ?, ?)";
+                        $execute_insert_into_recommendation1 = $db->execute(
+                            $sql_insert_into_recommendation1 ,
+                            [$user_id, $item_id, 'GROUP']);
+                        $count_number = $count_number + 1;
+                        $old_user_id = $user_id;
+                    } else {
+                        $sql_update_into_recommendation = "UPDATE ".
+                        "$recommendation_table_groups SET $col = ? , ".
+                        "USER_ID = ? ,RECOMMEND_TYPE = 'GROUP' WHERE USER_ID = ?";
+                        $execute_update_into_recommendation = $db->execute(
+                            $sql_update_into_recommendation,
+                            [$item_id, $user_id, $user_id]);
+                        $count_number = $count_number + 1;
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Deletes the temp tables for incrremental job
+     * @param database connection $db
+     * @param string $recommendation_table_threads table to be updated
+     * @param string $recommendation_table_groups table to be updated
+     */
+    public function incrementalDelete($db,
+        $recommendation_table_threads, $recommendation_table_groups)
+    {
+        /**
+            Delete the item_word_frequency table
+            Delete all rows from IDF_TEMP, WORDS_TEMP,
+            USER_WORD_FREQUENCY_TEMP, USER_WORD_WEIGHTS_TEMP,SIMILARITY_TEMP
+            RECOMMENDATION LIST tables before inserting
+        **/
+        $tables = array("IDF_TEMP", "USER_WORD_FREQUENCY_TEMP",
+            "USER_WORD_WEIGHTS_TEMP", "WORDS_TEMP", "SIMILARITY_TEMP",
+            $recommendation_table_threads, $recommendation_table_groups);
+        foreach ($tables as $value){
+            $sql_delete ="DELETE FROM $value";
+            $db->execute($sql_delete);
+        }
+    }
+    /**
+     * Computes the term frequency for newly added items and updates table
+     * @param database connection $db
+     * @param int $number_items number of items
+     * @param int $last_update last updated time
+     * @param int $count word_id of the last word added in words table
+     */
+    public function termFrequencyItemIncremental
+        ($db, $number_items, $last_update, $count)
+    {
+        $first_word = true;
+        $sql_new_threads_added =  "SELECT ID AS ITEM_ID, TITLE, DESCRIPTION ".
+            "FROM GROUP_ITEM WHERE PUBDATE > ? AND TITLE NOT LIKE '%wiki%'";
+        $execute_sql_new_threads =
+            $db->execute($sql_new_threads_added,
+            [$last_update]);
+        /*
+            Updating the item word frequencies
+            based on the newly added threads/groups
+        */
+        while ($item = $db->fetchArray($execute_sql_new_threads)) {
+            $recommend_id = $item['ITEM_ID'];
+            $record = $item['TITLE']." ".$item['DESCRIPTION'];
+            $word_frequency = $this->wordCount($record);
+            $sql_creating_words_list = "INSERT INTO ".
+                        "WORDS VALUES (?,'$key')";
+            $sql_insert_word_freq1 = "INSERT INTO ".
+                        "ITEM_WORD_FREQUENCY VALUES(?, ?, ?, ?)";
+            $sql_insert_IDF_items = "INSERT INTO IDF VALUES(?, ?, 1)";
+            $sql_insert_iww = "INSERT INTO ITEM_WORD_WEIGHTS VALUES(?, ?, ?)";
+            $sql_item_with_word ="SELECT COUNT(ITEM_ID) AS ".
+                        "COUNT_ITEMS_WITH_WORD FROM ".
+                        "ITEM_WORD_FREQUENCY WHERE WORD_ID = ? ";
+            $sql_update_IDF_items = "UPDATE IDF SET ITEM_IMP = ".
+                        "$idf_word_items WHERE WORD_ID = ? ";
+            foreach ($word_frequency as $key => $value) {
+                if ($first_word) {
+                    $first_word = false;
+                    $this->new_word_start = $count;
+                }
+                $sql_check_words_exists = "SELECT COUNT(*) AS COUNT, ".
+                    "WORD_ID AS WORD_ID FROM WORDS WHERE WORD LIKE '{$key}'";
+                $g = $db->execute($sql_check_words_exists);
+                $exists = $db->fetchArray($g);
+                $t=$exists['COUNT'];
+                $word_id = $exists['WORD_ID'];
+                if ($t == 0) {
+                    $insert = $db->execute($sql_creating_words_list, [$count]);
+                    /*
+                        Insert a new record IDF for items
+                    */
+                    $logValue = log($value,10) + 1;
+                    $inserted_row = $db->execute($sql_insert_word_freq1 ,
+                        [$recommend_id, $count, $value, $logValue]);
+                    $idf_word_items = log($number_items/1, 10);
+                    $execute_insert_IDF_items =
+                        $db->execute($sql_insert_IDF_items,
+                            [$count, $idf_word_items]);
+                    $weight = floatval($idf_word_items * $logValue);
+                    $execute_insert_iww= $db->execute($sql_insert_iww,
+                        [$count, $recommend_id, $weight]);
+                    $count = $count + 1;
+                    $this->word_count = $this->word_count + 1;
+                } else if ($t != 0) {
+                    $logValue = log($value,10) + 1;
+                    $inserted_row = $db->execute($sql_insert_word_freq1,
+                        [$recommend_id,$word_id,$value,$logValue]);
+                    $execute_sql_item_with_word =
+                        $db->execute($sql_item_with_word, [$word_id]);
+                    $fetched_sql_item_with_word =
+                        $db->fetchArray($execute_sql_item_with_word);
+                    $number_items_with_word =
+                        $fetched_sql_item_with_word['COUNT_ITEMS_WITH_WORD'];
+                    $idf_word_items =
+                        log($number_items/$number_items_with_word, 10);
+                    $execute_update_IDF_items =
+                        $db->execute($sql_update_IDF_items, [$word_id]);
+                    $weight = floatval($idf_word_items * $logValue);
+                    $execute_insert_iww =
+                        $db->execute($sql_insert_iww,
+                            [$word_id, $recommend_id, $weight]);
+                }
+            }
+        }
+    }
+    /**
+     * Computes the term frequency for users and items for newly
+     * updated word impressions
+     * @param database connection $db
+     * @param int $last_update last updated time
+     * @param int $current current time
+     */
+    public function termFrequencyUserIncremental($db, $last_update, $current)
+    {
+        $sql_new_users_added = "SELECT COUNT(*) AS ".
+            "COUNT FROM USERS WHERE CREATION_TIME > ? ";
+        $sql_user_word_count_update = "SELECT COUNT(USER_ID) ".
+                "AS FREQUENCY, USER_ID AS UID, WORD_ID AS WID, ".
+                "IWF.ITEM_ID FROM ITEM_WORD_FREQUENCY IWF INNER JOIN".
+                " ITEM_IMPRESSION II ON IWF.ITEM_ID = II.ITEM_ID ".
+                "WHERE II.VIEW_DATE > ? AND II.VIEW_DATE <= ? ".
+                "GROUP BY USER_ID, WORD_ID";
+        $sql_insert_word_freq2 = "INSERT INTO USER_WORD_FREQUENCY_TEMP".
+                " VALUES(?, ?, ?, ?)";
+        $execute_new_users =
+            $db->execute($sql_new_users_added, [$last_update]);
+        $new_users = $db->fetchArray($execute_new_users);
+        $result_user_word_count_update =
+            $db->execute($sql_user_word_count_update,
+                [$last_update, $current]);
+        while($rows_fetched =
+            $db->fetchArray($result_user_word_count_update)) {
+            $frequency = $rows_fetched['FREQUENCY'];
+            $user_id = $rows_fetched['UID'];
+            $word_id = $rows_fetched['WID'];
+            $logValue = log($frequency,10) + 1;
+            $inserted1 = $db->execute($sql_insert_word_freq2,
+                [$frequency, $user_id, $word_id, $logValue]);
+            }
+    }
+    /**
+     * Stores the newly seen words for users in a table
+     * @param database connection $db
+     */
+    public function updatedWordListIncremental($db)
+    {
+        $sql_word = "SELECT DISTINCT WORD_ID ".
+            "FROM USER_WORD_FREQUENCY_TEMP";
+        $result_word = $db->execute($sql_word);
+        while($list = $db->fetchArray($result_word)) {
+            $word_id = $list['WORD_ID'];
+            $sql_word_insert = "INSERT INTO WORDS_TEMP VALUES(?)";
+            $execute_words_list = $db->execute($sql_word_insert, [$word_id]);
+        }
+    }
+    /**
+     * Computes the idf for users and items for newly updated word impressions
+     * @param database connection $db
+     * @param int $number_items number of items
+     * @param int $number_distinct_users number of distinct users in
+     * updated impressions
+     */
+    public function idfIncremental($db, $number_items, $number_distinct_users)
+    {
+        $sql_words = "SELECT WORD_ID AS WID FROM WORDS_TEMP";
+        $result_list = $db->execute($sql_words);
+        $sql_item_with_word ="SELECT COUNT(DISTINCT ITEM_ID) AS ".
+            "COUNT_ITEMS_WITH_WORD FROM ".
+            "ITEM_WORD_FREQUENCY WHERE WORD_ID = ? ";
+        $sql_user_with_word ="SELECT COUNT(DISTINCT USER_ID) AS ".
+            "COUNT_USERS_WITH_WORD FROM USER_WORD_FREQUENCY_TEMP".
+            " WHERE WORD_ID = ? ";
+        $sql_insert_idf = "INSERT INTO ".
+            "IDF_TEMP VALUES (?, ?, ?)";
+        $number_distinct_users = $number_distinct_users + 1;
+        while($list = $db->fetchArray($result_list)) {
+            $word_id= $list['WID'];
+            /*
+                ID for items with new words seen
+            */
+            $execute_sql_item_with_word =
+                $db->execute($sql_item_with_word, [$word_id]);
+            $fetched_sql_item_with_word =
+                $db->fetchArray($execute_sql_item_with_word);
+            $number_items_with_word =
+                $fetched_sql_item_with_word['COUNT_ITEMS_WITH_WORD'];
+            $idf_word_item =
+                log($number_items/$number_items_with_word, 10);
+            /*
+                IDF for users with new words seen
+            */
+            $execute_sql_user_with_word =
+                $db->execute($sql_user_with_word, [$word_id]);
+            $fetched_sql_user_with_word =
+                $db->fetchArray($execute_sql_user_with_word);
+            $number_users_with_word =
+                $fetched_sql_user_with_word['COUNT_USERS_WITH_WORD'];
+            $idf_word_user =
+                log($number_distinct_users/$number_users_with_word, 10);
+            /*
+                Inserting the inverse document frequency value
+            */
+            $execute_insert_idf =
+                $db->execute($sql_insert_idf,
+                    [$word_id, $idf_word_item, $idf_word_user]);
+        }
+    }
+    /**
+     * Computes the user weights for words and updates user word weights
+     * @param database connection $db
+     */
+    public function tfidfusersIncremental($db, $old_factor, $new_factor)
+    {
+        $sql_weights_users_new = "SELECT IDF_TEMP.USER_IMP* UWF.LOGFREQUENCY".
+            " AS WEIGHT, UWF.WORD_ID AS WORD_ID, UWF.USER_ID AS USER_ID FROM".
+            "  IDF_TEMP JOIN USER_WORD_FREQUENCY_TEMP AS UWF ".
+            "WHERE IDF_TEMP.WORD_ID = UWF.WORD_ID";
+        $execute_weights_users_new = $db->execute($sql_weights_users_new);
+        $sql_insert_uww1 = "UPDATE USER_WORD_WEIGHTS SET ".
+            "WEIGHT = $final_weight WHERE WORD_ID = ? ".
+            "AND USER_ID = ? ";
+        $sql_insert_uwwt = "INSERT INTO USER_WORD_WEIGHTS_TEMP ".
+            "VALUES (?, ?, ?)";
+        $sql_insert_uww = "INSERT INTO ".
+            "USER_WORD_WEIGHTS VALUES (?, ?, ?)";
+        while($list_new = $db->fetchArray($execute_weights_users_new)) {
+            $weight = $list_new['WEIGHT'];
+            $word_id = $list_new['WORD_ID'];
+            $user_id = $list_new['USER_ID'];
+            $sql_check_uww_new = "SELECT USER_ID, WORD_ID, WEIGHT, ".
+                "COUNT(USER_ID) AS NUM FROM USER_WORD_WEIGHTS ".
+                "WHERE USER_ID = $user_id and WORD_ID = $word_id";
+            $execute_check_uww_new = $db->execute($sql_check_uww_new);
+            $old_list = $db->fetchArray($execute_check_uww_new);
+            $check = $old_list['USER_ID'];
+            $counter = $old_list['NUM'];
+            $old_weight = $old_list['WEIGHT'];
+            $final_weight = 0;
+            if ($counter == 0) {
+                $final_weight = $weight;
+                $execute_insert_uww = $db->execute($sql_insert_uww,
+                    [$word_id, $user_id, $weight]);
+            } else {
+                /*
+                calculate the  weights depending on new users
+                */
+                $final_weight = floatval(floatval($old_factor *
+                $old_weight) + floatval($new_factor * $weight));
+                $execute_insert_uww =
+                $db->execute($sql_insert_uww1, [$word_id, $user_id]);
+            }
+
+            $execute_insert_uww = $db->execute($sql_insert_uwwt,
+                [$word_id, $user_id, $final_weight]);
+        }
+    }
+    /**
+     * Computes the similarity and stores in the table similarity
+     * @param database connection $db
+     */
+    public function computeSimilarityIncremental($db)
+    {
+        $sql_similarity_temp ="DELETE FROM SIMILARITY_TEMP";
+        $db->execute($sql_similarity_temp);
+        $sql_weights = "SELECT SUM(UWW.WEIGHT*IWW.WEIGHT) AS ".
+            "DOT_PRODUCT, SUM(UWW.WEIGHT) AS USER_MAG, SUM(IWW.WEIGHT)".
+            " AS ITEM_MAG, UWW.USER_ID AS USER_ID, IWW.ITEM_ID AS ITEM_ID ".
+            "FROM ITEM_WORD_WEIGHTS IWW INNER JOIN ".
+            "USER_WORD_WEIGHTS UWW GROUP BY USER_ID, ITEM_ID";
+        $result_news = $db->execute($sql_weights);
+        $sql_insert_into_recommendation = "INSERT ".
+            "INTO SIMILARITY_TEMP VALUES (?, ?, ?)";
+        while ($rows = $db->fetchArray($result_news)) {
+                $dot_product= $rows['DOT_PRODUCT'];
+                $user_magnitude = $rows['USER_MAG'];
+                $item_magnitude = $rows['ITEM_MAG'];
+                $item_id = $rows['ITEM_ID'];
+                $user_id = $rows['USER_ID'];
+                $cosine_similarity =
+                    floatval($dot_product)/floatval($user_magnitude
+                        *$item_magnitude);
+
+                $execute_insert_into_recommendation =
+                $db->execute($sql_insert_into_recommendation,
+                    [$user_id, $item_id, $cosine_similarity]);
+        }
+    }
+    /**
+     * Counts the number of new users
+     * @param database connection $db
+     * @param int $last_update last time updated
+     * @return number of new users
+     */
+    public function countNewUsers($db, $last_update)
+    {
+        $sql_new_users = "SELECT count(*) AS NUMBER FROM USERS ".
+                    "WHERE CREATION_TIME > ? ";
+        $result_number_new_users = $db->execute($sql_new_users,
+            [$last_update]);
+        $number_new_users = 0;
+        $rows = $db->fetchArray($result_number_new_users);
+        $number_new_users = $rows['NUMBER'];
+        return $number_new_users;
+    }
+    /**
+     * Counts the number of old users
+     * @param database connection $db
+     * @param int $update_time current update time
+     * @return number of old users
+     */
+    public function countOldUsers($db, $update_time)
+    {
+        $sql_old_users = "SELECT count(*) AS NUMBER FROM USERS ".
+            "WHERE CREATION_TIME <= ? ";
+        $result_number_old_users = $db->execute($sql_old_users,
+            [$update_time]);
+        $number_old_users = 0;
+        $rows = $db->fetchArray($result_number_old_users);
+        $number_old_users = $rows['NUMBER'];
+        return $number_old_users;
+
+    }
+    /**
+     * Counts the number of distinct users
+     * @param database connection $db
+     * @return int $number_distinct_users number of distinct users
+     */
+    public function countDistinctUsers($db)
+    {
+        $sql_number = "SELECT COUNT(DISTINCT USER_ID) ".
+            "AS NUMBER FROM USER_WORD_FREQUENCY_TEMP";
+        $result_number = $db->execute($sql_number);
+        $list_count = $db->fetchArray($result_number);
+        $number_distinct_users = $list_count['NUMBER'];
+        return $number_distinct_users;
+    }
+    /**
+     * Deleting the  recommendation tables for updation
+     * @param database connection $db
+     * @param string $recommendation_trending_threads_table 
+     * deletes rows from the updating table
+     */
+    public function deleteTables($db ,$recommendation_trending_threads_table)
+    {
+        $sql = "delete from $recommendation_trending_threads_table";
+        $result = $db->execute($sql);
+    }
+    /**
+     * Drops the user_bias, item_bias and predicted_table
+     * to populate with new values
+     * @param database connection $db
+     */
+    public function dropTables($db)
+    {
+        $tables = array("USER_BIAS", "ITEM_BIAS", "PREDICTED_TABLE");
+        foreach ($tables as $value) {
+            $sql = "DROP TABLE $value";
+            $result = $db->execute($sql);
+        }
+    }
+    /**
+     * Calculating the user bias
+     *
+     * @param database connection $db
+     * @param the average rating table name to be updated
+     *
+     * @return float which gives average rating
+     * Average Rating is the total number of views / total number of distinct items
+     */
+    public function averageRating($db, $average_table)
+    {
+        $average_rating = 0;
+        $sql_update_average_rating = "UPDATE $average_table ".
+        "SET AVG_RATING = (SELECT CAST (COUNT(*)AS FLOAT) ".
+        "/ CAST (COUNT( DISTINCT ITEM_ID) AS FLOAT) FROM ITEM_IMPRESSION)";
+        $result_average_rating = $db->execute($sql_update_average_rating);
+        $sql_retrieve_average_rating = "SELECT AVG_RATING as ".
+            "AVERAGE FROM $average_table";
+        $result_set = $db->execute($sql_retrieve_average_rating);
+        while ($item = $db->fetchArray($result_set)) {
+        $average_rating = $item['AVERAGE'];
+        }
+        return $average_rating;
+
+    }
+    /**
+     * Calculating the user bias
+     *
+     * @param resource& $db database handle where Yioop database stored
+     * @param the user bias table name to be updated
+     */
+    public function userBias($db, $user_bias_table)
+    {
+        $sql_calculating_user_bias = "CREATE TABLE $user_bias_table ".
+            "AS SELECT USER_ID, CAST(COUNT(*) ".
+            "AS FLOAT)/CAST(COUNT(DISTINCT ITEM_ID) AS FLOAT)".
+            " AS USER_BIAS FROM ITEM_IMPRESSION WHERE ITEM_ID != '' ".
+            "GROUP BY USER_ID";
+        $user_bias_calculated = $db->execute($sql_calculating_user_bias);
+
+    }
+    /**
+     * Calculating the item bias
+     *
+     * @param resource& $db database handle where Yioop database stored
+     * @param the item bias table name to be updated
+     */
+    public function itemBias($db, $item_bias_table)
+    {
+        $sql_calculating_item_bias = "CREATE TABLE $item_bias_table ".
+            "AS SELECT ITEM_ID, CAST(COUNT(*) ".
+            "AS FLOAT)/CAST(COUNT(DISTINCT USER_ID) AS FLOAT)".
+            " AS ITEM_BIAS FROM ITEM_IMPRESSION ".
+            "GROUP BY ITEM_ID HAVING ITEM_ID != ''";
+        $item_bias_calculated = $db->execute($sql_calculating_item_bias);
+
+    }
+    /**
+     * Calculating the item bias
+     *
+     * @param resource& $db database handle where Yioop database stored
+     * @param the predict rating table name to be updated
+     */
+    public function predictRating($db, $predicted_table,
+        $item_bias_table, $user_bias_table)
+    {
+        $sql_create_prediction = "CREATE TABLE $predicted_table ".
+            "AS SELECT USER_ID, ITEM_ID, USER_BIAS, ITEM_BIAS, ".
+            "ITEM_BIAS + USER_BIAS + (SELECT AVG_RATING FROM AVERAGE_RATING)".
+            " AS RATING FROM $item_bias_table JOIN $user_bias_table";
+        $prediction_table_created = $db->execute($sql_create_prediction);
+
+    }
+    /**
+     * Recommending the top 3 threads which a user might like
+     */
+    public function recommendingTrendingThreads($db,
+        $recommendation_trending_threads_table, $predicted_table)
+    {
+        L\crawlLog("Building the recommendation ".
+            "table for trending threads");
+        $predictions = "SELECT USER_ID, ITEM_ID ".
+            "FROM $predicted_table ORDER BY USER_ID ASC, RATING DESC";
+        $result_set = $db->execute($predictions);
+        $first_record = $db->fetchArray($result_set);
+        $user_id = $first_record['USER_ID'];
+        $item_id = $first_record['ITEM_ID'];
+        $current_user_id = $user_id;
+        $old_user_id = $user_id;
+        $sql_not_wiki = "SELECT COUNT(*) AS COUNT, GROUP_ID ".
+            "AS GROUP_ID FROM GROUP_ITEM WHERE".
+            " TITLE NOT LIKE '%WIKI%' AND ID = ? ";
+        $result_set_wiki = $db->execute($sql_not_wiki, [$item_id]);
+        $record = $db->fetchArray($result_set_wiki);
+        $not_wiki = $record['COUNT'];
+        $group_id = $record['GROUP_ID'];
+        $sql_access = "SELECT REGISTER_TYPE AS REGISTER ".
+            "FROM GROUPS WHERE GROUP_ID = ? ";
+        $result_sql_access = $db->execute($sql_access, [$group_id]);
+        $accessible_res = $db->fetchArray($result_sql_access);
+        $accessible = $accessible_res['REGISTER'];
+        $sql_already_member = "SELECT COUNT(*) as EXISTS FROM USER_THREAD_GROUP ".
+            "WHERE USER_ID = ? AND ITEM_ID = ? ";
+        $result_sql_already_member = $db->execute($sql_already_member,
+            [$user_id, $item_id]);
+        $already_member = $db->fetchArray($result_sql_already_member);
+        $ignore = $already_member['MEMBER'];
+        if ($not_wiki != 0 AND
+            ($accessible == 3 OR $accessible == 4) AND $ignore == 0)
+        {
+            $sql_insert_into_recommendation = "INSERT INTO ".
+                "$recommendation_trending_threads_table ".
+                "VALUES ($user_id, $item_id,0, 0, 'THREAD')";
+            $execute_insert_into_recommendation =
+                $db->execute($sql_insert_into_recommendation);
+            $count_number = 1;
+            $first_record = true;
+        }
+        while ($rows = $db->fetchArray($result_set)) {
+            $user_id= $rows['USER_ID'];
+            $item_id = $rows['ITEM_ID'];
+            $result_set_wiki = $db->execute($sql_not_wiki, [$item_id]);
+            $record = $db->fetchArray($result_set_wiki);
+            $group_id = $record['GROUP_ID'];
+            $not_wiki = $record['COUNT'];
+            $result_sql_access = $db->execute($sql_access, [$group_id]);
+            $accessible_res = $db->fetchArray($result_sql_access);
+            $accessible = $accessible_res['REGISTER'];
+            $result_sql_already_member = $db->execute($sql_already_member,
+                [$user_id, $item_id]);
+            $already_member = $db->fetchArray($result_sql_already_member);
+            $ignore = $already_member['MEMBER'];
+            if ($not_wiki != 0 AND
+                ($accessible == 3 OR $accessible == 4) AND $ignore == 0) {
+                $current_user_id = $user_id;
+                if ($current_user_id != $old_user_id) {
+                    $old_user_id = $current_user_id;
+                    $count_number = 1;
+                }
+                if ($count_number <= 3 and $old_user_id == $current_user_id) {
+                    if ($count_number == 1 and $first_record) {
+                            $count_number =2;
+                            $first_record = false;
+                    }
+                    if ($first_record) {
+                        $first_record = false;
+                        $col = "GROUP_1";
+                    }else if ($count_number == 1) {
+                        $col = "GROUP_1";
+                    } else if ($count_number == 2) {
+                        $col = "GROUP_2";
+                    } else if ($count_number == 3) {
+                        $col = "GROUP_3";
+                    }
+                $sql_exists = "SELECT * FROM ".
+                    "$recommendation_trending_threads_table ".
+                    "WHERE USER_ID = $user_id";
+                $execute_exists = $db->execute($sql_exists);
+                if (!$db->fetchArray($execute_exists)) {
+                    $sql_insert_into_recommendation = "INSERT INTO ".
+                        "$recommendation_trending_threads_table ".
+                        "(USER_ID, $col, RECOMMEND_TYPE) VALUES ".
+                        "($user_id, $item_id, 'THREAD')";
+                    $execute_insert_into_recommendation =
+                        $db->execute($sql_insert_into_recommendation);
+                    $count_number = $count_number + 1;
+                    $old_user_id = $user_id;
+                } else {
+                    $sql_update_into_recommendation = "UPDATE ".
+                        "$recommendation_trending_threads_table ".
+                        "SET $col = $item_id, USER_ID = $user_id, ".
+                        "RECOMMEND_TYPE = 'THREAD' WHERE USER_ID = $user_id";
+                    $execute_update_into_recommendation =
+                        $db->execute($sql_update_into_recommendation);
+                    $count_number = $count_number + 1;
+                }
+                }
+            }
+        }
+    }
+    /* Updating Live table
+     * @param database connection $db
+     * @param int $updating the latest table updated
+     */
+    public function updateLiveTableTrending($db, $updating)
+    {
+        $sql_live ="UPDATE LIVE_TABLE SET TRENDING = ? ";
+        $db->execute($sql_live, [$updating]);
+    }
+    /**
+     * Trending Jobs
+     *
+     */
+    public function trendingJob($db,$updating)
+    {
+        L\crawlLog("Inside Trending Threads Recommendation job");
+        $live = $this->live_table;
+        $updating = 1;
+        if ($live == 1) {
+            $updating = 2;
+        }
+        $user_bias_table = "USER_BIAS";
+        $item_bias_table = "ITEM_BIAS";
+        $predicted_table = "PREDICTED_TABLE";
+        /*
+            Updating the recommendation trending update table
+        */
+        $recommendation_trending_threads_table = "RECOMMENDATION_".
+            "TRENDING_THREADS_".$updating;
+        L\crawlLog("Currently the tables being altered: $user_bias_table, ".
+            "$item_bias_table, $recommendation_trending_threads_table");
+        /*
+            Connecting to the database
+        */
+        $db = $this->db;
+        $drop_table = $this->dropTables($db);
+        $delete_table = $this->
+            deleteTables($db, $recommendation_trending_threads_table);
+        /*
+            Calculating average rating, user bias and item_bias
+            Building the predicted table and making recommendations
+        */
+        $average = $this->averageRating($db, 'AVERAGE_RATING');
+        $this->userBias($db, $user_bias_table);
+        $this->itemBias($db, $item_bias_table);
+        $this->predictRating($db, $predicted_table,
+            $item_bias_table ,$user_bias_table);
+        $recommend_trending_threads = $this->recommendingTrendingThreads($db,
+            $recommendation_trending_threads_table, $predicted_table);
+        $this->live_table = $updating;
+        $update_live_table = $this->updateLiveTableTrending($db, $updating);
+        $this->doTasks([]);
+    }
+    /**
+     *
+     * @param array $tasks array of news feed info (url to download, paths to
+     *  extract etc)
+     */
+    public function doTasks($tasks)
+    {
+    }
+} 
\ No newline at end of file
diff --git a/src/models/ProfileModel.php b/src/models/ProfileModel.php
index 20676485..b87841a7 100755
--- a/src/models/ProfileModel.php
+++ b/src/models/ProfileModel.php
@@ -121,6 +121,7 @@ class ProfileModel extends Model
         $this->create_statements = ["ACTIVE_PROCESS" =>
             "CREATE TABLE ACTIVE_PROCESS (NAME VARCHAR(".C\NAME_LEN.
                 "), ID INTEGER, TYPE VARCHAR(" . C\NAME_LEN . "))",
+            "AVERAGE_RATING" => "CREATE TABLE AVERAGE_RATING (AVG_RATING INT)",    
             "ADVERTISEMENT" => "CREATE TABLE ADVERTISEMENT (ID
                 $serial PRIMARY KEY $auto_increment, USER_ID INTEGER,
                 NAME VARCHAR(". C\ADVERTISEMENT_NAME_LEN ."),
@@ -258,6 +259,19 @@ class ProfileModel extends Model
                 TMP_NUM_VIEWS INTEGER DEFAULT -1,
                 PRIMARY KEY(USER_ID, ITEM_ID, ITEM_TYPE,
                 UPDATE_PERIOD, UPDATE_TIMESTAMP))",
+            "IDF"=> "CREATE TABLE IDF (WORD_ID INT, ITEM_IMP FLOAT,
+                USER_IMP FLOAT)",
+            "IDF_TEMP"=> "CREATE TABLE IDF_TEMP (WORD_ID INT, ITEM_IMP
+                FLOAT, USER_IMP FLOAT)",
+            "ITEM_BIAS"=> "CREATE TABLE ITEM_BIAS(ITEM_ID INT,
+                ITEM_BIAS FLOAT)",
+            "ITEM_WORD_FREQUENCY"=> "CREATE TABLE ITEM_WORD_FREQUENCY 
+                (ITEM_ID INT,
+                WORD_ID INT, FREQUENCY INT, LOGFREQUENCY FLOAT)",
+            "ITEM_WORD_WEIGHTS"=> "CREATE TABLE ITEM_WORD_WEIGHTS(WORD_ID INT,
+                ITEM_ID INT ,WEIGHT FLOAT)",
+            "LIVE_TABLE"=> "CREATE TABLE LIVE_TABLE (TRENDING INT,
+                THREADS INT, GROUPS INT)",        
             "LOCALE" => "CREATE TABLE LOCALE(LOCALE_ID $serial PRIMARY KEY
                 $auto_increment, LOCALE_TAG VARCHAR(" . C\NAME_LEN . "),
                 LOCALE_NAME VARCHAR(" . C\LONG_NAME_LEN .
@@ -290,12 +304,38 @@ class ProfileModel extends Model
             "PAGE_RELATIONSHIPS" => "CREATE TABLE PAGE_RELATIONSHIP (
                 ID $serial PRIMARY KEY $auto_increment, NAME VARCHAR(" .
                 C\NAME_LEN . ") UNIQUE)",
+            "PREDICTED_TABLE"=> "CREATE TABLE PREDICTED_TABLE(USER_ID INT,
+                ITEM_ID INT, USER_BIAS FLOAT, ITEM_BIAS FLOAT, RATING FLOAT)",    
             "QUERY_ITEM" => "CREATE TABLE QUERY_ITEM (ID $serial PRIMARY KEY
                 $auto_increment, QUERY_HASH CHAR(". C\TIMESTAMP_LEN .
                 ") UNIQUE, QUERY VARCHAR(" . C\MAX_QUERY_LEN."),
                 CREATION NUMERIC(" . C\TIMESTAMP_LEN . "))",
             "QI_GROUP_ID_INDEX" => "CREATE INDEX QI_QUERY_HASH_INDEX ON
                 QUERY_ITEM (QUERY_HASH)",
+            "RECOMMENDATION_LIST_GROUPS_1"=>"CREATE TABLE 
+                RECOMMENDATION_LIST_GROUPS_1
+                (USER_ID INT, GROUP_1 INT, GROUP_2
+                INT, GROUP_3 INT, RECOMMEND_TYPE VARCHAR)",
+            "RECOMMENDATION_LIST_GROUPS_2"=>"CREATE TABLE 
+                RECOMMENDATION_LIST_GROUPS_2 
+                (USER_ID INT, GROUP_1 INT, GROUP_2
+                INT, GROUP_3 INT, RECOMMEND_TYPE VARCHAR)",
+            "RECOMMENDATION_LIST_THREADS_1"=>"CREATE TABLE 
+                RECOMMENDATION_LIST_THREADS_1 
+                (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+                RECOMMEND_TYPE VARCHAR)",
+            "RECOMMENDATION_LIST_THREADS_2" => "CREATE TABLE 
+                RECOMMENDATION_LIST_THREADS_2 
+                (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+                RECOMMEND_TYPE VARCHAR)",
+            "RECOMMENDATION_TRENDING_THREADS_1"=> "CREATE TABLE 
+                RECOMMENDATION_TRENDING_THREADS_1
+                (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+                RECOMMEND_TYPE VARCHAR)",
+            "RECOMMENDATION_TRENDING_THREADS_2"=> "CREATE TABLE 
+                RECOMMENDATION_TRENDING_THREADS_2
+                (USER_ID INT, GROUP_1 INT, GROUP_2 INT, GROUP_3 INT, 
+                RECOMMEND_TYPE VARCHAR)",    
             "ROLE" => "CREATE TABLE ROLE (
                 ROLE_ID $serial PRIMARY KEY $auto_increment, NAME VARCHAR(".
                 C\NAME_LEN."))",
@@ -307,6 +347,10 @@ class ProfileModel extends Model
                 C\TITLE_LEN . "), SIGNATURE VARCHAR("
                 . C\TITLE_LEN . ")," .
                 "SCRAPE_RULES VARCHAR(" . C\TITLE_LEN . "))",
+            "SIMILARITY" => "CREATE TABLE SIMILARITY 
+                (USER_ID INT, ITEM_ID INT, COSINE_SIMILARITY FLOAT)",
+            "SIMILARITY_TEMP"=> "CREATE TABLE SIMILARITY_TEMP 
+                (USER_ID INT, ITEM_ID INT, COSINE_SIMILARITY FLOAT)",   
             "SUBSEARCH" => "CREATE TABLE SUBSEARCH (
                 LOCALE_STRING VARCHAR(" . C\LONG_NAME_LEN . ") PRIMARY KEY,
                 FOLDER_NAME VARCHAR(" . C\NAME_LEN."),
@@ -345,6 +389,21 @@ class ProfileModel extends Model
             "USER_SESSION" => "CREATE TABLE USER_SESSION(
                 USER_ID INTEGER PRIMARY KEY, SESSION VARCHAR(".
                 C\MAX_GROUP_POST_LEN . "))",
+            "USER_BIAS"=>"CREATE TABLE USER_BIAS 
+                (USER_ID INT,USER_BIAS FLOAT)",
+            "USER_WORD_FREQUENCY"=>"CREATE TABLE USER_WORD_FREQUENCY 
+                (FREQUENCY INT, USER_ID INT, WORD_ID INT, 
+                LOGFREQUENCY FLOAT)",
+            "USER_WORD_FREQUENCY_TEMP"=>"CREATE TABLE 
+                USER_WORD_FREQUENCY_TEMP
+                (FREQUENCY INT, USER_ID INT, WORD_ID INT, 
+                LOGFREQUENCY FLOAT)",
+            "USER_WORD_WEIGHTS"=>"CREATE TABLE USER_WORD_WEIGHTS 
+                (WORD_ID INT, USER_ID INT, WEIGHT FLOAT)",
+            "USER_WORD_WEIGHTS_TEMP"=>"CREATE TABLE USER_WORD_WEIGHTS_TEMP 
+                (WORD_ID INT, USER_ID INT, WEIGHT FLOAT)",
+            "USER_THREAD_GROUP" => "CREATE TABLE USER_THREAD_GROUP 
+                (USER_ID INT, ITEM_ID INT, GROUP_ID INT)",
             "VISITOR" => "CREATE TABLE VISITOR(ADDRESS VARCHAR(".
                 C\MAX_IP_ADDRESS_AS_STRING_LEN . "),
                 PAGE_NAME VARCHAR(" . C\NAME_LEN . "),
@@ -352,6 +411,9 @@ class ProfileModel extends Model
                 ACCESS_COUNT INTEGER,
                 PRIMARY KEY(ADDRESS, PAGE_NAME))",
             "VERSION" => "CREATE TABLE VERSION(ID INTEGER PRIMARY KEY)",
+            "WORDS"=> "CREATE TABLE WORDS 
+                (WORD_ID INT, WORD VARCHAR)",
+            "WORDS_TEMP"=> "CREATE TABLE WORDS_TEMP (WORD_ID INT)",
             ];
     }
     /**
diff --git a/src/models/UserModel.php b/src/models/UserModel.php
index 296838f9..4c12f019 100755
--- a/src/models/UserModel.php
+++ b/src/models/UserModel.php
@@ -564,4 +564,134 @@ class UserModel extends Model
             $this->db->execute($sql, $params);
         }
     }
+    /**
+     * Rendering the threads from the RECOMMENDATION_LIST_THREADS LIVE TABLE
+     * Table built using TF -IDF
+     * @param INT $user user id to render the recommendations for a user
+     */
+    public function getRecommendationsThreads($user_id)
+    {
+        $db = $this->db;
+        $sql_table_name = "SELECT THREADS AS TABLE_NAME FROM LIVE_TABLE ";
+        $exec_table = $db->execute($sql_table_name);
+        $table = $db->fetchArray($exec_table);
+        $table_name = $table['TABLE_NAME'];
+        $full_table_name = "RECOMMENDATION_LIST_THREADS_".$table_name;
+        $sql_thread = "SELECT * FROM $full_table_name ".
+            "WHERE USER_ID = ? ";
+        $user = (int)$user_id;
+        $result = $db->execute($sql_thread, [$user]);
+        $list = $db->fetchArray($result);
+        $thread_1_id = $list['GROUP_1'];
+        $thread_2_id = $list['GROUP_2'];
+        $thread_3_id = $list['GROUP_3'];
+        $sql_1 = "SELECT TITLE FROM GROUP_ITEM ".
+            "WHERE ID = ? " . $db->limitOffset(1);
+        $result_1 = $db->execute($sql_1, [$thread_1_id]);
+        $execute_1 = $db->fetchArray($result_1);
+        $thread_1 = $execute_1['TITLE'];
+        $sql_2 = "SELECT TITLE FROM GROUP_ITEM ".
+            "WHERE ID = ? " . $db->limitOffset(1);
+        $result_2 = $db->execute($sql_1, [$thread_2_id]);
+        $execute_2 = $db->fetchArray($result_2);
+        $thread_2 = $execute_2['TITLE'];
+        $sql_3 = "SELECT TITLE FROM GROUP_ITEM ".
+            "WHERE ID = ? " . $db->limitOffset(1);
+        $result_3 = $db->execute($sql_3, [$thread_3_id]);
+        $execute_3 = $db->fetchArray($result_3);
+        $thread_3 = $execute_3['TITLE'];
+        $threads = array(
+            $thread_1_id=>$thread_1, 
+            $thread_2_id=>$thread_2, 
+            $thread_3_id=>$thread_3,
+        );
+        return $threads;
+    }
+    /**
+     * Rendering the groups from the RECOMMENDATION_LIST_GROUPS LIVE TABLE
+     * Table built using TF -IDF
+     * @param INT $user user id to render the recommendations for a user
+     */
+    public function getRecommendationsGroups($user_id)
+    {
+        $db = $this->db;
+        $sql_table_name = "SELECT GROUPS AS TABLE_NAME FROM LIVE_TABLE";
+        $exec_table = $db->execute($sql_table_name);
+        $table = $db->fetchArray($exec_table);
+        $table_name = $table['TABLE_NAME'];
+        $full_table_name = "RECOMMENDATION_LIST_GROUPS_".$table_name;
+        $sql1 = "SELECT * FROM $full_table_name ".
+            "WHERE USER_ID = ? " . $db->limitOffset(1);
+        $user = (int)$user_id;
+        $result = $db->execute($sql1, [$user]);
+        $list = $db->fetchArray($result);
+        $thread_1_id = $list['GROUP_1'];
+        $thread_2_id = $list['GROUP_2'];
+        $thread_3_id = $list['GROUP_3'];
+        $sql_1 = "SELECT GROUP_NAME FROM GROUPS ".
+            "WHERE GROUP_ID = ? " . $db->limitOffset(1);
+        $result_1 = $db->execute($sql_1, [$thread_1_id]);
+        $execute_1 = $db->fetchArray($result_1);
+        $thread_1 = $execute_1['GROUP_NAME'];
+        $sql_2 = "SELECT GROUP_NAME FROM GROUPS ".
+            "WHERE GROUP_ID = ? " . $db->limitOffset(1);
+        $result_2 = $db->execute($sql_1, [$thread_2_id]);
+        $execute_2 = $db->fetchArray($result_2);
+        $thread_2 = $execute_2['GROUP_NAME'];
+        $sql_3 = "SELECT GROUP_NAME FROM GROUPS ".
+            "WHERE GROUP_ID = ? " . $db->limitOffset(1);
+        $result_3 = $db->execute($sql_3, [$thread_3_id]);
+        $execute_3 = $db->fetchArray($result_3);
+        $thread_3 = $execute_3['GROUP_NAME'];
+        $threads = array(
+            $thread_1_id  => $thread_1,
+            $thread_2_id  => $thread_2,
+            $thread_3_id  => $thread_3,
+        );
+        return $threads;
+    }
+    /**
+     * Rendering the trending threads from the 
+     * RECOMMENDATION_TRENDING_THREADS LIVE TABLE
+     * Table built using classic baseline predictor
+     * @param INT $user user id to render the recommendations for a user
+     */
+    public function getRecommendationsTrendingGroups($user_id)
+    {
+        $db = $this->db;
+        $sql_table_name = "SELECT THREADS AS TABLE_NAME FROM LIVE_TABLE";
+        $exec_table = $db->execute($sql_table_name);
+        $table = $db->fetchArray($exec_table);
+        $table_name = $table['TABLE_NAME'];
+        $full_table_name = "RECOMMENDATION_TRENDING_THREADS_".$table_name;
+        $sql2 = "SELECT * FROM $full_table_name ".
+            "WHERE USER_ID = ? " . $db->limitOffset(1);
+        $user = (int)$user_id;
+        $result = $db->execute($sql2, [$user]);
+        $list = $db->fetchArray($result);
+        $thread_1_id = $list['GROUP_1'];
+        $thread_2_id = $list['GROUP_2'];
+        $thread_3_id = $list['GROUP_3'];
+        $sql_1 = "SELECT TITLE FROM GROUP_ITEM ".
+            "WHERE ID = ? " . $db->limitOffset(1);
+        $result_1 = $db->execute($sql_1, [$thread_1_id]);
+        $execute_1 = $db->fetchArray($result_1);
+        $thread_1 = $execute_1['TITLE'];
+        $sql_2 = "SELECT TITLE FROM GROUP_ITEM ".
+            "WHERE ID = ? " . $db->limitOffset(1);
+        $result_2 = $db->execute($sql_1, [$thread_2_id]);
+        $execute_2 = $db->fetchArray($result_2);
+        $thread_2 = $execute_2['TITLE'];
+        $sql_3 = "SELECT TITLE FROM GROUP_ITEM ".
+            "WHERE ID = ? " . $db->limitOffset(1);
+        $result_3 = $db->execute($sql_3, [$thread_3_id]);
+        $execute_3 = $db->fetchArray($result_3);
+        $thread_3 = $execute_3['TITLE'];
+        $threads = array(
+            $thread_1_id  => $thread_1,
+            $thread_2_id  => $thread_2,
+            $thread_3_id  => $thread_3,
+        );
+        return $threads;
+    }
 }
diff --git a/src/views/elements/ManageaccountElement.php b/src/views/elements/ManageaccountElement.php
index 719716d4..8a89cdc3 100755
--- a/src/views/elements/ManageaccountElement.php
+++ b/src/views/elements/ManageaccountElement.php
@@ -359,6 +359,43 @@ class ManageaccountElement extends Element
                 <?php
             }
             ?>
+            <h2><?=tl('manageaccount_element_recommendations')?></h2>
+            <div class="access-result">
+            <b><?=tl('manageaccount_element_recommendation_threads')?></b>
+                <?php
+                foreach ($data['RECOMMENDATIONS_THREADS'] as 
+                    $thread => $threadName) {
+                ?>
+                    <a href="<?=htmlentities(B\feedsUrl("thread",
+                        $group['THREAD_ID'], true, $admin_or_group)) . 
+                            $token ?>"
+                        ><?= $threadName ?></a>
+                <?php
+                }
+                ?></br>
+            <b><?=tl('manageaccount_element_recommendation_groups')?></b>
+            <?php
+                foreach ($data['RECOMMENDATIONS_GROUPS'] as $group) {
+                ?>
+                    <a href="<?=htmlentities(B\feedsUrl("group",
+                        $group['GROUP_NAME'], true, $admin_or_group)) . 
+                            $token ?>"
+                        ><?= $group ?></a> 
+                <?php
+                }   
+                ?></br>
+            <b><?=tl('manageaccount_element_trending_groups')?></b>
+                <?php
+                foreach ($data['RECOMMENDATIONS_TRENDING'] as $threadName) {
+                ?>
+                    <a href="<?=htmlentities(B\feedsUrl("group",
+                        $group['GROUP_NAME'], true, $admin_or_group)) . 
+                            $token ?>"
+                        ><?= $threadName ?></a> 
+                <?php
+                }   
+                ?>
+            </div>
             <p>[<a href="<?= $group_url ?>"><?=
                 tl('manageaccount_element_manage_all_groups')
                 ?></a>] [<a href="<?=$feed_url ?>"><?=
diff --git a/tests/RecommendationTest.php b/tests/RecommendationTest.php
new file mode 100644
index 00000000..6009a482
--- /dev/null
+++ b/tests/RecommendationTest.php
@@ -0,0 +1,83 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2017  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Sarika Padmashali padmashalisarika@gmail.com
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2017
+ * @filesource/
+ */
+namespace seekquarry\yioop\tests;
+
+use seekquarry\yioop\library\media_jobs\RecommendationJob;
+use seekquarry\yioop\library\UnitTest;
+
+/**
+ * Used to test that the XlsxProcessor class provides the basic functionality
+ * of getting the tile, description, languages and links
+ *
+ * @author Sarika Padmashali
+ */
+class RecommendationTest extends UnitTest 
+{
+    /**
+     * sets up the initial content for the testcase by extracting
+     * it from the xlsx file
+     */
+    public function setUp()
+    {
+    
+    }
+    /**
+     * Can be used for clenup activity
+     */
+    public function tearDown()
+    {
+    }
+    /**
+     * Tests that the title is correct
+     */
+    public function wordCountTestCase()
+    {
+
+        $phrase_string = <<< EOD
+WEB SERVER SIDE CLASS
+EOD;
+        $word = array("WEB"=>1, "SERVER"=>1, "SIDE"=>1, "CLASS"=>1);
+        $extracted_data = RecommendationJob::wordCount($phrase_string);
+        foreach ($word as $key=>$value) {
+            $this->assertTrue(($extracted_data[$key] == $value),"hiii");
+        }
+
+        $phrase_string = <<< EOD
+INFORMATION RETRIEVAL PHP LANGUAGE
+EOD;
+        $word = array("INFORMATION"=>1, "LANGUAGE"=>1, "PHP"=>1, "RETRIEVAL"=>1);
+        $extracted_data = RecommendationJob::wordCount($phrase_string);
+        foreach ($word as $key=>$value) {
+            $this->assertTrue(($extracted_data[$key] == $value),"hiii");
+        }
+
+    }
+}
\ No newline at end of file
-- 
2.11.0 (Apple Git-81)

