_binary From 1a476b0c3215ac95bb98cc5f0f1f45e286119bd6 Mon Sep 17 00:00:00 2001
From: TimmahC <tph.chow@gmail.com>
Date: Wed, 24 Apr 2019 18:46:49 -0700
Subject: [PATCH] Adding new feature, Trending, accessible through /trending or
 the more tools page, a=chris

---
 src/configs/Config.php                         |   2 +-
 src/configs/Createdb.php                       |   4 +
 src/controllers/AdminController.php            |   2 +-
 src/controllers/SearchController.php           |  13 +-
 src/controllers/components/SocialComponent.php | 211 +++++++++++++++++++++
 src/css/search.css                             |  27 +++
 src/index.php                                  |  16 ++
 src/library/VersionFunctions.php               |  13 ++
 src/library/media_jobs/FeedsUpdateJob.php      | 245 ++++++++++++++++---------
 src/locale/en_US/configure.ini                 |   1 +
 src/models/ProfileModel.php                    |   6 +
 src/models/TrendingModel.php                   |  72 ++++++++
 src/views/SearchView.php                       | 208 ++++++++++-----------
 src/views/elements/MoreoptionsElement.php      |   4 +-
 src/views/elements/TrackerElement.php          |  86 +++++++++
 15 files changed, 720 insertions(+), 190 deletions(-)
 create mode 100644 src/models/TrendingModel.php
 create mode 100644 src/views/elements/TrackerElement.php

diff --git a/src/configs/Config.php b/src/configs/Config.php
index fc5d8bd..edb49d7 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -158,7 +158,7 @@ function nsconddefine($constant, $value)
  * Version number for upgrade database function
  * @var int
  */
-nsdefine('DATABASE_VERSION', 61);
+nsdefine('DATABASE_VERSION', 62);
 /**
  * Minimum Version fo Yioop for which keyword ad script
  * still works with this version
diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php
index 49cb46d..3d85479 100755
--- a/src/configs/Createdb.php
+++ b/src/configs/Createdb.php
@@ -400,6 +400,10 @@ $activities = [
             "nl" => 'Configureren',
             "vi-VN" => 'Sắp xếp hoạt động dựa theo hoạch định',
         ]],
+        "suggestNews" => ['db_activity_suggest_news',
+            [
+                "en-US" => 'Suggest a News Source',
+            ]],
 ];
 $i = 1;
 foreach ($activities as $activity => $translation_info) {
diff --git a/src/controllers/AdminController.php b/src/controllers/AdminController.php
index e2f3aab..ee3382e 100755
--- a/src/controllers/AdminController.php
+++ b/src/controllers/AdminController.php
@@ -73,7 +73,7 @@ class AdminController extends Controller implements CrawlConstants
             ["manageAccount", "manageUsers", "manageRoles"],
         "crawl" => ["manageCrawls", "pageOptions", "searchSources",
             "resultsEditor", "scrapers", "manageClassifiers"],
-        "social" => ["manageGroups", "groupFeeds", "mixCrawls", "wiki"],
+        "social" => ["manageGroups", "groupFeeds", "mixCrawls", "suggestNews", "wiki"],
         "chatbot" => ["botStory"],
         "store" => ["manageCredits", "manageAdvertisements"],
         "system" => ["manageMachines", "manageLocales", "serverSettings",
diff --git a/src/controllers/SearchController.php b/src/controllers/SearchController.php
index e35f9ae..3cae411 100755
--- a/src/controllers/SearchController.php
+++ b/src/controllers/SearchController.php
@@ -54,7 +54,7 @@ class SearchController extends Controller implements CrawlConstants
      * @var array
      */
     public $activities = ["query", "more", "cache", "related", "signout",
-        "recordClick"];
+        "recordClick", "trending"];
     /**
      * Name of the sub-search currently in use
      * @var string
@@ -339,7 +339,7 @@ class SearchController extends Controller implements CrawlConstants
             $activity = $_REQUEST['a'];
             if (isset($_REQUEST['arg'])) {
                 $arg = $_REQUEST['arg'];
-            } else if ($activity != "more"){
+            } else if ($activity != "more" && $activity != "trending"){
                 $activity = "query";
             }
         }
@@ -391,6 +391,15 @@ class SearchController extends Controller implements CrawlConstants
                     $this->clean($_REQUEST['more_page'], "int"));
             }
             $_SESSION['REMOTE_ADDR'] = L\remoteAddress();
+        } else if ($activity == "trending") {
+			$trending_model = $this->model('trending');
+			$data['TRENDING'] = true;
+            $data['ACTIVITY_METHOD'] = 'trending';
+            $data['ACTIVITY_CONTROLLER'] = 'search';
+            $data['NO_QUERY'] = true;
+            $data['MORE_PAGE'] = 0;
+			$data['TREND_DATA'] = $trending_model->getResults();
+            $_SESSION['REMOTE_ADDR'] = L\remoteAddress();
         } else if (!empty($_REQUEST['s'])) {
             $data['ACTIVITY_METHOD'] = 's/'.$this->subsearch_name;
             $data['ACTIVITY_CONTROLLER'] = 'search';
diff --git a/src/controllers/components/SocialComponent.php b/src/controllers/components/SocialComponent.php
index 5f7649d..aff91fa 100644
--- a/src/controllers/components/SocialComponent.php
+++ b/src/controllers/components/SocialComponent.php
@@ -4535,4 +4535,215 @@ EOD;
         }
         $data['SCRIPT'] .= ']; drawFragments();';
     }
+    /**
+     * Handles admin request related to the search sources activity
+     *
+     * The search sources activity allows a user to add/delete search sources
+     * for news and podcasts, it also allows a user to control which subsearches
+     * appear on the SearchView page
+     *
+     * @return array $data info about current search sources, and current
+     *     sub-searches
+     */
+    public function suggestNews()
+    {
+        $parent = $this->parent;
+        $crawl_model = $parent->model("crawl");
+        $source_model = $parent->model("source");
+        $possible_arguments = ["addsource"];
+        $request_fields = ['start_row', 'num_show', 'end_row',
+            'SUBstart_row','SUBnum_show', 'SUBend_row'];
+        $data = [];
+        $data["ELEMENT"] = "suggestnews";
+        $data['SCRIPT'] = "";
+        $data['SOURCE_TYPES'] = [-1 => tl('crawl_component_media_kind'),
+            "rss" => tl('crawl_component_rss_feed'),
+            "json" => tl('crawl_component_json_feed'),
+            "html" => tl('crawl_component_html_feed'),
+            "regex" => tl('crawl_component_regex_feed'),
+            "feed_podcast" => tl('crawl_component_feed_podcast'),
+            "scrape_podcast" => tl('crawl_component_scrape_podcast'),
+            ];
+        $data['PODCAST_EXPIRES'] = [
+            -1 => tl('crawl_component_never'),
+            C\ONE_DAY => tl('crawl_component_one_day'),
+            C\ONE_WEEK => tl('crawl_component_one_week'),
+            C\ONE_MONTH => tl('crawl_component_one_month'),
+            C\ONE_YEAR => tl('crawl_component_one_year')
+            ];
+        $source_type_flag = false;
+        if (isset($_REQUEST['type']) &&
+            in_array($_REQUEST['type'],
+            array_keys($data['SOURCE_TYPES']))) {
+            $data['SOURCE_TYPE'] = $_REQUEST['type'];
+            $source_type_flag = true;
+        } else {
+            $data['SOURCE_TYPE'] = -1;
+        }
+        $machine_urls = $parent->model("machine")->getQueueServerUrls();
+        $search_lists = $crawl_model->getCrawlList(false, true,
+            $machine_urls);
+        $data["SEARCH_LISTS"] = [-1 =>
+            tl('crawl_component_sources_indexes')];
+        foreach ($search_lists as $item) {
+            $data["SEARCH_LISTS"]["i:" . $item["CRAWL_TIME"]] =
+                $item["DESCRIPTION"];
+        }
+        if (isset($_SESSION['USER_ID'])) {
+            $user = $_SESSION['USER_ID'];
+        } else {
+            $user = L\remoteAddress();
+        }
+        $search_lists= $crawl_model->getMixList($user);
+        foreach ($search_lists as $item) {
+            $data["SEARCH_LISTS"]["m:".$item["TIMESTAMP"]] =
+                $item["NAME"];
+        }
+        $n = C\NUM_RESULTS_PER_PAGE;
+        $data['PER_PAGE'] = [$n => $n, 2*$n => 2*$n, 5*$n=> 5*$n,
+            10*$n=>10*$n];
+        if (isset($_REQUEST['per_page']) &&
+            in_array($_REQUEST['per_page'], array_keys($data['PER_PAGE']))) {
+            $data['PER_PAGE_SELECTED'] = $_REQUEST['per_page'];
+        } else {
+            $data['PER_PAGE_SELECTED'] = C\NUM_RESULTS_PER_PAGE;
+        }
+        $locales = $parent->model("locale")->getLocaleList();
+        $data["LANGUAGES"] = [];
+        foreach ($locales as $locale) {
+            $data["LANGUAGES"][$locale['LOCALE_TAG']] = $locale['LOCALE_NAME'];
+        }
+        if (isset($_REQUEST['language']) &&
+            in_array($_REQUEST['language'],
+                array_keys($data["LANGUAGES"]))) {
+            $data['SOURCE_LOCALE_TAG'] =
+                $_REQUEST['language'];
+        } else {
+            $data['SOURCE_LOCALE_TAG'] = C\DEFAULT_LOCALE;
+        }
+        $data["CURRENT_SOURCE"] = [
+            "name" => "", "type"=> $data['SOURCE_TYPE'], "source_url" => "",
+            "aux_info" => "", 'category' => "news", 'channel_path' => "",
+            "image_xpath" =>"", 'item_path' => "", 'title_path' => "",
+            'description_path' => "", 'link_path' => "",
+            "language" => $data['SOURCE_LOCALE_TAG']];
+        $data["CURRENT_SUBSEARCH"] = [
+            "locale_string" => "", "folder_name" =>"",
+            "index_identifier" => "",
+            "per_page" => $data['PER_PAGE_SELECTED']];
+        $data['SOURCE_FORM_TYPE'] = "addsource";
+        $data["SEARCH_FORM_TYPE"] = "addsubsearch";
+        print_r($_REQUEST);
+        if (isset($_REQUEST['arg']) &&
+            $_REQUEST['arg'] == "addsource") {
+                if (!$source_type_flag) {
+                    return $parent->redirectWithMessage(
+                        tl('crawl_component_no_source_type'),
+                        $request_fields);
+                }
+                $must_have = ["name", "type", 'source_url'];
+                $is_parse_feed = false;
+                if (isset($_REQUEST['type']) &&
+                    in_array($_REQUEST['type'], ['html', 'json',
+                    'regex'] )) {
+                    $is_parse_feed = true;
+                    $must_have = array_merge($must_have, [
+                        'channel_path', 'item_path', 'title_path',
+                        'description_path', 'link_path']);
+                } else if (isset($_REQUEST['type']) &&
+                    in_array($_REQUEST['type'], ['feed_podcast',
+                    'scrape_podcast'])) {
+                    if (isset($_REQUEST['expires'])) {
+                        $_REQUEST['category'] = $_REQUEST['expires'];
+                    }
+                    $is_parse_feed = true;
+                }
+                if (isset($_REQUEST['type']) && $_REQUEST['type'] == -1) {
+                    return $parent->redirectWithMessage(
+                        tl('crawl_component_missing_type'),
+                        array_merge($request_fields, $must_have));
+                }
+                $to_clean = array_merge($must_have,
+                    ['aux_info', 'category','language', 'image_xpath',
+                    'channel_path', 'item_path', 'title_path',
+                    'description_path', 'link_path']);
+                foreach ($to_clean as $clean_me) {
+                    $r[$clean_me] = (isset($_REQUEST[$clean_me])) ?
+                        trim($parent->clean($_REQUEST[$clean_me],
+                        "string"))  : "";
+                    if ($clean_me == "source_url") {
+                        $r[$clean_me] = UrlParser::canonicalLink(
+                            $r[$clean_me], "");
+                        if (!$r[$clean_me]) {
+                            return $parent->redirectWithMessage(
+                                tl('crawl_component_invalid_url'),
+                                array_merge($request_fields, $to_clean));
+                        }
+                    }
+                    if (in_array($clean_me, $must_have) &&
+                        $r[$clean_me] == "" ) {
+                        return $parent->redirectWithMessage(
+                            tl('crawl_component_missing_fields'),
+                            array_merge($request_fields, $to_clean));
+                    }
+                }
+                if ($is_parse_feed) {
+                    $r['aux_info'] = $r['channel_path'] . "###" .
+                        $r['item_path'] . "###" . $r['title_path'].
+                        "###".$r['description_path']."###".$r['link_path'].
+                        "###" . $r['image_xpath'];
+                } else if (isset($_REQUEST['type']) &&
+                    $_REQUEST['type'] == 'rss') {
+                    $r['aux_info'] = $r['image_xpath'];
+                }
+                $this->sendNewsRequestMail("test");
+               /* return $parent->redirectWithMessage(
+                    tl('crawl_component_media_source_added'),
+                    $request_fields);*/
+        }
+        $data['CAN_LOCALIZE'] = $parent->model("user")->isAllowedUserActivity(
+            $_SESSION['USER_ID'], "manageLocales");
+        $parent->pagingLogic($data, $source_model, "MEDIA_SOURCES",
+            C\DEFAULT_ADMIN_PAGING_NUM/5, [["NAME", "", "", "ASC"]]);
+        $parent->pagingLogic($data, $source_model,
+            "SUBSEARCHES", C\DEFAULT_ADMIN_PAGING_NUM/5, [
+            ["FOLDER_NAME", "", "", "ASC"]], "SUB",
+            "SUBSEARCH");
+        foreach ($data["SUBSEARCHES"] as $search) {
+            if (!isset($data["SEARCH_LISTS"]
+                [trim($search['INDEX_IDENTIFIER'])])) {
+                $source_model->deleteSubsearch($search["FOLDER_NAME"]);
+            }
+        }
+        $data['SCRIPT'] .= "source_type = elt('source-type');".
+            "source_type.onchange = switchSourceType;".
+            "switchSourceType()";
+        return $data;
+    }
+    /**
+     * Use to send the account activation email
+     *
+     * @param array $user associative array of user
+     * @param array $data field to be sent to view so can set sent mail
+     *  interface message
+     */
+    function sendNewsRequestMail($user)
+    {
+        $server = new MailServer(C\MAIL_SENDER, C\MAIL_SERVER,
+            C\MAIL_SERVERPORT, C\MAIL_USERNAME, C\MAIL_PASSWORD,
+            C\MAIL_SECURITY);
+        /*$data['SCRIPT']/* .= "doMessage('<h1 class=\"red\" >".
+            tl('register_controller_registration_email_sent').
+            "</h1>');";*/
+        $subject = "nothing";
+        $message = "test";
+        $message .= tl('register_controller_email_body')."\n";
+        $creation_time = vsprintf('%d.%06d', gettimeofday());
+        /*$message .= C\BASE_URL.
+            "?c=register&a=emailVerification&email=".
+            $user['EMAIL'].
+            "&hash=" . urlencode(L\crawlCrypt($user['HASH'])).
+            "&time=" . $user['CREATION_TIME'];*/
+        $server->send($subject, C\MAIL_SENDER, "timmahc4@gmail.com", $message);
+    }
 }
diff --git a/src/css/search.css b/src/css/search.css
index 59a960f..2a8d0e7 100755
--- a/src/css/search.css
+++ b/src/css/search.css
@@ -3166,3 +3166,30 @@ p.start-ad
     min-height: 0.5in;
     width: 3in;
 }
+
+.trending-container
+{
+    display:flex;
+}
+
+.trending-floatLeft
+{
+    width: 36%;
+    float: left;
+}
+
+.trending-table
+{
+    border: 1px solid black;
+}
+
+.trending-tr
+{
+    border: 1px solid black;
+}
+
+.trending-th
+{
+    border: 1px solid black;
+    font-weight: normal;
+}
diff --git a/src/index.php b/src/index.php
index 510dd28..de9f098 100644
--- a/src/index.php
+++ b/src/index.php
@@ -190,6 +190,7 @@ function configureRewrites($web_site)
         'register' => 'routeController',
         'settings' => 'routeController',
         'tests' => 'routeController',
+        'trending' => 'routeTrending',
         's' => "routeSubsearch",
         'more' => 'routeMore',
         'suggest' => 'routeSuggest',
@@ -418,6 +419,21 @@ function routeMore($route_args)
     return true;
 }
 /**
+ * Used to route requests to the trending page page.
+ * If redirects on, then /trending routes to this trending page.
+ *
+ * @param array $route_args of url parts (split on slash).
+ * @return bool whether was able to compute a route or not
+ */
+function routeTrending($route_args)
+{
+    $_REQUEST['c'] = "search";
+    $_REQUEST['a'] = "trending";
+    $_REQUEST['route']['c'] = true;
+    $_REQUEST['route']['a'] = true;
+    return true;
+}
+/**
  * Return the url for the more and tools link on the landing page making use of
  * the defined variable REDIRECTS_ON.
  *
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index fd07b50..a5660b0 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1764,3 +1764,16 @@ function upgradeDatabaseVersion61(&$db)
         FROM MEDIA_SOURCE_OLD");
     $db->execute("DROP TABLE MEDIA_SOURCE_OLD");
 }
+/**
+ * Upgrades a Version 61 version of the Yioop database to a Version 62 version
+ * @param object $db datasource to use to upgrade.
+ */
+function upgradeDatabaseVersion62(&$db)
+{
+    $db->execute("CREATE TABLE TOP_WORDS (
+        TERM VARCHAR(" . C\TITLE_LEN . "),
+        OCCURRENCE INTEGER DEFAULT 0,
+        UPDATE_PERIOD NUMERIC,
+        TIMESTAMP NUMERIC(" . C\TIMESTAMP_LEN . ")
+        )");
+}
diff --git a/src/library/media_jobs/FeedsUpdateJob.php b/src/library/media_jobs/FeedsUpdateJob.php
index 5d55883..fef84e4 100644
--- a/src/library/media_jobs/FeedsUpdateJob.php
+++ b/src/library/media_jobs/FeedsUpdateJob.php
@@ -595,88 +595,169 @@ class FeedsUpdateJob extends MediaJob
      * @param int $age how many seconds old records should be deleted
      * @return bool whether job executed to complete
      */
-    public function rebuildFeedShard($age)
-    {
-        $time = time();
-        $feed_shard_name = C\WORK_DIRECTORY . "/feeds/index";
-        $prune_shard_name = C\WORK_DIRECTORY . "/feeds/prune_index";
-        $prune_shard =  new IndexShard($prune_shard_name);
-        $too_old = $time - $age;
-        if (!$prune_shard) {
-            return false;
-        }
-        $pre_feeds = $this->tasks;
-        if (!$pre_feeds) {
-            return false;
-        }
-        $feeds = [];
-        foreach ($pre_feeds as $pre_feed) {
-            if (!isset($pre_feed['NAME'])) {
-                continue;
-            }
-            $feeds[$pre_feed['NAME']] = $pre_feed;
-        }
-        $db = $this->db;
-        // we now rebuild the inverted index with the remaining items
-        $sql = "SELECT * FROM FEED_ITEM ".
-            "WHERE PUBDATE >= ? ".
-            "ORDER BY PUBDATE DESC";
-        $result = $db->execute($sql, [$too_old]);
-        if ($result) {
-            $completed = true;
-            L\crawlLog("----..Making new index" .
-                " of non-pruned items.");
-            $i = 0;
-            while ($item = $db->fetchArray($result)) {
-                L\crawlTimeoutLog(
-                    "----..have added %s non-pruned items to index.", $i);
-                $i++;
-                if (!isset($item['SOURCE_NAME'])) {
-                    continue;
-                }
-                $source_name = $item['SOURCE_NAME'];
-                if (isset($feeds[$source_name])) {
-                    $lang = $feeds[$source_name]['LANGUAGE'];
-                    $media_category = $feeds[$source_name]['CATEGORY'];
-                } else {
-                    $lang = "";
-                    $media_category = "news";
-                }
-                $phrase_string = $item["TITLE"] . " ". $item["DESCRIPTION"];
-                $word_and_qa_lists = PhraseParser::extractPhrasesInLists(
-                    $phrase_string, $lang);
-                $raw_guid = L\unbase64Hash($item["GUID"]);
-                $doc_keys = L\crawlHash($item["LINK"], true) .
-                    $raw_guid . "d". substr(L\crawlHash(
-                    UrlParser::getHost($item["LINK"])."/", true), 1);
-                $meta_ids = $this->calculateMetas($lang, $item['PUBDATE'],
-                    $source_name, $item["GUID"], $media_category);
-                $prune_shard->addDocumentWords($doc_keys, $item['PUBDATE'],
-                    $word_and_qa_lists["WORD_LIST"], $meta_ids, true, false);
-            }
-        }
-        $prune_shard->save();
-        set_error_handler(null);
-        @chmod($prune_shard_name, 0777);
-        @chmod($feed_shard_name, 0777);
-        @rename($prune_shard_name, $feed_shard_name);
-        @chmod($feed_shard_name, 0777);
-        set_error_handler(C\NS_CONFIGS . "yioop_error_handler");
-        L\crawlLog("----..deleting old feed items");
-        $sql = "DELETE FROM FEED_ITEM WHERE PUBDATE < ?";
-        $db->execute($sql, [$too_old]);
-        $base_dir = C\APP_DIR . "/resources";
-        $subfolder = L\crawlHash(
-            'group' . C\PUBLIC_GROUP_ID . C\AUTH_KEY);
-        $prefix_folder = substr($subfolder, 0, 3);
-        $subfolder = "t" . $subfolder;
-        $date = date('Y-m-d', $too_old - C\ONE_DAY);
-        $old_folder = "$base_dir/$prefix_folder/$subfolder/$date";
-        L\crawlLog("----..deleting old feed image thumb folder");
-        $db->unlinkRecursive($old_folder);
-        L\crawlLog("----..done deleting old items");
-    }
-    /**
+     public function rebuildFeedShard($age)
+     {
+         $time = time();
+         $feed_shard_name = C\WORK_DIRECTORY . "/feeds/index";
+         $prune_shard_name = C\WORK_DIRECTORY . "/feeds/prune_index";
+         $prune_shard =  new IndexShard($prune_shard_name);
+         $too_old = $time - $age;
+         if (!$prune_shard) {
+             return false;
+         }
+         $pre_feeds = $this->tasks;
+         if (!$pre_feeds) {
+             return false;
+         }
+         $feeds = [];
+         foreach ($pre_feeds as $pre_feed) {
+             if (!isset($pre_feed['NAME'])) {
+                 continue;
+             }
+             $feeds[$pre_feed['NAME']] = $pre_feed;
+         }
+         $db = $this->db;
+         // we now rebuild the inverted index with the remaining items
+         $sql = "SELECT * FROM FEED_ITEM ".
+             "WHERE PUBDATE >= ? ".
+             "ORDER BY PUBDATE DESC";
+         $result = $db->execute($sql, [$too_old]);
+         if ($result) {
+             $completed = true;
+             L\crawlLog("----..Making new index" .
+                 " of non-pruned items.");
+             $i = 0;             
+             $word_counts = [];
+             while ($item = $db->fetchArray($result)) {
+                 L\crawlTimeoutLog(
+                     "----..have added %s non-pruned items to index.", $i);
+                 $i++;
+                 if (!isset($item['SOURCE_NAME'])) {
+                     continue;
+                 }
+                 $source_name = $item['SOURCE_NAME'];
+                 if (isset($feeds[$source_name])) {
+                     $lang = $feeds[$source_name]['LANGUAGE'];
+                     $media_category = $feeds[$source_name]['CATEGORY'];
+                 } else {
+                     $lang = "";
+                     $media_category = "news";
+                 }
+                 $phrase_string = $item["TITLE"] . " ". $item["DESCRIPTION"];
+                 $tokenizer = PhraseParser::getTokenizer($lang);
+                 // parsing words to be added into the word_and_count list
+                 $removed_stop_word_string = $tokenizer->stopwordsRemover($phrase_string);
+                 $stemmed_word_string = PhraseParser::stemTerms($removed_stop_word_string, $lang);
+                 $word_and_count_lists = PhraseParser::extractPhrasesAndCount(
+                     join(' ', $stemmed_word_string), $lang);
+                 foreach($word_and_count_lists as $word => $occurence) {
+                     if(!in_array($word, ['-', '--', '#39'])){
+                         // if word already exists in list, then just add to occurence
+                         if (array_key_exists($word, $word_counts)){
+                             $word_counts[$word]["OCCURRENCE"] = $word_counts[$word]["OCCURRENCE"] + $occurence;
+                         }
+                         // otherwise, add new entry
+                         else{
+                             $word_counts[$word]["OCCURRENCE"] = $occurence;
+                         }
+                     }
+                 }
+                 $word_and_qa_lists = PhraseParser::extractPhrasesInLists(
+                     $phrase_string, $lang);
+                 $raw_guid = L\unbase64Hash($item["GUID"]);
+                 $doc_keys = L\crawlHash($item["LINK"], true) .
+                     $raw_guid . "d". substr(L\crawlHash(
+                     UrlParser::getHost($item["LINK"])."/", true), 1);
+                 $meta_ids = $this->calculateMetas($lang, $item['PUBDATE'],
+                     $source_name, $item["GUID"], $media_category);
+                 $prune_shard->addDocumentWords($doc_keys, $item['PUBDATE'],
+                     $word_and_qa_lists["WORD_LIST"], $meta_ids, true, false);
+             }
+             /**
+             * Get all entries in TOP_WORDS that are of a certain update period
+             * There are actually the 25 top words per a unique timestamp, but
+             * we'll refer to this set of 25 as a single timed entry.
+             * Here, we update our table so that 24 hourly entries are converted
+             * into a daily entry, and 7 daily entries are converted into a
+             * weekly entry.
+             * Entries that are older than a week are subsequently removed from
+             * the database.
+             */
+             $sql = "SELECT COUNT(DISTINCT TIMESTAMP) as COUNT
+ 						FROM TOP_WORDS WHERE UPDATE_PERIOD = ?";
+             $result = $db->execute($sql, [C\ONE_HOUR]);
+             $num_of_results = 25;
+             $days_in_week = 7;
+             $hours_in_day = 24;
+             if($result){
+                 while($item = $db->fetchArray($result)) {
+                     if ($item['COUNT'] >= $hours_in_day){
+                         $sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+                             "FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+                             "GROUP BY TERM ORDER BY OCCURRENCE DESC LIMIT ?";
+                         $res = $db->execute($sql, [C\ONE_HOUR, $num_of_results]);
+                         $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+                         while ($item2 = $db->fetchArray($res)) {
+                             $db->execute($sql, [$item2['TERM'], $item2['OCCURRENCE'], C\ONE_DAY, $time]);
+                         }
+                         $sql = "DELETE FROM TOP_WORDS WHERE UPDATE_PERIOD = ? AND TIMESTAMP < ?";
+                         $db->execute($sql, [C\ONE_HOUR, $time - C\ONE_DAY]);
+                     }
+                 }
+ 			}
+             $sql = "SELECT COUNT(DISTINCT TIMESTAMP) as COUNT
+                 FROM TOP_WORDS WHERE UPDATE_PERIOD = ?";
+             $result = $db->execute($sql, [C\ONE_DAY]);
+             if($result){
+                 while($item = $db->fetchArray($result)){
+                     if ($item['COUNT'] >= $days_in_week){
+                         $sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+                             "FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+                             "GROUP BY TERM ORDER BY OCCURRENCE DESC LIMIT ?";
+                         $res = $db->execute($sql, [C\ONE_DAY, $num_of_results]);
+                         $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+                         while ($item2 = $db->fetchArray($res)) {
+                             $db->execute($sql, [$item2['TERM'], $item2['OCCURRENCE'], C\ONE_WEEK, $time]);
+                         }
+                         $sql = "DELETE FROM TOP_WORDS WHERE OCCURRENCE = ? AND TIMESTAMP < ?";
+                         $db->execute($sql, [C\ONE_DAY, $time - C\ONE_WEEK]);
+                     }
+                 }
+             }
+             // Sort the word occurrence list by occurrence and insert into database.
+             arsort($word_counts);
+             $counter = 0;
+             $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+             foreach($word_counts as $word => $value){
+                 $db->execute($sql, [$word, $value["OCCURRENCE"], C\ONE_HOUR, $time]);
+                 $counter += 1;
+                 if($counter == $num_of_results) {
+                     break;
+                 }
+             }
+         }
+         $prune_shard->save();
+         set_error_handler(null);
+         @chmod($prune_shard_name, 0777);
+         @chmod($feed_shard_name, 0777);
+         @rename($prune_shard_name, $feed_shard_name);
+         @chmod($feed_shard_name, 0777);
+         set_error_handler(C\NS_CONFIGS . "yioop_error_handler");
+         L\crawlLog("----..deleting old feed items");
+         $sql = "DELETE FROM FEED_ITEM WHERE PUBDATE < ?";
+         $db->execute($sql, [$too_old]);
+         $base_dir = C\APP_DIR . "/resources";
+         $subfolder = L\crawlHash(
+             'group' . C\PUBLIC_GROUP_ID . C\AUTH_KEY);
+         $prefix_folder = substr($subfolder, 0, 3);
+         $subfolder = "t" . $subfolder;
+         $date = date('Y-m-d', $too_old - C\ONE_DAY);
+         $old_folder = "$base_dir/$prefix_folder/$subfolder/$date";
+         L\crawlLog("----..deleting old feed image thumb folder");
+         $db->unlinkRecursive($old_folder);
+         L\crawlLog("----..done deleting old items");
+     }
+     /**
      * Adds $item to FEED_ITEM table in db if it isn't already there
      *
      * @param array $item data from a single feed item
diff --git a/src/locale/en_US/configure.ini b/src/locale/en_US/configure.ini
index ad00459..6372b24 100644
--- a/src/locale/en_US/configure.ini
+++ b/src/locale/en_US/configure.ini
@@ -1604,6 +1604,7 @@ moreoptions_element_create_account = "Create Account"
 moreoptions_element_suggest = "Suggest a URL"
 moreoptions_element_wiki_pages = "Wiki Pages"
 moreoptions_element_tools = "Tools"
+moreoptions_element_trending = "Trending"
 ;
 ; ManagerolesElement.php
 manageroles_element_roles = "Role List"
diff --git a/src/models/ProfileModel.php b/src/models/ProfileModel.php
index 775dd07..4066a1d 100755
--- a/src/models/ProfileModel.php
+++ b/src/models/ProfileModel.php
@@ -330,6 +330,12 @@ class ProfileModel extends Model
                 FOLDER_NAME VARCHAR(" . C\NAME_LEN."),
                 INDEX_IDENTIFIER CHAR(".(strlen('m:') + C\TIMESTAMP_LEN)."),
                 PER_PAGE INTEGER)",
+            "TOP_WORDS" => "CREATE TABLE TOP_WORDS (
+				TERM VARCHAR(" . C\TITLE_LEN . "),
+				OCCURRENCE INTEGER DEFAULT 0,
+				UPDATE_PERIOD NUMERIC,
+				TIMESTAMP NUMERIC(" . C\TIMESTAMP_LEN . ")
+				)",
             "TRANSLATION" => "CREATE TABLE TRANSLATION (
                 TRANSLATION_ID $serial PRIMARY KEY
                 $auto_increment, IDENTIFIER_STRING VARCHAR(" . C\TITLE_LEN
diff --git a/src/models/TrendingModel.php b/src/models/TrendingModel.php
new file mode 100644
index 0000000..b43b416
--- /dev/null
+++ b/src/models/TrendingModel.php
@@ -0,0 +1,72 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Timothy Chow
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+namespace seekquarry\yioop\models;
+
+use seekquarry\yioop as B;
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+use seekquarry\yioop\library\MediaConstants;
+use seekquarry\yioop\library\VersionManager;
+use seekquarry\yioop\library\WikiParser;
+use seekquarry\yioop\library\processors\ImageProcessor;
+use seekquarry\yioop\models\ImpressionModel;
+
+/**
+ * This is class is used to handle
+ * db results related to Group Administration. Groups are collections of
+ * users who might access a common blog/news feed and set of pages. This
+ * method also controls adding and deleting entries to a group feed and
+ * does limited access control checks of these operations.
+ *
+ * @author Timothy Chow
+ */
+class TrendingModel extends Model implements MediaConstants
+{
+	public function getResults(){
+		$db = $this->db;
+		$update_periods = [C\ONE_HOUR, C\ONE_DAY, C\ONE_WEEK];
+		foreach($update_periods as $period){
+			$sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+					"FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+					"GROUP BY TERM ORDER BY TIMESTAMP DESC, OCCURRENCE ".
+                    "DESC LIMIT 25";
+			$result = $db->execute($sql, [$period]);
+			$temp = [];
+			if($result){
+				while($res = $db->fetchArray($result)){
+					array_push($temp, $res);
+				}
+			}
+			$item[$period] = $temp;
+		}
+        return $item;
+	}
+}
diff --git a/src/views/SearchView.php b/src/views/SearchView.php
index bc7797b..6717e5d 100755
--- a/src/views/SearchView.php
+++ b/src/views/SearchView.php
@@ -61,110 +61,112 @@ class SearchView extends View implements CrawlConstants
      * of results, how to fetch the next results, etc.
      *
      */
-    public function renderView($data)
-    {
-        $data['LAND'] = (!isset($data['PAGES']) && !isset($data['MORE']))
-            ? 'landing-' : '';
-        if (C\SIGNIN_LINK || C\SUBSEARCH_LINK) {?>
-        <div class="<?= $data['LAND'] ?>top-bar"><?php
-            $this->element("subsearch")->render($data);
-            $this->element("signin")->render($data);
-            ?>
-        </div>
-        <?php
-        }
-        $logged_in = isset($data["ADMIN"]) && $data["ADMIN"];
-        $query_parts = [];
-        if ($logged_in) {
-            $query_parts[C\CSRF_TOKEN] = $data[C\CSRF_TOKEN];
-        }
-        $logo = C\BASE_URL . C\LOGO;
-        $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']));
-        if ($is_landing) { ?>
-            <div class="landing">
-        <?php
-        } else if ($_SERVER["MOBILE"]) {
-            $logo = C\BASE_URL .C\M_LOGO;
-        }
-        ?>
-        <h1 class="logo"><a href="<?= C\BASE_URL ?><?php if ($logged_in) {
-            e("?".http_build_query($query_parts));
-            } ?>"><img
-            src="<?php e($logo); ?>" alt="<?= tl('search_view_title')
-                 ?>"
-            /></a>
-        </h1>
-        <?php
-        if (isset($data['PAGES']) || isset($data['MORE'])){?>
-            <div class="serp">
-            <?php
-        }
-        ?>
-        <div class="search-box">
-            <form id="search-form" method="get" action='?'
-                onsubmit="processSubmit()">
-            <p>
-            <?php if (isset($data["SUBSEARCH"]) && $data["SUBSEARCH"] != "") {
-                ?><input type="hidden" name="s" value="<?=
-                $data['SUBSEARCH'] ?>" />
-            <?php } ?>
-            <?php if ($logged_in) { ?>
-            <input id="csrf-token" type="hidden" name="<?= C\CSRF_TOKEN ?>"
-                value="<?= $data[C\CSRF_TOKEN] ?>" />
-            <?php } ?>
-            <input id="its-value" type="hidden" name="its" value="<?=
-                $data['its'] ?>" />
-            <input type="search" <?php if (C\WORD_SUGGEST) { ?>
-                autocomplete="off"  onkeyup="onTypeTerm(event, this)"
-                <?php } ?>
-                title="<?= tl('search_view_input_label') ?>"
-                id="query-field" name="q" value="<?php
-                if (isset($data['QUERY']) && !isset($data['NO_QUERY'])) {
-                    e(urldecode($data['QUERY']));} ?>"
-                placeholder="<?= tl('search_view_input_placeholder') ?>"/>
-            <button class="button-box" type="submit"><img
-                src='<?=C\BASE_URL ?>resources/search-button.png'
-                alt='<?= tl('search_view_search') ?>'/></button>
-            </p>
-            </form>
-        </div>
-        <div id="suggest-dropdown">
-            <ul id="suggest-results" class="suggest-list">
-            </ul>
-        </div>
-        <?php
-        if (isset($data['PAGES']) && !isset($data['MORE'])) {
-            ?></div><?php
-            $this->renderSearchResults($data);
-        } else if (isset($data['MORE'])) {
-            $top ="";
-            if (!$_SERVER["MOBILE"]) {
-                $top = "class='medium-top more-options'";
-            } else {
-                $top = "class='more-options'";
-            }
-            e("</div><div $top>");
-            $this->element("moreoptions")->render($data);
-            e("</div>");
-        }
-        ?>
-        <div class="landing-footer">
-            <div class="center"><b><?php
-            if (isset($data['INDEX_INFO'])) {
-                e($data['INDEX_INFO']);
-            } else {
-                e(tl('search_view_no_index_set'));
-            } ?></b>
-            </div><?php $this->element("footer")->render($data);?>
+     public function renderView($data)
+     {
+         $data['LAND'] = (!isset($data['PAGES']) && !isset($data['MORE']) && !isset($data['TRENDING']))
+             ? 'landing-' : '';
+         if (C\SIGNIN_LINK || C\SUBSEARCH_LINK) {?>
+         <div class="<?= $data['LAND'] ?>top-bar"><?php
+             $this->element("subsearch")->render($data);
+             $this->element("signin")->render($data);
+             ?>
+         </div>
+         <?php
+         }
+         $logged_in = isset($data["ADMIN"]) && $data["ADMIN"];
+         $query_parts = [];
+         if ($logged_in) {
+             $query_parts[C\CSRF_TOKEN] = $data[C\CSRF_TOKEN];
+         }
+         $logo = C\BASE_URL . C\LOGO;
+         $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']) && !isset($data['TRENDING']));
+         if ($is_landing) { ?>
+             <div class="landing">
+         <?php
+         } else if ($_SERVER["MOBILE"]) {
+             $logo = C\BASE_URL .C\M_LOGO;
+         }
+         ?>
+         <h1 class="logo"><a href="<?= C\BASE_URL ?><?php if ($logged_in) {
+             e("?".http_build_query($query_parts));
+             } ?>"><img
+             src="<?php e($logo); ?>" alt="<?= tl('search_view_title')
+                  ?>"
+             /></a>
+         </h1>
+         <?php
+         if (isset($data['PAGES']) || isset($data['MORE']) || isset($data['TRENDING'])){?>
+             <div class="serp">
+             <?php
+         }
+         ?>
+         <div class="search-box">
+             <form id="search-form" method="get" action='?'
+                 onsubmit="processSubmit()">
+             <p>
+             <?php if (isset($data["SUBSEARCH"]) && $data["SUBSEARCH"] != "") {
+                 ?><input type="hidden" name="s" value="<?=
+                 $data['SUBSEARCH'] ?>" />
+             <?php } ?>
+             <?php if ($logged_in) { ?>
+             <input id="csrf-token" type="hidden" name="<?= C\CSRF_TOKEN ?>"
+                 value="<?= $data[C\CSRF_TOKEN] ?>" />
+             <?php } ?>
+             <input id="its-value" type="hidden" name="its" value="<?=
+                 $data['its'] ?>" />
+             <input type="search" <?php if (C\WORD_SUGGEST) { ?>
+                 autocomplete="off"  onkeyup="onTypeTerm(event, this)"
+                 <?php } ?>
+                 title="<?= tl('search_view_input_label') ?>"
+                 id="query-field" name="q" value="<?php
+                 if (isset($data['QUERY']) && !isset($data['NO_QUERY'])) {
+                     e(urldecode($data['QUERY']));} ?>"
+                 placeholder="<?= tl('search_view_input_placeholder') ?>"/>
+             <button class="button-box" type="submit"><img
+                 src='<?=C\BASE_URL ?>resources/search-button.png'
+                 alt='<?= tl('search_view_search') ?>'/></button>
+             </p>
+             </form>
+         </div>
+         <div id="suggest-dropdown">
+             <ul id="suggest-results" class="suggest-list">
+             </ul>
+         </div>
+         <?php
+         if (isset($data['PAGES']) && !isset($data['MORE']) && !isset($data['TRENDING'])) {
+             ?></div><?php
+             $this->renderSearchResults($data);
+         } else if (isset($data['MORE']) && !isset($data['TRENDING'])) {
+             $top ="";
+             if (!$_SERVER["MOBILE"]) {
+                 $top = "class='medium-top more-options'";
+             } else {
+                 $top = "class='more-options'";
+             }
+             e("</div><div $top>");
+             $this->element("moreoptions")->render($data);
+             e("</div>");
+         } else if (isset($data['TRENDING'])){
+ 			$this->element("tracker")->render($data);
+ 		}
+         ?>
+         <div class="landing-footer">
+             <div class="center"><b><?php
+             if (isset($data['INDEX_INFO'])) {
+                 e($data['INDEX_INFO']);
+             } else {
+                 e(tl('search_view_no_index_set'));
+             } ?></b>
+             </div><?php $this->element("footer")->render($data);?>
 
-        </div>
-        <?php
-        if ($is_landing) { ?>
-            </div>
-            <div class='landing-spacer'></div>
-            <?php
-        }
-    }
+         </div>
+         <?php
+         if ($is_landing) { ?>
+             </div>
+             <div class='landing-spacer'></div>
+             <?php
+         }
+     }
     /**
      * Used to draw the results of a query to the Yioop Search Engine
      *
diff --git a/src/views/elements/MoreoptionsElement.php b/src/views/elements/MoreoptionsElement.php
index bfae5f6..d9bf6f0 100644
--- a/src/views/elements/MoreoptionsElement.php
+++ b/src/views/elements/MoreoptionsElement.php
@@ -178,6 +178,7 @@ class MoreoptionsElement extends Element
         if (empty($token_query))  {
             $suggest_url = B\suggestUrl();
             $pages_url = B\wikiUrl('pages');
+            $trending_url = B\controllerUrl('trending');
         } else {
             $suggest_url = B\suggestUrl(true) . $token_query;
             $pages_url = B\wikiUrl('pages', true) . $token_query;
@@ -187,6 +188,7 @@ class MoreoptionsElement extends Element
             $tools[$suggest_url] = tl('moreoptions_element_suggest');
         }
         $tools[$pages_url] = tl('moreoptions_element_wiki_pages');
+        $tools[$trending_url] = tl('moreoptions_element_trending');
         if ($tools != []) {
             $max_column_num = 10;
             if ($_SERVER["MOBILE"]) {
@@ -224,4 +226,4 @@ class MoreoptionsElement extends Element
             <?php
         }
     }
-}
\ No newline at end of file
+}
diff --git a/src/views/elements/TrackerElement.php b/src/views/elements/TrackerElement.php
new file mode 100644
index 0000000..4d8a244
--- /dev/null
+++ b/src/views/elements/TrackerElement.php
@@ -0,0 +1,86 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+namespace seekquarry\yioop\views\elements;
+
+use seekquarry\yioop as B;
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+
+/**
+ * Element responsible for drawing the page tracker info
+ *
+ * @author Chris Pollett
+ */
+class TrackerElement extends Element
+{
+    /**
+     * Method responsible for drawing the page with tracker info
+     *
+     * @param array $data to draw tables on page
+     */
+    public function render($data)
+    {
+		$date_map = [
+			C\ONE_HOUR => 'hourly',
+			C\ONE_DAY => 'daily',
+			C\ONE_WEEK => 'weekly',
+		];?>
+		<p><b>Trending Data for Words and Phrases<b></p>
+		<?php
+		if(isset($data['TREND_DATA'])){
+			?><div class="trending-container"><?php
+			foreach($data['TREND_DATA'] as $period => $occurrences){
+				?>
+				<div class="trending-floatLeft">
+					<p><b>The top <?=$date_map[$period]?></b></p>
+					<table class="trending-table">
+						<tr class="trending-tr">
+						<th class="trending-th">Term</th>
+						<th class="trending-th">Occurrences</th>
+						</tr>
+				<?php
+				foreach($occurrences as $i=>$item){
+					?>
+					<tr>
+					<th class="trending-th"><a href="/?q=<?=$item['TERM']?>"><?=$item['TERM']?></a></th>
+					<th class="trending-th"><?=$item['OCCURRENCE']?></th>
+					</tr>
+					<?php
+				}
+				?></table>
+				</div>
+				<?php
+			}?>
+			</div><br>
+			<p>Data grabbed at <?=date('m/d/Y h:i:s')?> </p> <?php
+		}
+    }
+}
-- 
2.7.4

