_binary From d0dbc40447914b4c8957c24044e1fdecf5571ee6 Mon Sep 17 00:00:00 2001
From: tchow <tph.chow@gmail.com>
Date: Wed, 1 May 2019 14:39:29 -0700
Subject: [PATCH] Adds trending tool to Yioop, a=chris

---
 src/configs/Config.php                    |  4 +-
 src/controllers/SearchController.php      | 18 +++++--
 src/css/search.css                        | 27 ++++++++++
 src/index.php                             | 16 ++++++
 src/library/VersionFunctions.php          | 13 +++++
 src/library/media_jobs/FeedsUpdateJob.php | 81 +++++++++++++++++++++++++++++
 src/locale/en_US/configure.ini            |  7 +++
 src/models/ProfileModel.php               |  6 +++
 src/models/TrendingModel.php              | 72 ++++++++++++++++++++++++++
 src/views/SearchView.php                  | 12 +++--
 src/views/elements/MoreoptionsElement.php |  3 ++
 src/views/elements/TrackerElement.php     | 86 +++++++++++++++++++++++++++++++
 12 files changed, 335 insertions(+), 10 deletions(-)
 create mode 100644 src/models/TrendingModel.php
 create mode 100644 src/views/elements/TrackerElement.php

diff --git a/src/configs/Config.php b/src/configs/Config.php
index fc5d8bd..0f80b9f 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -158,7 +158,7 @@ function nsconddefine($constant, $value)
  * Version number for upgrade database function
  * @var int
  */
-nsdefine('DATABASE_VERSION', 61);
+nsdefine('DATABASE_VERSION', 62);
 /**
  * Minimum Version fo Yioop for which keyword ad script
  * still works with this version
diff --git a/src/controllers/SearchController.php b/src/controllers/SearchController.php
index e35f9ae..8434810 100755
--- a/src/controllers/SearchController.php
+++ b/src/controllers/SearchController.php
@@ -54,7 +54,7 @@ class SearchController extends Controller implements CrawlConstants
      * @var array
      */
     public $activities = ["query", "more", "cache", "related", "signout",
-        "recordClick"];
+        "recordClick", "trending"];
     /**
      * Name of the sub-search currently in use
      * @var string
@@ -339,7 +339,7 @@ class SearchController extends Controller implements CrawlConstants
             $activity = $_REQUEST['a'];
             if (isset($_REQUEST['arg'])) {
                 $arg = $_REQUEST['arg'];
-            } else if ($activity != "more"){
+            } else if ($activity != "more" && $activity != "trending"){
                 $activity = "query";
             }
         }
@@ -391,7 +391,17 @@ class SearchController extends Controller implements CrawlConstants
                     $this->clean($_REQUEST['more_page'], "int"));
             }
             $_SESSION['REMOTE_ADDR'] = L\remoteAddress();
-        } else if (!empty($_REQUEST['s'])) {
+        } else if ($activity == "trending") {
+            $trending_model = $this->model('trending');
+            $data['TRENDING'] = true;
+            $data['ACTIVITY_METHOD'] = 'trending';
+            $data['ACTIVITY_CONTROLLER'] = 'search';
+            $data['NO_QUERY'] = true;
+            $data['MORE_PAGE'] = 0;
+            $data['TREND_DATA'] = $trending_model->getResults();
+            $_SESSION['REMOTE_ADDR'] = L\remoteAddress();
+        }
+        else if (!empty($_REQUEST['s'])) {
             $data['ACTIVITY_METHOD'] = 's/'.$this->subsearch_name;
             $data['ACTIVITY_CONTROLLER'] = 'search';
         }
@@ -974,7 +984,7 @@ class SearchController extends Controller implements CrawlConstants
      * Given a page summary extract the words from it and try to find documents
      * which match the most relevant words. The algorithm for "relevant" is
      * pretty weak. For now we pick the $num many words whose ratio
-     * of number of occurences in crawl item/ number of occurences in all
+     * of number of occurrences in crawl item/ number of occurrences in all
      * documents is the largest
      *
      * @param string $crawl_item a page summary
diff --git a/src/css/search.css b/src/css/search.css
index 59a960f..de0ecd7 100755
--- a/src/css/search.css
+++ b/src/css/search.css
@@ -3166,3 +3166,30 @@ p.start-ad
     min-height: 0.5in;
     width: 3in;
 }
+
+.trending-container
+{
+    display:flex;
+}
+
+.trending-floatLeft
+{
+    width: 36%;
+    float: left;
+}
+
+.trending-table
+{
+    border: 1px solid black;
+}
+
+.trending-tr
+{
+    border: 1px solid black;
+}
+
+.trending-th
+{
+    border: 1px solid black;
+    font-weight: normal;
+}
\ No newline at end of file
diff --git a/src/index.php b/src/index.php
index 36967d3..0b41ce1 100644
--- a/src/index.php
+++ b/src/index.php
@@ -190,6 +190,7 @@ function configureRewrites($web_site)
         'register' => 'routeController',
         'settings' => 'routeController',
         'tests' => 'routeController',
+        'trending' => 'routeTrending',
         's' => "routeSubsearch",
         'more' => 'routeMore',
         'suggest' => 'routeSuggest',
@@ -418,6 +419,21 @@ function routeMore($route_args)
     return true;
 }
 /**
++ * Used to route requests to the trending page page.
++ * If redirects on, then /trending routes to this trending page.
++ *
++ * @param array $route_args of url parts (split on slash).
++ * @return bool whether was able to compute a route or not
++ */
+function routeTrending($route_args)
+{
+    $_REQUEST['c'] = "search";
+    $_REQUEST['a'] = "trending";
+    $_REQUEST['route']['c'] = true;
+    $_REQUEST['route']['a'] = true;
+    return true;
+}
+/**
  * Return the url for the more and tools link on the landing page making use of
  * the defined variable REDIRECTS_ON.
  *
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index fd07b50..421bd6a 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1764,3 +1764,16 @@ function upgradeDatabaseVersion61(&$db)
         FROM MEDIA_SOURCE_OLD");
     $db->execute("DROP TABLE MEDIA_SOURCE_OLD");
 }
+/**
+ * Upgrades a Version 61 version of the Yioop database to a Version 62 version
+ * @param object $db datasource to use to upgrade.
+ */
+function upgradeDatabaseVersion62(&$db)
+{
+    $db->execute("CREATE TABLE TOP_WORDS (
+        TERM VARCHAR(" . C\TITLE_LEN . "),
+        OCCURRENCE INTEGER DEFAULT 0,
+        UPDATE_PERIOD NUMERIC,
+        TIMESTAMP NUMERIC(" . C\TIMESTAMP_LEN . ")
+        )");
+}
\ No newline at end of file
diff --git a/src/library/media_jobs/FeedsUpdateJob.php b/src/library/media_jobs/FeedsUpdateJob.php
index 17e823b..d7e2d1d 100644
--- a/src/library/media_jobs/FeedsUpdateJob.php
+++ b/src/library/media_jobs/FeedsUpdateJob.php
@@ -628,6 +628,7 @@ class FeedsUpdateJob extends MediaJob
             L\crawlLog("----..Making new index" .
                 " of non-pruned items.");
             $i = 0;
+            $word_counts = [];
             while ($item = $db->fetchArray($result)) {
                 L\crawlTimeoutLog(
                     "----..have added %s non-pruned items to index.", $i);
@@ -644,6 +645,24 @@ class FeedsUpdateJob extends MediaJob
                     $media_category = "news";
                 }
                 $phrase_string = $item["TITLE"] . " ". $item["DESCRIPTION"];
+                $tokenizer = PhraseParser::getTokenizer($lang);
+                // parsing words to be added into the word_and_count list
+                $removed_stop_word_string = $tokenizer->stopwordsRemover($phrase_string);
+                $stemmed_word_string = PhraseParser::stemTerms($removed_stop_word_string, $lang);
+                $word_and_count_lists = PhraseParser::extractPhrasesAndCount(
+                    join(' ', $stemmed_word_string), $lang);
+                foreach($word_and_count_lists as $word => $occurrence) {
+                    if(!in_array($word, ['-', '--', '#39'])) {
+                        // if word already exists in list, then just add to occurrence
+                        if (array_key_exists($word, $word_counts)) {
+                            $word_counts[$word]["OCCURRENCE"] = $word_counts[$word]["OCCURRENCE"] + $occurrence;
+                        }
+                        // otherwise, add new entry
+                        else {
+                            $word_counts[$word]["OCCURRENCE"] = $occurrence;
+                        }
+                    }
+                }
                 $word_and_qa_lists = PhraseParser::extractPhrasesInLists(
                     $phrase_string, $lang);
                 $raw_guid = L\unbase64Hash($item["GUID"]);
@@ -655,6 +674,68 @@ class FeedsUpdateJob extends MediaJob
                 $prune_shard->addDocumentWords($doc_keys, $item['PUBDATE'],
                     $word_and_qa_lists["WORD_LIST"], $meta_ids, true, false);
             }
+             /**
+             * Get all entries in TOP_WORDS that are of a certain update period
+             * There are actually the 25 top words per a unique timestamp, but
+             * we'll refer to this set of 25 as a single timed entry.
+             * Here, we update our table so that 24 hourly entries are converted
+             * into a daily entry, and 7 daily entries are converted into a
+             * weekly entry.
+             * Entries that are older than a week are subsequently removed from
+             * the database.
+             */
+             $sql = "SELECT COUNT(DISTINCT TIMESTAMP) as COUNT
+                       FROM TOP_WORDS WHERE UPDATE_PERIOD = ?";
+             $result = $db->execute($sql, [C\ONE_HOUR]);
+             $num_of_results = 25;
+             $days_in_week = 7;
+             $hours_in_day = 24;
+             if($result) {
+                 while($item = $db->fetchArray($result)) {
+                     if ($item['COUNT'] >= $hours_in_day) {
+                         $sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+                             "FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+                             "GROUP BY TERM ORDER BY OCCURRENCE DESC LIMIT ?";
+                         $res = $db->execute($sql, [C\ONE_HOUR, $num_of_results]);
+                         $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+                         while ($item2 = $db->fetchArray($res)) {
+                             $db->execute($sql, [$item2['TERM'], $item2['OCCURRENCE'], C\ONE_DAY, $time]);
+                         }
+                         $sql = "DELETE FROM TOP_WORDS WHERE UPDATE_PERIOD = ? AND TIMESTAMP < ?";
+                         $db->execute($sql, [C\ONE_HOUR, $time - C\ONE_DAY]);
+                     }
+                 }
+            }
+            $sql = "SELECT COUNT(DISTINCT TIMESTAMP) as COUNT
+                FROM TOP_WORDS WHERE UPDATE_PERIOD = ?";
+            $result = $db->execute($sql, [C\ONE_DAY]);
+            if($result) {
+                while($item = $db->fetchArray($result)) {
+                    if ($item['COUNT'] >= $days_in_week) {
+                        $sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+                            "FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+                            "GROUP BY TERM ORDER BY OCCURRENCE DESC LIMIT ?";
+                        $res = $db->execute($sql, [C\ONE_DAY, $num_of_results]);
+                        $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+                        while ($item2 = $db->fetchArray($res)) {
+                           $db->execute($sql, [$item2['TERM'], $item2['OCCURRENCE'], C\ONE_WEEK, $time]);
+                        }
+                        $sql = "DELETE FROM TOP_WORDS WHERE OCCURRENCE = ? AND TIMESTAMP < ?";
+                        $db->execute($sql, [C\ONE_DAY, $time - C\ONE_WEEK]);
+                    }
+                }
+            }
+            // Sort the word occurrence list by occurrence and insert into database.
+            arsort($word_counts);
+            $counter = 0;
+            $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+            foreach($word_counts as $word => $value) {
+                $db->execute($sql, [$word, $value["OCCURRENCE"], C\ONE_HOUR, $time]);
+                $counter += 1;
+                if($counter == $num_of_results) {
+                    break;
+                }
+            }
         }
         $prune_shard->save();
         set_error_handler(null);
diff --git a/src/locale/en_US/configure.ini b/src/locale/en_US/configure.ini
index ad00459..8ebf75d 100644
--- a/src/locale/en_US/configure.ini
+++ b/src/locale/en_US/configure.ini
@@ -1604,6 +1604,13 @@ moreoptions_element_create_account = "Create Account"
 moreoptions_element_suggest = "Suggest a URL"
 moreoptions_element_wiki_pages = "Wiki Pages"
 moreoptions_element_tools = "Tools"
+moreoptions_element_trending = "Trending"
+;
+; TrackerElement.php
+tracker_element_title = "Trending Data for Words and Phrases"
+tracker_element_term = "Term"
+tracker_element_occurrence = "Occurrence"
+tracker_element_data_time = "Data grabbed at"
 ;
 ; ManagerolesElement.php
 manageroles_element_roles = "Role List"
diff --git a/src/models/ProfileModel.php b/src/models/ProfileModel.php
index 775dd07..9b01591 100755
--- a/src/models/ProfileModel.php
+++ b/src/models/ProfileModel.php
@@ -330,6 +330,12 @@ class ProfileModel extends Model
                 FOLDER_NAME VARCHAR(" . C\NAME_LEN."),
                 INDEX_IDENTIFIER CHAR(".(strlen('m:') + C\TIMESTAMP_LEN)."),
                 PER_PAGE INTEGER)",
+            "TOP_WORDS" => "CREATE TABLE TOP_WORDS (
+                TERM VARCHAR(" . C\TITLE_LEN . "),
+                OCCURRENCE INTEGER DEFAULT 0,
+                UPDATE_PERIOD NUMERIC,
+                TIMESTAMP NUMERIC(" . C\TIMESTAMP_LEN . ")
+                )",
             "TRANSLATION" => "CREATE TABLE TRANSLATION (
                 TRANSLATION_ID $serial PRIMARY KEY
                 $auto_increment, IDENTIFIER_STRING VARCHAR(" . C\TITLE_LEN
diff --git a/src/models/TrendingModel.php b/src/models/TrendingModel.php
new file mode 100644
index 0000000..b43b416
--- /dev/null
+++ b/src/models/TrendingModel.php
@@ -0,0 +1,72 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Timothy Chow
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+namespace seekquarry\yioop\models;
+
+use seekquarry\yioop as B;
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+use seekquarry\yioop\library\MediaConstants;
+use seekquarry\yioop\library\VersionManager;
+use seekquarry\yioop\library\WikiParser;
+use seekquarry\yioop\library\processors\ImageProcessor;
+use seekquarry\yioop\models\ImpressionModel;
+
+/**
+ * This is class is used to handle
+ * db results related to Group Administration. Groups are collections of
+ * users who might access a common blog/news feed and set of pages. This
+ * method also controls adding and deleting entries to a group feed and
+ * does limited access control checks of these operations.
+ *
+ * @author Timothy Chow
+ */
+class TrendingModel extends Model implements MediaConstants
+{
+	public function getResults(){
+		$db = $this->db;
+		$update_periods = [C\ONE_HOUR, C\ONE_DAY, C\ONE_WEEK];
+		foreach($update_periods as $period){
+			$sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+					"FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+					"GROUP BY TERM ORDER BY TIMESTAMP DESC, OCCURRENCE ".
+                    "DESC LIMIT 25";
+			$result = $db->execute($sql, [$period]);
+			$temp = [];
+			if($result){
+				while($res = $db->fetchArray($result)){
+					array_push($temp, $res);
+				}
+			}
+			$item[$period] = $temp;
+		}
+        return $item;
+	}
+}
diff --git a/src/views/SearchView.php b/src/views/SearchView.php
index bc7797b..c387fb4 100755
--- a/src/views/SearchView.php
+++ b/src/views/SearchView.php
@@ -63,7 +63,8 @@ class SearchView extends View implements CrawlConstants
      */
     public function renderView($data)
     {
-        $data['LAND'] = (!isset($data['PAGES']) && !isset($data['MORE']))
+        $data['LAND'] = (!isset($data['PAGES']) && !isset($data['MORE']) &&
+            !isset($data['TRENDING']))
             ? 'landing-' : '';
         if (C\SIGNIN_LINK || C\SUBSEARCH_LINK) {?>
         <div class="<?= $data['LAND'] ?>top-bar"><?php
@@ -79,7 +80,8 @@ class SearchView extends View implements CrawlConstants
             $query_parts[C\CSRF_TOKEN] = $data[C\CSRF_TOKEN];
         }
         $logo = C\BASE_URL . C\LOGO;
-        $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']));
+        $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']) &&
+            !isset($data['TRENDING']));
         if ($is_landing) { ?>
             <div class="landing">
         <?php
@@ -95,7 +97,7 @@ class SearchView extends View implements CrawlConstants
             /></a>
         </h1>
         <?php
-        if (isset($data['PAGES']) || isset($data['MORE'])){?>
+        if (isset($data['PAGES']) || isset($data['MORE']) || isset($data['TRENDING'])){?>
             <div class="serp">
             <?php
         }
@@ -136,7 +138,7 @@ class SearchView extends View implements CrawlConstants
         if (isset($data['PAGES']) && !isset($data['MORE'])) {
             ?></div><?php
             $this->renderSearchResults($data);
-        } else if (isset($data['MORE'])) {
+        } else if (isset($data['MORE']) && !isset($data['TRENDING'])) {
             $top ="";
             if (!$_SERVER["MOBILE"]) {
                 $top = "class='medium-top more-options'";
@@ -146,6 +148,8 @@ class SearchView extends View implements CrawlConstants
             e("</div><div $top>");
             $this->element("moreoptions")->render($data);
             e("</div>");
+        } else if (isset($data['TRENDING'])) {
+            $this->element("tracker")->render($data);
         }
         ?>
         <div class="landing-footer">
diff --git a/src/views/elements/MoreoptionsElement.php b/src/views/elements/MoreoptionsElement.php
index bfae5f6..76ee490 100644
--- a/src/views/elements/MoreoptionsElement.php
+++ b/src/views/elements/MoreoptionsElement.php
@@ -178,15 +178,18 @@ class MoreoptionsElement extends Element
         if (empty($token_query))  {
             $suggest_url = B\suggestUrl();
             $pages_url = B\wikiUrl('pages');
+            $trending_url = B\controllerUrl('trending');
         } else {
             $suggest_url = B\suggestUrl(true) . $token_query;
             $pages_url = B\wikiUrl('pages', true) . $token_query;
+            $trending_url = B\controllerUrl('trending', true) . $token_query;
         }
         if (in_array(C\REGISTRATION_TYPE, ['no_activation',
             'email_registration', 'admin_activation'])) {
             $tools[$suggest_url] = tl('moreoptions_element_suggest');
         }
         $tools[$pages_url] = tl('moreoptions_element_wiki_pages');
+        $tools[$trending_url] = tl('moreoptions_element_trending');
         if ($tools != []) {
             $max_column_num = 10;
             if ($_SERVER["MOBILE"]) {
diff --git a/src/views/elements/TrackerElement.php b/src/views/elements/TrackerElement.php
new file mode 100644
index 0000000..7adec32
--- /dev/null
+++ b/src/views/elements/TrackerElement.php
@@ -0,0 +1,86 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+namespace seekquarry\yioop\views\elements;
+
+use seekquarry\yioop as B;
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+
+/**
+ * Element responsible for drawing the page tracker info
+ *
+ * @author Chris Pollett
+ */
+class TrackerElement extends Element
+{
+    /**
+     * Method responsible for drawing the page with tracker info
+     *
+     * @param array $data to draw tables on page
+     */
+    public function render($data)
+    {
+		$date_map = [
+			C\ONE_HOUR => 'hourly',
+			C\ONE_DAY => 'daily',
+			C\ONE_WEEK => 'weekly',
+		];?>
+		<p><b><?tl=('tracker_element_title')?><b></p>
+		<?php
+		if(isset($data['TREND_DATA'])){
+			?><div class="trending-container"><?php
+			foreach($data['TREND_DATA'] as $period => $occurrences){
+				?>
+				<div class="trending-floatLeft">
+					<p><b>The top <?=$date_map[$period]?></b></p>
+					<table class="trending-table">
+						<tr class="trending-tr">
+						<th class="trending-th"> <?=tl('tracker_element_term')?></th>
+						<th class="trending-th"><?=tl('tracker_element_occurrence')?></th>
+						</tr>
+				<?php
+				foreach($occurrences as $i=>$item){
+					?>
+					<tr>
+					<th class="trending-th"><a href="/?q=<?=$item['TERM']?>"><?=$item['TERM']?></a></th>
+					<th class="trending-th"><?=$item['OCCURRENCE']?></th>
+					</tr>
+					<?php
+				}
+				?></table>
+				</div>
+				<?php
+			}?>
+			</div><br>
+			<p>Data grabbed at <?=date('m/d/Y h:i:s')?> </p> <?php
+		}
+    }
+}
-- 
2.7.4

