_binary From 1376933f81c3e6102d0e8bfa62c8593a61f089bc Mon Sep 17 00:00:00 2001
From: TimmahC <tph.chow@gmail.com>
Date: Mon, 15 Apr 2019 16:54:05 -0700
Subject: [PATCH] Adding new feature, Trending, accessible through /trending or
 the more tools page, a=chris

---
 src/configs/Config.php                    |   2 +-
 src/controllers/SearchController.php      |  13 +++-
 src/index.php                             |  16 +++++
 src/library/VersionFunctions.php          |  13 ++++
 src/library/media_jobs/FeedsUpdateJob.php |  74 +++++++++++++++++++
 src/locale/en_US/configure.ini            |   1 +
 src/models/ProfileModel.php               |   6 ++
 src/models/TrendingModel.php              |  72 +++++++++++++++++++
 src/views/SearchView.php                  |  16 +++--
 src/views/elements/MoreoptionsElement.php |   4 +-
 src/views/elements/TrackerElement.php     | 114 ++++++++++++++++++++++++++++++
 11 files changed, 320 insertions(+), 11 deletions(-)
 create mode 100644 src/models/TrendingModel.php
 create mode 100644 src/views/elements/TrackerElement.php

diff --git a/src/configs/Config.php b/src/configs/Config.php
index 0754ada..7a6aecd 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -158,7 +158,7 @@ function nsconddefine($constant, $value)
  * Version number for upgrade function
  * @var int
  */
-nsdefine('YIOOP_VERSION', 61);
+nsdefine('YIOOP_VERSION', 62);
 /**
  * Minimum Version fo Yioop for which keyword ad script
  * still works with this version
diff --git a/src/controllers/SearchController.php b/src/controllers/SearchController.php
index e35f9ae..cf27ae3 100755
--- a/src/controllers/SearchController.php
+++ b/src/controllers/SearchController.php
@@ -54,7 +54,7 @@ class SearchController extends Controller implements CrawlConstants
      * @var array
      */
     public $activities = ["query", "more", "cache", "related", "signout",
-        "recordClick"];
+        "recordClick", "trending"];
     /**
      * Name of the sub-search currently in use
      * @var string
@@ -339,7 +339,7 @@ class SearchController extends Controller implements CrawlConstants
             $activity = $_REQUEST['a'];
             if (isset($_REQUEST['arg'])) {
                 $arg = $_REQUEST['arg'];
-            } else if ($activity != "more"){
+            } else if ($activity != "more" && $activity != "trending"){
                 $activity = "query";
             }
         }
@@ -391,6 +391,15 @@ class SearchController extends Controller implements CrawlConstants
                     $this->clean($_REQUEST['more_page'], "int"));
             }
             $_SESSION['REMOTE_ADDR'] = L\remoteAddress();
+        } else if ($activity == "trending"){
+			$trending_model = $this->model('trending');
+			$data['TRENDING'] = true;
+            $data['ACTIVITY_METHOD'] = 'trending';
+            $data['ACTIVITY_CONTROLLER'] = 'search';
+            $data['NO_QUERY'] = true;
+            $data['MORE_PAGE'] = 0;
+			$data['TREND_DATA'] = $trending_model->getResults();
+            $_SESSION['REMOTE_ADDR'] = L\remoteAddress();
         } else if (!empty($_REQUEST['s'])) {
             $data['ACTIVITY_METHOD'] = 's/'.$this->subsearch_name;
             $data['ACTIVITY_CONTROLLER'] = 'search';
diff --git a/src/index.php b/src/index.php
index 4a89e64..104b099 100644
--- a/src/index.php
+++ b/src/index.php
@@ -188,6 +188,7 @@ function configureRewrites($web_site)
         'register' => 'routeController',
         'settings' => 'routeController',
         'tests' => 'routeController',
+        'trending' => 'routeTrending',
         's' => "routeSubsearch",
         'more' => 'routeMore',
         'suggest' => 'routeSuggest',
@@ -416,6 +417,21 @@ function routeMore($route_args)
     return true;
 }
 /**
+ * Used to route requests to the trending page page.
+ * If redirects on, then /trending routes to this trending page.
+ *
+ * @param array $route_args of url parts (split on slash).
+ * @return bool whether was able to compute a route or not
+ */
+function routeTrending($route_args)
+{
+    $_REQUEST['c'] = "search";
+    $_REQUEST['a'] = "trending";
+    $_REQUEST['route']['c'] = true;
+    $_REQUEST['route']['a'] = true;
+    return true;
+}
+/**
  * Return the url for the more and tools link on the landing page making use of
  * the defined variable REDIRECTS_ON.
  *
diff --git a/src/library/VersionFunctions.php b/src/library/VersionFunctions.php
index fd07b50..a5660b0 100644
--- a/src/library/VersionFunctions.php
+++ b/src/library/VersionFunctions.php
@@ -1764,3 +1764,16 @@ function upgradeDatabaseVersion61(&$db)
         FROM MEDIA_SOURCE_OLD");
     $db->execute("DROP TABLE MEDIA_SOURCE_OLD");
 }
+/**
+ * Upgrades a Version 61 version of the Yioop database to a Version 62 version
+ * @param object $db datasource to use to upgrade.
+ */
+function upgradeDatabaseVersion62(&$db)
+{
+    $db->execute("CREATE TABLE TOP_WORDS (
+        TERM VARCHAR(" . C\TITLE_LEN . "),
+        OCCURRENCE INTEGER DEFAULT 0,
+        UPDATE_PERIOD NUMERIC,
+        TIMESTAMP NUMERIC(" . C\TIMESTAMP_LEN . ")
+        )");
+}
diff --git a/src/library/media_jobs/FeedsUpdateJob.php b/src/library/media_jobs/FeedsUpdateJob.php
index 5d55883..28056d4 100644
--- a/src/library/media_jobs/FeedsUpdateJob.php
+++ b/src/library/media_jobs/FeedsUpdateJob.php
@@ -643,6 +643,24 @@ class FeedsUpdateJob extends MediaJob
                     $media_category = "news";
                 }
                 $phrase_string = $item["TITLE"] . " ". $item["DESCRIPTION"];
+                $tokenizer = PhraseParser::getTokenizer($lang);
+                // parsing words to be added into the word_and_count list
+                $removed_stop_word_string = $tokenizer->stopwordsRemover($phrase_string);
+                $stemmed_word_string = PhraseParser::stemTerms($removed_stop_word_string, $lang);
+                $word_and_count_lists = PhraseParser::extractPhrasesAndCount(
+                                        join(' ', $stemmed_word_string), $lang);
+                foreach($word_and_count_lists as $word => $occurence) {
+                    if(!in_array($word, ['-', '--', '#39'])){
+                        // if word already exists in list, then just add to occurence
+                        if (array_key_exists($word, $word_counts)){
+                            $word_counts[$word]["OCCURRENCE"] = $word_counts[$word]["OCCURRENCE"] + $occurence;
+                        }
+                        // otherwise, add new entry
+                        else{
+                            $word_counts[$word]["OCCURRENCE"] = $occurence;
+                        }
+                    }
+                }
                 $word_and_qa_lists = PhraseParser::extractPhrasesInLists(
                     $phrase_string, $lang);
                 $raw_guid = L\unbase64Hash($item["GUID"]);
@@ -654,6 +672,62 @@ class FeedsUpdateJob extends MediaJob
                 $prune_shard->addDocumentWords($doc_keys, $item['PUBDATE'],
                     $word_and_qa_lists["WORD_LIST"], $meta_ids, true, false);
             }
+            // Get all entries in TOP_WORDS that are of a certain update period
+            // There are actually the 25 top words per a unique timestamp, but
+            // we'll refer to this set of 25 as a single timed entry.
+            // Here, we update our table so that 24 hourly entries are converted
+            // into a daily entry, and 7 daily entries are converted into a
+            // weekly entry.
+            // Entries that are older than a week are subsequently removed from
+            // the database.
+            $sql = "SELECT COUNT(DISTINCT TIMESTAMP) as COUNT
+						FROM TOP_WORDS WHERE UPDATE_PERIOD = ?";
+            $result = $db->execute($sql, [C\ONE_HOUR]);
+            if($result){
+				while($item = $db->fetchArray($result)){
+					//print_r($item);
+					if ($item['COUNT'] >= 24){
+						$sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+						"FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+						"GROUP BY TERM ORDER BY OCCURRENCE DESC LIMIT 25";
+						$res = $db->execute($sql, [C\ONE_HOUR]);
+						$sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+						while ($item2 = $db->fetchArray($res)) {
+							$db->execute($sql, [$item2['TERM'], $item2['OCCURRENCE'], C\ONE_DAY, $time]);
+						}
+						$sql = "DELETE FROM TOP_WORDS WHERE UPDATE_PERIOD = ? AND TIMESTAMP < ?";
+						$db->execute($sql, [C\ONE_HOUR, $time - C\ONE_DAY]);
+					}
+				}
+			}
+            $sql = "SELECT COUNT(DISTINCT TIMESTAMP) as COUNT
+                FROM TOP_WORDS WHERE UPDATE_PERIOD = ?";
+            $result = $db->execute($sql, [C\ONE_DAY]);
+            if($result){
+                while($item = $db->fetchArray($result)){
+                    if ($item['COUNT'] >= 7){
+                        $sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+                        "FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+                        "GROUP BY TERM ORDER BY OCCURRENCE DESC LIMIT 25";
+                        $res = $db->execute($sql, [C\ONE_DAY]);
+                        $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+                        while ($item2 = $db->fetchArray($res)) {
+                            $db->execute($sql, [$item2['TERM'], $item2['OCCURRENCE'], C\ONE_WEEK, $time]);
+                        }
+                        $sql = "DELETE FROM TOP_WORDS WHERE OCCURRENCE = ? AND TIMESTAMP < ?";
+                        $db->execute($sql, [C\ONE_DAY, $time - C\ONE_WEEK]);
+                    }
+                }
+            }
+            // Sort the word occurrence list by occurrence and insert into database.
+            arsort($word_counts);
+            $counter = 0;
+            $sql = "INSERT INTO TOP_WORDS (TERM, OCCURRENCE, UPDATE_PERIOD, TIMESTAMP) VALUES (?, ?, ?, ?)";
+            foreach($word_counts as $word => $value){
+                $db->execute($sql, [$word, $value["OCCURRENCE"], C\ONE_HOUR, $time]);
+                $counter += 1;
+                if($counter == 25) { break;}
+            }
         }
         $prune_shard->save();
         set_error_handler(null);
diff --git a/src/locale/en_US/configure.ini b/src/locale/en_US/configure.ini
index d91d654..e259e97 100644
--- a/src/locale/en_US/configure.ini
+++ b/src/locale/en_US/configure.ini
@@ -1600,6 +1600,7 @@ moreoptions_element_my_accounts = "My Accounts"
 moreoptions_element_settings = "Settings"
 moreoptions_element_signin = "Sign In"
 moreoptions_element_admin = "Admin"
+moreoptions_element_trending = "Trending"
 moreoptions_element_create_account = "Create Account"
 moreoptions_element_suggest = "Suggest a URL"
 moreoptions_element_wiki_pages = "Wiki Pages"
diff --git a/src/models/ProfileModel.php b/src/models/ProfileModel.php
index 775dd07..4066a1d 100755
--- a/src/models/ProfileModel.php
+++ b/src/models/ProfileModel.php
@@ -330,6 +330,12 @@ class ProfileModel extends Model
                 FOLDER_NAME VARCHAR(" . C\NAME_LEN."),
                 INDEX_IDENTIFIER CHAR(".(strlen('m:') + C\TIMESTAMP_LEN)."),
                 PER_PAGE INTEGER)",
+            "TOP_WORDS" => "CREATE TABLE TOP_WORDS (
+				TERM VARCHAR(" . C\TITLE_LEN . "),
+				OCCURRENCE INTEGER DEFAULT 0,
+				UPDATE_PERIOD NUMERIC,
+				TIMESTAMP NUMERIC(" . C\TIMESTAMP_LEN . ")
+				)",
             "TRANSLATION" => "CREATE TABLE TRANSLATION (
                 TRANSLATION_ID $serial PRIMARY KEY
                 $auto_increment, IDENTIFIER_STRING VARCHAR(" . C\TITLE_LEN
diff --git a/src/models/TrendingModel.php b/src/models/TrendingModel.php
new file mode 100644
index 0000000..d7afeb5
--- /dev/null
+++ b/src/models/TrendingModel.php
@@ -0,0 +1,72 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Mallika Perepa, Chris Pollett
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+namespace seekquarry\yioop\models;
+
+use seekquarry\yioop as B;
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+use seekquarry\yioop\library\MediaConstants;
+use seekquarry\yioop\library\VersionManager;
+use seekquarry\yioop\library\WikiParser;
+use seekquarry\yioop\library\processors\ImageProcessor;
+use seekquarry\yioop\models\ImpressionModel;
+
+/**
+ * This is class is used to handle
+ * db results related to Group Administration. Groups are collections of
+ * users who might access a common blog/news feed and set of pages. This
+ * method also controls adding and deleting entries to a group feed and
+ * does limited access control checks of these operations.
+ *
+ * @author Mallika Perepa (creator), Chris Pollett (rewrite)
+ */
+class TrendingModel extends Model implements MediaConstants
+{
+	public function getResults(){
+		$db = $this->db;
+		$update_periods = [C\ONE_HOUR, C\ONE_DAY, C\ONE_WEEK];
+		foreach($update_periods as $period){
+			$sql = "SELECT TERM, OCCURRENCE, UPDATE_PERIOD, MAX(TIMESTAMP)".
+					"FROM TOP_WORDS WHERE UPDATE_PERIOD = ? ".
+					"GROUP BY TERM ORDER BY TIMESTAMP DESC, OCCURRENCE ".
+                    "DESC LIMIT 25";
+			$result = $db->execute($sql, [$period]);
+			$temp = [];
+			if($result){
+				while($res = $db->fetchArray($result)){
+					array_push($temp, $res);
+				}
+			}
+			$item[$period] = $temp;
+		}
+        return $item;
+	}
+}
diff --git a/src/views/SearchView.php b/src/views/SearchView.php
index bc7797b..245ac04 100755
--- a/src/views/SearchView.php
+++ b/src/views/SearchView.php
@@ -63,7 +63,7 @@ class SearchView extends View implements CrawlConstants
      */
     public function renderView($data)
     {
-        $data['LAND'] = (!isset($data['PAGES']) && !isset($data['MORE']))
+        $data['LAND'] = (!isset($data['PAGES']) && !isset($data['MORE']) && !isset($data['TRENDING']))
             ? 'landing-' : '';
         if (C\SIGNIN_LINK || C\SUBSEARCH_LINK) {?>
         <div class="<?= $data['LAND'] ?>top-bar"><?php
@@ -79,7 +79,7 @@ class SearchView extends View implements CrawlConstants
             $query_parts[C\CSRF_TOKEN] = $data[C\CSRF_TOKEN];
         }
         $logo = C\BASE_URL . C\LOGO;
-        $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']));
+        $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']) && !isset($data['TRENDING']));
         if ($is_landing) { ?>
             <div class="landing">
         <?php
@@ -95,7 +95,7 @@ class SearchView extends View implements CrawlConstants
             /></a>
         </h1>
         <?php
-        if (isset($data['PAGES']) || isset($data['MORE'])){?>
+        if (isset($data['PAGES']) || isset($data['MORE']) || isset($data['TRENDING'])){?>
             <div class="serp">
             <?php
         }
@@ -133,10 +133,10 @@ class SearchView extends View implements CrawlConstants
             </ul>
         </div>
         <?php
-        if (isset($data['PAGES']) && !isset($data['MORE'])) {
+        if (isset($data['PAGES']) && !isset($data['MORE']) && !isset($data['TRENDING'])) {
             ?></div><?php
             $this->renderSearchResults($data);
-        } else if (isset($data['MORE'])) {
+        } else if (isset($data['MORE']) && !isset($data['TRENDING'])) {
             $top ="";
             if (!$_SERVER["MOBILE"]) {
                 $top = "class='medium-top more-options'";
@@ -146,7 +146,9 @@ class SearchView extends View implements CrawlConstants
             e("</div><div $top>");
             $this->element("moreoptions")->render($data);
             e("</div>");
-        }
+        } else if (isset($data['TRENDING'])){
+			$this->element("tracker")->render($data);
+		}
         ?>
         <div class="landing-footer">
             <div class="center"><b><?php
@@ -173,7 +175,7 @@ class SearchView extends View implements CrawlConstants
      */
     public function renderSearchResults($data)
     {
-        $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']));
+        $is_landing = (!isset($data['PAGES']) && !isset($data['MORE']) && !isset($data['TRENDING']));
         $logged_in = isset($data["ADMIN"]) && $data["ADMIN"];
         $token = ($logged_in) ? $data[C\CSRF_TOKEN] : "";
         $token_string = ($logged_in) ? C\CSRF_TOKEN . "=". $data[C\CSRF_TOKEN] .
diff --git a/src/views/elements/MoreoptionsElement.php b/src/views/elements/MoreoptionsElement.php
index bfae5f6..d9bf6f0 100644
--- a/src/views/elements/MoreoptionsElement.php
+++ b/src/views/elements/MoreoptionsElement.php
@@ -178,6 +178,7 @@ class MoreoptionsElement extends Element
         if (empty($token_query))  {
             $suggest_url = B\suggestUrl();
             $pages_url = B\wikiUrl('pages');
+            $trending_url = B\controllerUrl('trending');
         } else {
             $suggest_url = B\suggestUrl(true) . $token_query;
             $pages_url = B\wikiUrl('pages', true) . $token_query;
@@ -187,6 +188,7 @@ class MoreoptionsElement extends Element
             $tools[$suggest_url] = tl('moreoptions_element_suggest');
         }
         $tools[$pages_url] = tl('moreoptions_element_wiki_pages');
+        $tools[$trending_url] = tl('moreoptions_element_trending');
         if ($tools != []) {
             $max_column_num = 10;
             if ($_SERVER["MOBILE"]) {
@@ -224,4 +226,4 @@ class MoreoptionsElement extends Element
             <?php
         }
     }
-}
\ No newline at end of file
+}
diff --git a/src/views/elements/TrackerElement.php b/src/views/elements/TrackerElement.php
new file mode 100644
index 0000000..f849afb
--- /dev/null
+++ b/src/views/elements/TrackerElement.php
@@ -0,0 +1,114 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2019  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license https://www.gnu.org/licenses/ GPL3
+ * @link https://www.seekquarry.com/
+ * @copyright 2009 - 2019
+ * @filesource
+ */
+namespace seekquarry\yioop\views\elements;
+
+use seekquarry\yioop as B;
+use seekquarry\yioop\configs as C;
+use seekquarry\yioop\library as L;
+
+/**
+ * Element responsible for drawing the page tracker info
+ *
+ * @author Chris Pollett
+ */
+class TrackerElement extends Element
+{
+    /**
+     * Method responsible for drawing the page with tracker info
+     *
+     * @param array $data to draw tables on page
+     */
+    public function render($data)
+    {
+		$date_map = [
+			C\ONE_HOUR => 'hourly',
+			C\ONE_DAY => 'daily',
+			C\ONE_WEEK => 'weekly',
+		];?>
+		<style>
+			.container {
+				display:flex;
+			}
+			.floatLeft {
+				width: 36%;
+				float: left;
+			}
+			.floatMiddle {
+				width: 33%;
+				float: left;
+			}
+			.floatRight {
+				width: 33%;
+				float: left;
+			}
+			table{
+				border: 1px solid black;
+			}
+			tr {
+				border: 1px solid black;
+			}
+			th {
+				border: 1px solid black;
+				font-weight: normal;
+			}
+		</style>
+		<p><b>Trending Data for Words and Phrases<b></p>
+		<br>
+		<?php
+		if(isset($data['TREND_DATA'])){
+			?><div class="container"><?php
+			foreach($data['TREND_DATA'] as $period => $occurrences){
+				?>
+				<div class="floatLeft">
+					<p><b>The top <?php echo $date_map[$period]?></b></p>
+					<table>
+						<tr style="border: 1px solid black;">
+						<th style="border: 1px solid black;">Term</th>
+						<th style="border: 1px solid black;">Occurrences</th>
+						</tr>
+				<?php
+				foreach($occurrences as $i=>$item){
+					?>
+					<tr style="border: 1px solid black;">
+					<th style="border: 1px solid black;"><a href="/?q=<?php echo $item['TERM']?>"><?php echo $item['TERM']?></a></th>
+					<th style="border: 1px solid black;"><?php echo $item['OCCURRENCE']?></th>
+					</tr>
+					<?php
+				}
+				?></table>
+				</div>
+				<?php
+			}?>
+			</div><br>
+			<p>Data grabbed at <?php echo date('m/d/Y h:i:s' )?> </p> <?php
+		}
+    }
+}
-- 
2.5.0.windows.1

