_binary From 852e7df814aa59c7c7cd3443e59436b0f3c1802f Mon Sep 17 00:00:00 2001
From: Anirudh Mallya <anirudh.mallya09@gmail.com>
Date: Mon, 27 Dec 2021 01:21:06 -0800
Subject: [PATCH] Hash2Vec Recommendaton, author=aniruddha

---
 src/configs/Config.php                       |   4 +
 src/library/media_jobs/RecommendationJob.php | 299 ++++++++++++++++++-
 2 files changed, 301 insertions(+), 2 deletions(-)

diff --git a/src/configs/Config.php b/src/configs/Config.php
index 1b4f74db6..1d957e00d 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -1208,3 +1208,7 @@ nsconddefine('AD_LOGO','resources/adv-logo.png');
 nsconddefine('SENTENCE_COMPRESSION_ENABLED', false);
 /** The number of rows to be used in bulk insert from Lexicon */
 nsconddefine('NUM_LEX_BULK_INSERTS',100000);
+/**
+ * Type used to indicate recommendation scheme
+ */
+nsdefine('RECOMMENDATION_TYPE', 1);
\ No newline at end of file
diff --git a/src/library/media_jobs/RecommendationJob.php b/src/library/media_jobs/RecommendationJob.php
index e7ce93e80..32dbbccf6 100644
--- a/src/library/media_jobs/RecommendationJob.php
+++ b/src/library/media_jobs/RecommendationJob.php
@@ -200,6 +200,9 @@ class RecommendationJob extends MediaJob
         $this->computeUserItemIdf($number_items, $number_users);
         $this->tfIdfUsers();
         $this->tfIdfItems();
+        if (C\RECOMMENDATION_TYPE == 1) {
+            $this->hash2Vec();
+        }
         $this->computeUserItemSimilarity();
         $not_belongs_subselect =  "NOT EXISTS (SELECT * FROM ".
             "GROUP_ITEM B WHERE S.USER_ID=B.USER_ID ".
@@ -491,12 +494,53 @@ class RecommendationJob extends MediaJob
     {
         L\crawlLog("...Computing User Item Similarity Scores.");
         $db = $this->db;
+        $user_weight_table = "USER_TERM_WEIGHTS";
+        if (C\RECOMMENDATION_TYPE == 1) {
+            // USER_TERM_WEIGHTS_HASH2VEC table created to store
+            // USER_TERM_WEIGHTS table values and add hash2vec score to it
+            $this->db->execute("CREATE TABLE IF NOT EXISTS
+                USER_TERM_WEIGHTS_HASH2VEC (
+                TERM_ID BIGINT,
+                USER_ID BIGINT,
+                WEIGHT FLOAT,
+                PRIMARY KEY(TERM_ID, USER_ID));
+                INSERT INTO USER_TERM_WEIGHTS_HASH2VEC
+                SELECT * FROM USER_TERM_WEIGHTS;");
+            $this->db->execute("UPDATE
+                USER_TERM_WEIGHTS_HASH2VEC
+                SET WEIGHT=IFNULL((
+                SELECT SUM(Hash2Vec.SCORE*UTW.WEIGHT)
+                FROM HASH2VEC_TERM_SIMILARITY AS Hash2Vec
+                INNER JOIN USER_TERM_WEIGHTS AS UTW
+                ON Hash2Vec.Term2 = UTW.TERM_ID
+                WHERE Hash2Vec.Term1 =
+                USER_TERM_WEIGHTS_HASH2VEC.TERM_ID
+                ),0);");
+            $this->db->execute("UPDATE
+                USER_TERM_WEIGHTS_HASH2VEC
+                SET WEIGHT=WEIGHT+IFNULL((
+                SELECT UTW.WEIGHT FROM USER_TERM_WEIGHTS AS UTW
+                WHERE USER_TERM_WEIGHTS_HASH2VEC.TERM_ID =
+                UTW.TERM_ID
+                AND USER_TERM_WEIGHTS_HASH2VEC.USER_ID =
+                UTW.USER_ID),0);");
+            $this->db->execute("CREATE TABLE  IF NOT EXISTS
+                USER_ITEM_SIMILARITY_HASH2VEC (
+                USER_ID BIGINT,
+                THREAD_ID BIGINT,
+                SIMILARITY FLOAT,
+                GROUP_MEMBER BIGINT,
+                PRIMARY KEY(USER_ID, THREAD_ID));
+                INSERT INTO USER_ITEM_SIMILARITY_HASH2VEC
+                SELECT * FROM USER_ITEM_SIMILARITY;");
+            $user_weight_table = "USER_TERM_WEIGHTS_HASH2VEC";
+        }
         $similarity_parts_sql =
             "SELECT SUM(UTW.WEIGHT * ITW.WEIGHT) AS THREAD_DOT_USER, ".
             "SUM(UTW.WEIGHT * UTW.WEIGHT) AS USER_MAG," .
             "SUM(ITW.WEIGHT * ITW.WEIGHT) AS ITEM_MAG," .
             "GI.PARENT_ID AS THREAD_ID, UTW.USER_ID AS USER_ID ".
-            "FROM ITEM_TERM_WEIGHTS ITW, USER_TERM_WEIGHTS UTW, GROUP_ITEM GI ".
+            "FROM ITEM_TERM_WEIGHTS ITW, $user_weight_table UTW, GROUP_ITEM GI ".
             "WHERE GI.ID = ITW.ITEM_ID AND UTW.TERM_ID=ITW.TERM_ID " .
             "GROUP BY UTW.USER_ID, GI.PARENT_ID";
         $similarity_parts_result = $db->execute($similarity_parts_sql);
@@ -618,4 +662,255 @@ class RecommendationJob extends MediaJob
             $db->execute($insert_ignore_sql);
         }
     }
-}
+    public $WORD_SPLIT = "/([.,!?\"':;)(])+/";
+    public $NB_DIMS = 200;
+    public $CTX_WIDTH = 5;
+    public $dic = [];
+    public $vec = [];
+    public $rev_dic = [];
+    public $nbWords = 0;
+    public $sigma = 0.0;
+    public $w;
+    public function hash2Vec()
+    {
+        $db = $this->db;
+        $db->execute("CREATE TABLE IF NOT EXISTS HASH2VEC_TERM_SIMILARITY (
+                    TERM1 BIGINT,
+                    TERM2 BIGINT,
+                    SCORE FLOAT,
+                    PRIMARY KEY(TERM1, TERM2)
+                    );");
+        $data = "";
+        $group_item_sql = "SELECT ID AS ITEM_ID, TITLE, DESCRIPTION ".
+            "FROM GROUP_ITEM ".
+            "WHERE LOWER(TITLE) NOT LIKE '%page%'" .
+            "AND LOWER(DESCRIPTION) NOT LIKE '%-0700%'" .
+            "ORDER BY PUBDATE DESC " . $db->limitOffset(self::MAX_GROUP_ITEMS);
+        $results = $db->execute($group_item_sql);
+        while ($item = $db->fetchArray($results)) {
+            $data .= $item['TITLE'] . "\n";
+            $data .= $item['DESCRIPTION'];
+        }
+        for ($i=0; $i<$this->CTX_WIDTH; $i++) {
+            $distVec[] = -$i + $this->CTX_WIDTH;
+        }
+        for ($i=0; $i<$this->CTX_WIDTH; $i++) {
+            $distVec[] = $i;
+        }
+        $this->std = $this->stats_standard_deviation($distVec);
+        $this->genVec($data, $this->WORD_SPLIT);
+    }
+
+    public function similarityEval() {
+        $db=$this->db;
+        $base_sql = "INSERT INTO HASH2VEC_TERM_SIMILARITY VALUES";
+        $insert_sql = $base_sql;
+        $insert_count = 0;
+        $comma = "";
+        foreach ($this->rev_dic as $ind => $word) {
+            $simW = $this->getSimilar($word);
+            if (!empty($simW)) {
+                $word_of_interest = floor(bindec(str_replace(" ", "",
+                    L\toBinString(hash("crc32b", strtolower($word), true))))/2);
+                $db->execute("DELETE FROM HASH2VEC_TERM_SIMILARITY WHERE
+                    TERM1 = " . $word_of_interest);
+                foreach ($simW as $item) {
+                    $term = floor(bindec(str_replace(" ", "", L\toBinString(
+                        hash("crc32b", $item[0], true))))/2);
+                    $hash_score = $item[1];
+                    $insert_sql .= "$comma ($word_of_interest, $term,
+                        $hash_score)";
+                    $comma = ",";
+                    $insert_count++;
+                    if ($insert_count == self::BATCH_SQL_INSERT_NUM) {
+                        $insert_ignore_sql = $db->insertIgnore($insert_sql);
+                        $db->execute($insert_ignore_sql);
+                        $insert_sql = $base_sql;
+                        $insert_count = 0;
+                        $comma = "";
+                    }
+                }
+            }
+        }
+        if ($insert_count > 0) {
+            $insert_ignore_sql = $db->insertIgnore($insert_sql);
+            $db->execute($insert_ignore_sql);
+        }
+    }
+
+    public function stats_standard_deviation($ary)
+    {
+        $avg = round(array_sum($ary) / count($ary), 1);
+        $diff_ary = [];
+        foreach ($ary as $val) {
+            $diff = $val - $avg;
+            $diff_ary[] = pow($diff, 2);
+        }
+        $diff_total = array_sum($diff_ary);
+        $diff_avg = $diff_total / count($diff_ary);
+        $stdev = sqrt($diff_avg);
+        return $stdev;
+    }
+
+    public function genVec($data, $pattern)
+    {
+        $lines = explode("\n", strtolower($data));
+        foreach ($lines as $ln) {
+            $ln = preg_replace("/[\n\r]/",'',$ln);
+            if (strlen($ln) == 0) continue;
+            $words = explode(" ", strtolower($ln));
+            if (count($words) <= 1) continue;
+            $words2 = [];
+            foreach ($words as $w) {
+                if ($w){
+                    $clean_word = preg_replace($pattern, '', $w);
+                    $words2[] = $clean_word;
+                }
+            }
+            $wordids = [];
+            foreach ($words2 as $w) {
+                $wordids[] = $this->wordToId($w);
+            }
+            $pos = 0;
+            foreach ($wordids as $w) {
+                list($cwl, $d) = $this->getContextWords($wordids, $pos);
+                $i = 0;
+                foreach ($cwl as $cw) {
+                    $power = pow($d[$i] / $this->std, 2);
+                    $dist = exp(-1 * $power);
+                    list($hid, $sign) = $this->getHash($this->rev_dic[$cw]);
+                    $this->vec[$w][$hid] = $this->vec[$w][$hid] + $sign * $dist;
+                    $i += 1;
+                }
+                $pos += 1;
+            }
+        }
+        $this->normalize();
+        $this->similarityEval();
+    }
+
+    public function wordToId($w)
+    {
+        if (!array_key_exists($w, $this->dic)) {
+            $this->dic[$w] = $this->nbWords;
+            $this->rev_dic[$this->nbWords] = $w;
+            $this->vec[] = array_fill(0,$this->NB_DIMS,0);
+            $this->nbWords += 1;
+        }
+        return $this->dic[$w];
+    }
+    public function getContextWords($words, $pos) {
+        $spos = 0;
+        $epos = $pos + 1 + $this->CTX_WIDTH;
+        if ($pos > $this->CTX_WIDTH) $spos = $pos - $this->CTX_WIDTH;
+        if ($epos > count($words)) $epos = count($words);
+        $arr = array_slice($words,$spos,$pos-$spos);
+        if ($pos <= $this->CTX_WIDTH) {
+            $arr2 = array_slice($words,$pos+1,$epos-$pos-$spos-1);
+        } else if ($pos >= (count($words)-$this->CTX_WIDTH)) {
+            $arr2 = array_slice($words,$pos+1,$epos-$pos-1);
+        }
+        else {
+            $arr2 = array_slice($words,$pos+1, $this->CTX_WIDTH);
+        }
+        $cw = array_merge($arr, $arr2);
+        if (($pos - $spos) == 0) {
+            $arr = [];
+        } else {
+            $arr = range(1, $pos - $spos);
+        }
+        if ($pos <= $this->CTX_WIDTH) {
+            if ($epos-$pos-$spos-1 == 0) {
+                $arr2 = [];
+            } else {
+                $arr2 = range($epos-$pos-$spos-1, 1, -1);
+            }
+        } else if ($pos >= (count($words)-$this->CTX_WIDTH)) {
+            if ($pos == count($words) - 1) {
+                $arr2 = [];
+            }
+        } else {
+            $arr2 = range($this->CTX_WIDTH, 1, -1);
+        }
+        $d = array_merge($arr, $arr2);
+        assert(count($d) == count($cw));
+        return [$cw, $d];
+    }
+
+    public function getHash($word)
+    {
+        $val = unpack("N", substr(md5($word), 0, 4))[1];
+        $hid = $val % $this->NB_DIMS;
+        $hsn = $val % 2;
+        $sign = -1;
+        if ($hsn) $sign = 1;
+        return [$hid, $sign];
+    }
+
+    public function normalize() {
+        for ($i = 0; $i < count($this->vec); $i++) {
+            $v = [];
+            foreach ($this->vec[$i] as $val) {
+                $v[] = abs($val);
+            }
+            $vsum = array_sum($v);
+            foreach ($v as $index=>$val) {
+                if ($vsum == 0) $vsum = 1;
+                $this->vec[$i][$index] = $val * 1. / $vsum;
+            }
+        }
+    }
+
+    public function insert($index, $val, $top10) {
+        for ($i=0; $i<count($top10); $i++) {
+            if ($top10[$i][0] < $val) {
+                $top10[$i] = [$val, $index];
+                break;
+            }
+        }
+        return;
+    }
+
+    public function getSimilar($sw) {
+        $sw = strtolower($sw);
+        if (!array_key_exists($sw, $this->dic)) {
+            return NULL;
+        }
+        $pos = $this->dic[$sw];
+        $swVec = $this->vec[$pos];
+        $nbTop = 0;
+        $den1 = 1;
+        $top10 = [];
+        foreach ($this->vec as $index => $v) {
+            if ($index == $pos) continue;
+            $sumV = [];
+            foreach ($v as $ve) {
+                $sumV[] = pow($ve, 2);
+            };
+            $arrSum = array_sum($sumV);
+            if ($arrSum == 0) $arrSum = 1;
+            $den2 = sqrt($arrSum);
+            $simVal = $this->dot_product($swVec, $v) / ($den1 * $den2);
+            if ($nbTop < 10) {
+                array_push($top10, [$simVal, $index]);
+                $nbTop += 1;
+            } else {
+                $this->insert($index, $simVal, $top10);
+            }
+        }
+        rsort($top10);
+        $similarTerms = [];
+        foreach ($top10 as $elm) {
+            array_push($similarTerms, [$this->rev_dic[$elm[1]], $elm[0]]);
+        }
+        return $similarTerms;
+    }
+
+
+    function dot_product($vect_A, $vect_B) {
+        $product = 0;
+        for ($i = 0; $i < count($vect_A); $i++)
+            $product = $product + $vect_A[$i] * $vect_B[$i];
+        return $product;
+    }
+}
\ No newline at end of file
-- 
2.29.2.windows.3

