_binary From 3991fb7ec8654e87b966f69ec8d6bf1b12ac8c8f Mon Sep 17 00:00:00 2001
From: Shailesh Padave <shaileshpadave49@gmail.com>
Date: Tue, 6 May 2014 21:27:18 -0700
Subject: [PATCH] Added WordNet Feature

---
 configs/config.php                         |   2 +
 configs/configure_tool.php                 |   2 +-
 controllers/components/crawl_component.php |   3 +-
 controllers/search_controller.php          |   3 +
 css/search.css                             |  15 +-
 lib/calculate_BM25.php                     | 197 ++++++++++++++++++++
 lib/crawl_constants.php                    |   2 +
 lib/part_of_speech_tagging.php             | 170 +++++++++++++++++
 lib/phrase_parser.php                      | 131 ++++++++++++++
 lib/process_WordNet.php                    | 282 +++++++++++++++++++++++++++++
 models/model.php                           |   2 +-
 models/phrase_model.php                    |  56 +++++-
 models/profile_model.php                   |   3 +-
 views/elements/pageoptions_element.php     |  20 ++
 views/search_view.php                      |  18 ++
 15 files changed, 898 insertions(+), 8 deletions(-)
 create mode 100644 lib/calculate_BM25.php
 create mode 100644 lib/part_of_speech_tagging.php
 create mode 100644 lib/process_WordNet.php

diff --git a/configs/config.php b/configs/config.php
index 222c0d5..7755983 100755
--- a/configs/config.php
+++ b/configs/config.php
@@ -193,6 +193,8 @@ if(file_exists(WORK_DIRECTORY.PROFILE_FILE_NAME)) {
     define('SIMILAR_LINK', true);
     define('IN_LINK', true);
     define('IP_LINK', true);
+    define('USE_WORDNET', false);
+    define('WORDNET_DIR', '');
     define('SIGNIN_LINK', true);
     define('NEWS_MODE', 'news_off');
     /** BM25F weight for title text */
diff --git a/configs/configure_tool.php b/configs/configure_tool.php
index a54cde0..aab0192 100644
--- a/configs/configure_tool.php
+++ b/configs/configure_tool.php
@@ -342,7 +342,7 @@ class ConfigureTool
             "SUBSEARCH_LINK"  => "Subsearch Links",
             "SIGNIN_LINK" => "Sign-in Links", "CACHE_LINK" => "Cache Link",
             "SIMILAR_LINK" => "Similar Link", "IN_LINK" => "Inlinks",
-            "IP_LINK"=> "IP Links");
+            "IP_LINK"=> "IP Links", "USE_WORDNET"=> "WordNet");
         $items = array();
         foreach($settings as $setting => $setting_string) {
             $toggle = ($data[$setting]) ? "On" : "Off";
diff --git a/controllers/components/crawl_component.php b/controllers/components/crawl_component.php
index 31fffbf..721d435 100644
--- a/controllers/components/crawl_component.php
+++ b/controllers/components/crawl_component.php
@@ -951,7 +951,8 @@ class CrawlComponent extends Component implements CrawlConstants
         } else {
             $parent->updateProfileFields($data, $profile,
                 array('IP_LINK','CACHE_LINK', 'SIMILAR_LINK', 'IN_LINK',
-                    'SIGNIN_LINK', 'SUBSEARCH_LINK','WORD_SUGGEST'));
+                    'SIGNIN_LINK', 'SUBSEARCH_LINK','WORD_SUGGEST',
+                    'USE_WORDNET', 'WORDNET_DIR'));
         }
         $weights = array('TITLE_WEIGHT' => 4,
             'DESCRIPTION_WEIGHT' => 1, 'LINK_WEIGHT' => 2,
diff --git a/controllers/search_controller.php b/controllers/search_controller.php
index effdd21..c5e27b8 100755
--- a/controllers/search_controller.php
+++ b/controllers/search_controller.php
@@ -747,6 +747,9 @@ class SearchController extends Controller implements CrawlConstants
         $data['RAW'] = $raw;
         $data['PAGES'] = (isset($phrase_results['PAGES'])) ?
              $phrase_results['PAGES']: array();
+        $data['WORDNET_SIMILAR_WORDS'] =
+            (isset($phrase_results['WORDNET_SIMILAR_WORDS'])) ?
+             $phrase_results['WORDNET_SIMILAR_WORDS']: array();
         $data['SAVE_POINT'] = (isset($phrase_results["SAVE_POINT"])) ?
              $phrase_results["SAVE_POINT"]: array( 0 => 1);
         if(isset($phrase_results["HARD_QUERY"])) {
diff --git a/css/search.css b/css/search.css
index 14b6b0f..98a6312 100755
--- a/css/search.css
+++ b/css/search.css
@@ -1125,9 +1125,9 @@ ul.in-list li
 
 .html-ltr .serp-results
 {
-    left: 0.5in;
+    left: 2.2in;
     position: relative;
-    top: -0.8in;
+    top: -0.9in;
     width: 8in;
 }
 
@@ -1169,7 +1169,16 @@ ul.in-list li
     position: relative;
     top: -1in;
 }
-
+.html-ltr .wordnet
+{
+    color: #666;
+    font-size: 13pt;
+    font-weight: bold;
+    left: -0.2in;
+    position: relative;
+    top: -0.8in;
+    width: 8in;
+}
 .result
 {
     clear: both;
diff --git a/lib/calculate_BM25.php b/lib/calculate_BM25.php
new file mode 100644
index 0000000..d91bff5
--- /dev/null
+++ b/lib/calculate_BM25.php
@@ -0,0 +1,197 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2014  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Shailesh Padave shaileshpadave49@gmail.com
+ * @package seek_quarry
+ * @subpackage library
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2014
+ * @filesource
+ */
+/**
+ * This class is used to rank the matching document from the generated summary
+ * during crawl using BM25 raking function.
+ * @package seek_quarry
+ * @subpackage library
+ */
+class BM25Calculator
+{
+    /**
+     * Gets array of BM25 score for given input array of summaries
+     * @param array $input_array an array of summaries which is generated
+     *      during crawl time.
+     * @param array $similarWords an array of synonyms which is generated by
+     *      WordNet.
+     *@returns array of BM25 score for each document
+     */
+    function getScore($input_array,$similarWords)
+    {
+        //if there are no similar words then
+        if(empty($similarWords)){
+            return;
+        }
+        else{
+                for($i=0;$i<count($similarWords);$i++){
+                    $query = $similarWords[$i];
+                    $input_array = self::changeCaseOfArray($input_array);
+                    $query = mb_strtolower($query);
+                    $IDF = self::calculateIDF($input_array,$query);
+                    $TF = self::calculateTF($input_array,$query);
+                    $count_inputArray = count($input_array);
+                    $count_query=count(explode(' ',$query));
+                    $BM25result[$i] =
+                        self::calculateBM25($IDF, $TF, $count_query,
+                        $count_inputArray);
+                }
+                for($i = 0;$i < $count_inputArray;$i++){
+                    $temp = 0;
+                    $temp = $BM25result[0][$i]*(2/3) + $BM25result[1][$i]*(1/3);
+                    $finalBM25Score[$i] = $temp;
+                }
+                return $finalBM25Score;
+            }
+        }
+    /**
+     * To make lower case for all the input variables
+     * @param array $input_array an array of summaries
+     * @returns array of with strings converted to
+     * lower case
+     */
+    function changeCaseOfArray($input_array)
+    {
+        for($i=0;$i<count($input_array);$i++)
+        {
+            $input_array[$i] = mb_strtolower($input_array[$i],'UTF-8');
+        }
+        return $input_array;
+    }
+    /**
+     * To get the total number of characters in input array
+     * @param array $input_array an array of summaries
+     * @returns number of characters in input array
+     */
+    function getlength($input_array)
+    {
+        $cnt_input_array = count($input_array);
+        $str_len =0;
+        for($i=0;$i<$cnt_input_array;$i++)
+        {
+            $str = $input_array[$i];
+            $str_len = $str_len + strlen($str);
+        }
+        return $str_len;
+    }
+    /**
+     * To get the term frequency in given summary array
+     * @param array $input_array an array of summaries
+     * @param array $query an array of words generated by wordnet output
+     * @returns array of term frequency for each query term
+     */
+    function calculateTF($input_array,$query)
+    {
+        $k1=1.5;
+        $b=0.75;
+        $l_d = self::getlength($input_array);
+        $l_avg = $l_d / count($input_array);
+        $query_array = explode(' ',$query);
+        $tf_array = self::calculateWordFreq($input_array,$query_array);
+        for($i=0;$i<count($query_array);$i++){
+            for($j=0;$j<count($input_array);$j++){
+                $temp_freq=$tf_array[$i][$j];
+                $tfbm25[$i][$j] =
+                ($temp_freq*($k1+1))/($temp_freq+$k1*((1-$b)+$b*($l_d/$l_avg)));
+            }
+        }
+        return $tfbm25;
+    }
+    function calculateWordFreq($input_array,$query_array)
+    {
+        for($i=0;$i<count($query_array);$i++){
+            for($j=0;$j<count($input_array);$j++){
+                  $freq = substr_count($input_array[$j],$query_array[$i]);
+                  $tf_array[$i][$j] = $freq;
+            }
+        }
+        return $tf_array;
+    }
+    /**
+     * To get the BM25 Score in given summary array
+     * @param array $IDF inverse Doc frequency for given query array
+     * @param array $TF term frequency for given query array
+     * @param $count_query count for number of queries
+     * @param $count_inputArray count for input summaries
+     * @returns array of BM25 score for each document
+     */
+    function calculateBM25($IDF,$TF,$count_query,$count_inputArray)
+    {
+        for($i=0;$i<$count_query;$i++){
+            for($j=0;$j<$count_inputArray;$j++){
+                $scoreBM25[$i][$j] = $IDF[$i] * $TF[$i][$j];
+            }
+        }
+        for($i=0;$i<$count_inputArray;$i++){
+            $val = 0;
+            for($j=0;$j<$count_query;$j++){
+                $val += $scoreBM25[$j][$i];
+            }
+            $score[$i] = $val;
+        }
+        return $score;
+    }
+    /**
+     * To get the Inverse Doc frequency in given summary array
+     * @param array $input_array an array of summaries
+     * @param array $query an array of words generated by wordnet output
+     * @returns array of Inverse Doc frequency for each query term
+     */
+    function calculateIDF($input_array,$query)
+    {
+        $N=count($input_array);
+        $query_array = explode(' ',$query);
+        $Nt=array();
+        $query_array_cnt=0;
+        for($i=0;$i<count($query_array);$i++){
+            $cnt_Nt=0;
+            $query_array_cnt++;
+            foreach($input_array as $value)
+            {
+                $pattern = "/".$query_array[$i]."/";
+                if(preg_match($pattern,$value)){
+                    $cnt_Nt++;
+                }
+            }
+            $Nt[$i]=$cnt_Nt;
+            if($Nt[$i] != 0){
+                $IDF[$i] = log10($N/$Nt[$i]);
+            }
+            else
+            {
+                $IDF[$i] = 0;
+            }
+        }
+        return $IDF;
+    }
+}
+?>
\ No newline at end of file
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
index 3bce0d3..ae5a6a0 100755
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -249,5 +249,7 @@ interface CrawlConstants
     const TOR_PROXY = 'dh';
     const PROXY_SERVERS = 'di';
     const NEEDS_OFFSET_FLAG = 0x7FFFFFFF;
+    const BM25_SCORE ='do';
+    const WORDNET_SIMILAR_WORDS='dp';
 }
 ?>
\ No newline at end of file
diff --git a/lib/part_of_speech_tagging.php b/lib/part_of_speech_tagging.php
new file mode 100644
index 0000000..da9cfd5
--- /dev/null
+++ b/lib/part_of_speech_tagging.php
@@ -0,0 +1,170 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2014  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Shailesh Padave shaileshpadave49@gmail.com
+ * @package seek_quarry
+ * @subpackage library
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2014
+ * @filesource http://phpir.com/part-of-speech-tagging
+ */
+/**
+ * This class is used to get part of speech tagging for given input.
+ * @package seek_quarry
+ * @subpackage library
+ */
+class PartOfSpeechTagger
+{
+    private $dictionary;
+    public function __construct($lexicon)
+    {
+        $fh = fopen($lexicon, 'r');
+        while($line = fgets($fh)) {
+                $tags = explode(' ', $line);
+                $this->dictionary[strtolower(array_shift($tags))] = $tags;
+        }
+        fclose($fh);
+    }
+    public function tag($text)
+    {
+        preg_match_all("/[\w\d]+/", $text, $matches);
+        $nouns = array('NN', 'NNS');
+        $result = array();
+        $i = 0;
+        foreach($matches[0] as $token) {
+            // default to a common noun
+            $result[$i] = array('token' => $token, 'tag' => 'NN');
+            // remove trailing full stops
+            if(substr($token, -1) == '.') {
+                    $token = preg_replace('/\.+$/', '', $token);
+            }
+            // get from dictionary if set
+            if(isset($this->dictionary[strtolower($token)])) {
+                    $result[$i]['tag'] =
+                        $this->dictionary[strtolower($token)][0];
+            }
+            // Converts verbs after 'the' to nouns
+            if($i > 0) {
+                    if($result[$i - 1]['tag'] == 'DT' &&
+                             in_array($result[$i]['tag'],
+                                            array('VBD', 'VBP', 'VB'))) {
+                            $result[$i]['tag'] = 'NN';
+                    }
+            }
+            // Convert noun to number if . appears
+            if($result[$i]['tag'][0] == 'N' && strpos($token, '.') !== false) {
+                    $result[$i]['tag'] = 'CD';
+            }
+            // Convert noun to past particile if ends with 'ed'
+            if($result[$i]['tag'][0] == 'N' && substr($token, -2) == 'ed') {
+                    $result[$i]['tag'] = 'VBN';
+            }
+            // Anything that ends 'ly' is an adverb
+            if(substr($token, -2) == 'ly') {
+                    $result[$i]['tag'] = 'RB';
+            }
+            // Common noun to adjective if it ends with al
+            if(in_array($result[$i]['tag'], $nouns)
+                                    && substr($token, -2) == 'al') {
+                    $result[$i]['tag'] = 'JJ';
+            }
+            // Noun to verb if the word before is 'would'
+            if($i > 0) {
+                    if($result[$i]['tag'] == 'NN'
+                            && strtolower($result[$i-1]['token']) == 'would') {
+                            $result[$i]['tag'] = 'VB';
+                    }
+            }
+            // Convert noun to plural if it ends with an s
+            if($result[$i]['tag'] == 'NN' && substr($token, -1) == 's') {
+                    $result[$i]['tag'] = 'NNS';
+            }
+            // Convert common noun to gerund
+            if(in_array($result[$i]['tag'], $nouns)
+                            && substr($token, -3) == 'ing') {
+                    $result[$i]['tag'] = 'VBG';
+            }
+            /* If we get noun noun, and the second can be a verb,
+             * convert to verb
+             */
+            if($i > 0) {
+                if(in_array($result[$i]['tag'], $nouns)
+                        && in_array($result[$i-1]['tag'], $nouns)
+                        && isset($this->dictionary[strtolower($token)])) {
+                    if(in_array('VBN', $this->dictionary[strtolower($token)])) {
+                            $result[$i]['tag'] = 'VBN';
+                    } else if(in_array('VBZ',
+                            $this->dictionary[strtolower($token)])) {
+                            $result[$i]['tag'] = 'VBZ';
+                    }
+                }
+            }
+            $i++;
+        }
+        return $result;
+    }
+    /* Function for print the results
+     * @param $tags input string on which Part of speech processing is performed
+     * @return $output_string with customized part of speech for query input
+     */
+    function printTag($tags)
+    {
+        $output_string = null;
+        $noun_arr = array("NN","NNS","NNP","NNPS","PRP","PRP$","WP","WP");
+        $verb_arr = array("VB","VBD","VBG","VBN","VBP","VBZ");
+        $adj_arr = array("JJ","JJR","JJS");
+        $adv_arr = array("RB","RBR","RBS","WRB");
+        foreach($tags as $t) {
+            if(in_array(trim($t['tag']),$noun_arr)==1)
+                $output_string =
+                    $output_string . $t['token'] . "~" . "NN" .  " ";
+            else if(in_array(trim($t['tag']),$verb_arr)==1)
+                $output_string =
+                    $output_string . $t['token'] . "~" . "VB" .  " ";
+            else if(in_array(trim($t['tag']),$adj_arr)==1)
+                $output_string =
+                    $output_string . $t['token'] . "~" . "AJ" .  " ";
+            else if(in_array(trim($t['tag']),$adv_arr)==1)
+                $output_string =
+                    $output_string . $t['token'] . "~" . "AV" .  " ";
+            else $output_string =
+                $output_string . $t['token'] . "~" . $t['tag'] .  " ";
+        }
+        return $output_string;
+    }
+    /* Function for part of speech for given query
+     * @param $string input string from user
+     * @return $output_string with part of speech processed on query input
+     */
+    function getPartOfSpeechTagging($string)
+    {
+        // Initialize the directory named as lexicon.txt
+        $tagger = new PartOfSpeechTagger(BASE_DIR . "/lib/lexicon.txt");
+        $tags = $tagger->tag($string);
+        $output_string  = self::printTag($tags);
+        return $output_string;
+    }
+}
+?>
\ No newline at end of file
diff --git a/lib/phrase_parser.php b/lib/phrase_parser.php
index cb2ff9c..14fe94a 100755
--- a/lib/phrase_parser.php
+++ b/lib/phrase_parser.php
@@ -71,6 +71,15 @@ require_once BASE_DIR."/lib/index_manager.php";
  */
 require_once BASE_DIR."/lib/suffix_tree.php";
 
+/**
+ * Get the part of speech tagging for given sentence
+ */
+require_once BASE_DIR . "/lib/part_of_speech_tagging.php";
+
+/**
+ * Get the similar words from wordnet for given query
+ */
+require_once BASE_DIR . "/lib/process_WordNet.php";
 
 /**
  * Library of functions used to manipulate words and phrases
@@ -1171,4 +1180,126 @@ vaffanculo fok hoer kut lul やりまん 打っ掛け
         $score = $num_unsafe_terms * $unsafe_count/($len + 1);
         return $score;
     }
+    /**
+     *  Extracts similar words from wordnet output for given query.
+     *  Part of speech tagging processed on input, output is given to wordnet
+     *  processing. Wordnet function provides list of words as per ranking.
+     *  For those words, count will be calculated from posting list and top 2
+     *  words are selected.
+     *  @param string $orig_query input query from user
+     *  @param string $index_name selected index for search engine
+     *  @param string $lang language
+     *  @param integer $threshold once count in posting list for any word
+     *      reaches to threshold then return the number
+     *  @return array of top two words
+     */
+    function getSimilarWords($orig_query, $index_name, $lang, $threshold = 10)
+    {
+        $replacements = array();
+        $string = PartOfSpeechTagger::getPartOfSpeechTagging($orig_query);
+        $replacements =
+        WordNetProcessor::processWordNetOutput(trim($orig_query),trim($string));
+        $i = 0;
+        $cnt = 0;
+        foreach ($replacements as $replacement) {
+            $cnt =
+            self::compute_score($replacement, $threshold, $index_name, $lang);
+            if($cnt)
+            {
+                $countNonZeroScore[$replacement] = $cnt;
+            }
+            else
+            {
+                $countZeroScore[$replacement] = $cnt;
+            }
+            $i++;
+        }
+        $word = array();
+        if(count($countNonZeroScore) >= 2)
+        {
+            $i = 0;
+            arsort($countNonZeroScore);
+            foreach($countNonZeroScore as $k=>$v)
+            {
+                $word[$i] = $k;
+                $i++;
+                if($i==2)
+                    break;
+            }
+        }
+        elseif(count($countNonZeroScore) == 1)
+        {
+            $i = 0;
+            foreach($countNonZeroScore as $k=>$v)
+            {
+                $word[$i] = $k;
+                $i++;
+                if($i==1)
+                    break;
+            }
+            foreach($countZeroScore as $k=>$v)
+            {
+                $word[$i] = $k;
+                $i++;
+                if($i==2)
+                    break;
+            }
+        }
+        else
+        {
+            $i = 0;
+            foreach($countZeroScore as $k=>$v)
+            {
+                $word[$i] = $k;
+                $i++;
+                if($i==2)
+                    break;
+            }
+        }
+    return $word;
+    }
+    /**
+     *  Computes count for each word in posting list
+     *  @param string $replacement input query part of speech tagging
+     *  @param integer $threshold once count in posting list for any word
+     *      reaches to threshold then return the number
+     *  @param string $index_name selected index for search engine
+     *  @param string $lang language
+     *  @return integer $count_whole_phrase in posting list for given word
+     */
+    function compute_score($replacement, $threshold, $index_name, $lang)
+    {
+        $string     = $replacement;
+        if (isset(self::$programming_language_map[$lang])) {
+            $control_word =
+            self::$programming_language_map[$lang].self::CONTROL_WORD_INDICATOR;
+            $string = trim(substr($string, strlen($control_word) + 1));
+        } else {
+            self::canonicalizePunctuatedTerms($string, $lang);
+        }
+        $terms = self::stemCharGramSegment($string, $lang);
+        $num   = count($terms);
+        /*  Changed as term size will not be zero.
+         *  if ($index_name == NULL || $num <= 1) {
+         */
+        if ($index_name == NULL) {
+            return $terms;
+        }
+        if (count($terms) > MAX_QUERY_TERMS) {
+            $first_terms  = array_slice($terms, 0, MAX_QUERY_TERMS);
+            $whole_phrase = implode(" ", $first_terms);
+        } else {
+            $whole_phrase = implode(" ", $terms);
+            $first_terms =& $terms;
+        }
+        if ($exact_match) {
+            return $terms;
+            /*  for exact phrase search do not use suffix tree
+             *  stuff for now
+            */
+        }
+        $count_whole_phrase =
+        IndexManager::numDocsTerm($whole_phrase, $index_name, $threshold);
+        return $count_whole_phrase;
+    }
 }
\ No newline at end of file
diff --git a/lib/process_WordNet.php b/lib/process_WordNet.php
new file mode 100644
index 0000000..a1a04fc
--- /dev/null
+++ b/lib/process_WordNet.php
@@ -0,0 +1,282 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2014  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @package seek_quarry
+ * @subpackage library
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2014
+ * @filesource
+ */
+class WordNetProcessor
+{
+    /**
+     *  Extracts the similar words from WordNet output generated on command line
+     *  It is stored in text file and used for parsing similar words and
+     *  sentences.
+     *  Sentences are then used in cosine ranking along with query string.
+     *  According to score, synonyms are selected.
+     *  @param String $query : Query entered by user
+     *  @param String $posQuery : Same query processed after Part of speech
+     *               tagging
+     *  @return String array $query with all possible replacements for
+     *             given query.
+     */
+    function processWordNetOutput($query, $posQuery)
+    {
+        $replacements = array();
+        // Get the query from user and do PartofSpeech Tagging on it
+        $queryWords = explode(" ", $query);
+        $output = $posQuery;
+        // Find out exact meaning of NN,JJ, in terms of Noun, Verb
+        $output_words = preg_split("/\s+/", $output, -1, PREG_SPLIT_NO_EMPTY);
+        $output_words_cnt = count($output_words);
+        $word_type = null;
+        $noun_arr = array(
+            "NN",
+            "NNS",
+            "NNP",
+            "NNPS",
+            "PRP",
+            "PRP$",
+            "WP",
+            "WP$"
+        );
+        $verb_arr = array(
+            "VB",
+            "VBD",
+            "VBG",
+            "VBN",
+            "VBP",
+            "VBZ"
+        );
+        $adj_arr = array(
+            "AJ",
+            "JJ",
+            "JJR",
+            "JJS"
+        );
+        $adv_arr = array(
+            "AV",
+            "RB",
+            "RBR",
+            "RBS",
+            "WRB"
+        );
+        $similar_words = array();
+        for ($i = 0; $i < $output_words_cnt; $i++) {
+            $pos = strpos($output_words[$i], '~');
+            $substring_output_word = substr($output_words[$i], $pos + 1);
+            $current_word = substr($output_words[$i], 0, $pos);
+            if (in_array(trim($substring_output_word), $noun_arr) == 1)
+                $word_type = "Noun";
+            else if (in_array(trim($substring_output_word), $verb_arr) == 1)
+                $word_type = "Verb";
+            else if (in_array(trim($substring_output_word), $adj_arr) == 1)
+                $word_type = "Adjective";
+            else if (in_array(trim($substring_output_word), $adv_arr) == 1)
+                $word_type = "Adverb";
+            else
+                $word_type = "NA";
+            if ($word_type != "NA") {
+                $os_name = PHP_OS;
+                if($os_name=='WINNT'){
+                shell_exec(WORDNET_DIR.'"/2.1/bin/wn.exe" ' . $queryWords[$i] .
+                ' -over > '.BASE_DIR.'\lib\wordnet_output.txt');
+                }
+                else{
+                shell_exec('wn ' . $queryWords[$i] .' -over > '.
+                BASE_DIR.'/lib/wordnet_output.txt');
+                }
+                /*  Calls to get an 2D array with matched words and respective
+                 *  cosine ranking score.
+                 */
+                $avg_score = self::splitWordNetOutput($word_type, $query);
+                if ($avg_score == null) {
+                    continue;
+                }
+                if ($avg_score) {
+                    asort($avg_score[0]);
+                    $extracted_words = $avg_score[1];
+                    foreach ($avg_score[0] as $key => $value) {
+                        $sorted_score_key = $key;
+                        $sorted_score_value = $value;
+                    }
+                    $similar_words[$i] = $extracted_words[$sorted_score_key];
+                    //To remove similar words as query word
+                    $patterns =
+                    array('/^'.$current_word.'\,/','/\,'.$current_word.'\,/');
+                    $extracted_words[$sorted_score_key] =
+                    preg_replace($patterns,'',
+                        $extracted_words[$sorted_score_key]);
+                    $replacements +=
+                    array($current_word => $extracted_words[$sorted_score_key]);
+                }
+            }
+        }
+        foreach ($replacements as $words => $similarWord) {
+            $arr_similarWord = explode(",", trim($similarWord));
+            $arr_similarWordCount = count($arr_similarWord);
+            $arr_similarWord[$arr_similarWordCount - 1] =
+            trim(preg_replace('/[\-]+/', '',
+                $arr_similarWord[$arr_similarWordCount - 1]));
+            $replacements[$words] = $arr_similarWord;
+        }
+        $i = 0;
+        foreach ($replacements as $word => $rep) {
+            foreach ($rep as $replace) {
+                if (substr_count(trim($replace), ' ')) {
+                    $replace = preg_replace('/~[\w]+/','',$replace);
+                    $new_string =
+                    preg_replace('/' . $word . '/', trim($replace), $query);
+                } else {
+                    $new_string =
+                    preg_replace('/' . $word . '/', trim($replace), $query);
+                }
+                if(mb_strtolower($new_string) != mb_strtolower($query)){
+                    $query1[$i] = $new_string;
+                    $i++;
+                }
+            }
+        }
+        return $query1;
+    }
+    /**
+     * This function is used to get similar words from WordNet as per word type.
+     * @param word_type is the type of word such as noun, verb, adjective,
+     * adverb
+     * @param query is query provided by user in search box
+     * @return array of similar words along with their score.
+     */
+    function splitWordNetOutput($word_type, $query)
+    {
+        if (file_exists(BASE_DIR . "/lib/wordnet_output.txt")) {
+            $doc = file_get_contents(BASE_DIR . "/lib/wordnet_output.txt");
+            if ($doc != "") {
+                $paragraphs =
+                preg_split("/\n\r/", $doc, -1, PREG_SPLIT_NO_EMPTY);
+                $pos_para = array();
+                $pos_type_para = array();
+                if ($word_type == "Verb")
+                    $pos_type_para = preg_grep("/\bThe\sverb\s/", $paragraphs);
+                else if ($word_type == "Noun")
+                    $pos_type_para = preg_grep("/\bThe\snoun\s/", $paragraphs);
+                else if ($word_type == "Adjective")
+                    $pos_type_para = preg_grep("/\bThe\sadj\s/", $paragraphs);
+                else if ($word_type == "Adverb")
+                    $pos_type_para = preg_grep("/\bThe\sadv\s/", $paragraphs);
+                if ($pos_type_para) {
+                    $temp_para = array_values($pos_type_para);
+                    $para = $temp_para[0];
+                    $pos_para  =
+                    preg_split("/\d\.\s/", $para, -1, PREG_SPLIT_NO_EMPTY);
+                    $cnt_pos_para = count($pos_para);
+                    $sentence = array();
+                    $wordmatch = array();
+                    $avg_score = array();
+                    for ($i = 1; $i < $cnt_pos_para; $i++) {
+                        preg_match_all('/\"(.*?)\"/', $pos_para[$i], $match);
+                        // to seperate out the words
+                        preg_match('/[\w+\s\,\.\']+\s\-+/',
+                            $pos_para[$i], $matchword);
+                        $wordmatch[$i] = trim($matchword[0]);
+                        $sent_count = count($match[1]);
+                        $score = array();
+                        for ($j = 0; $j < $sent_count; $j++) {
+                            $sentence = $match[1][$j];
+                            $score[$j] =
+                                self::getCosineRank(explode(' ', $query),
+                                    explode(' ', $match[1][$j]));
+                            /*  If Cosine similarity is zero then go for
+                             *  intersection similarity ranking
+                             */
+                            if($score[$j] == 0){
+                                $score[$j] =
+                                self::getIntersection($query,$match[1][$j]);
+                            }
+                            $cosineScore = $score[$j];
+                        }
+                        if ($sent_count) {
+                            $avg_score[$i] = array_sum($score) / $sent_count;
+                        } else {
+                            $avg_score[$i] = 0;
+                        }
+                    }
+                    return array(
+                        $avg_score,
+                        $wordmatch
+                    );
+                } else {
+                    return null;
+                }
+            } else {
+                return null;
+            }
+        }
+    }
+    /**
+     *  This function is used to get the score by Cosine-similarity
+     *  ranking method between two sentences.
+     *  @param array tokenA, tokenB as input sentences
+     *  @return score by Cosine-similarity ranking method
+     */
+    function getCosineRank(array $sentenceA, array $sentenceB)
+    {
+        $dotProduct = $cntSentA = $cntSentB = 0;
+        $arrayA = $arrayB = array();
+        $mergedWords = array_unique(array_merge($sentenceA, $sentenceB));
+        foreach ($sentenceA as $word)
+            $arrayA[$word] = 0;
+        foreach ($sentenceB as $word)
+            $arrayB[$word] = 0;
+        foreach ($mergedWords as $word) {
+            $resA = isset($arrayA[$word]) ? 1 : 0;
+            $resB = isset($arrayB[$word]) ? 1 : 0;
+            $dotProduct += $resA * $resB;
+            $cntSentA += $x;
+            $cntSentB += $resB;
+        }
+        $score = ($cntSentA * $cntSentB) != 0 ?
+                    $dotProduct / sqrt($normA * $cntSentB) : 0;
+        return $score;
+    }
+    /**
+     *  This function is used to get the score by intersection
+     *  method between two sentences.
+     *  @param s1, s2 are the input sentences
+     *  @return score by intersection method
+     */
+    function getIntersection($s1, $s2)
+    {
+        $sentence1 = explode(" ", $s1);
+        $sentence2 = explode(" ", $s2);
+        if ((count($sentence1) + count($sentence2)) == 0)
+            return 0;
+        $count1 = count(array_intersect($sentence1, $sentence2));
+        $count2 = ((count($sentence1) + count($sentence2)) / 2);
+        return $count1 / $count2;
+    }
+}
+?>
\ No newline at end of file
diff --git a/models/model.php b/models/model.php
index 8baacb3..c2f1b5d 100755
--- a/models/model.php
+++ b/models/model.php
@@ -214,7 +214,7 @@ class Model implements CrawlConstants
 
         $output['TOTAL_ROWS'] = $results['TOTAL_ROWS'];
         $output['PAGES'] = $pages;
-
+        $output['WORDNET_SIMILAR_WORDS'] = $results['WORDNET_SIMILAR_WORDS'];
         return $output;
     }
 
diff --git a/models/phrase_model.php b/models/phrase_model.php
index 16c710d..66a0ac4 100755
--- a/models/phrase_model.php
+++ b/models/phrase_model.php
@@ -52,6 +52,15 @@ require_once BASE_DIR."/lib/phrase_parser.php";
 require_once(BASE_DIR."/lib/file_cache.php");
 
 /**
+ * Load Part of Speech Tagging class in case used
+ */
+require_once(BASE_DIR."/lib/part_of_speech_tagging.php");
+
+/**
+ * Load Calculate BM25 class  in case used
+ */
+require_once(BASE_DIR."/lib/calculate_BM25.php");
+/**
  * Load iterators to get docs out of index archive
  */
 foreach(glob(BASE_DIR."/lib/index_bundle_iterators/*_iterator.php")
@@ -426,6 +435,10 @@ class PhraseModel extends ParallelModel
         if(isset($out_results['HARD_QUERY'])) {
             $results['HARD_QUERY'] = $out_results['HARD_QUERY'];
         }
+        if(isset($out_results['WORDNET_SIMILAR_WORDS'])){
+            $results['WORDNET_SIMILAR_WORDS'] =
+                    $out_results['WORDNET_SIMILAR_WORDS'];
+        }
         if(count($results) == 0) {
             $results = NULL;
         }
@@ -1211,7 +1224,7 @@ class PhraseModel extends ParallelModel
                     }
                 }
             }
-            $subscore_fields = array(self::DOC_RANK, self::RELEVANCE);
+            $subscore_fields = array(self::DOC_RANK, self::RELEVANCE, self::BM25_SCORE);
             if($use_proximity) {
                 $subscore_fields[] = self::PROXIMITY;
             }
@@ -1344,6 +1357,47 @@ class PhraseModel extends ParallelModel
         if(USE_CACHE && $save_timestamp_name == "") {
             $CACHE->set($summary_hash, $results);
         }
+        $lang = guessLocaleFromString($original_query);
+        /* If user selects Wordnet feature in page options then only
+         * do wordnet processing. Also user has to specify the wordnet directory
+         */
+         $wordnet_flag = 0;
+         $os_name = PHP_OS;
+         if(trim($os_name)=='WINNT' && USE_WORDNET &&
+            (WORDNET_DIR!="") && $lang=="en-US"){
+            $wordnet_flag = 1;
+         }else if(USE_WORDNET && $lang=="en-US"){
+            $wordnet_flag = 1;
+         }
+        if($wordnet_flag){
+            //Code added to sort the pages as per the BM25 score
+            $tempPages = array();
+            $tempSummary = array();
+            $tempPages = $results['PAGES'];
+                for($i = 0; $i<count($tempPages);$i++){
+                    $tempSummary[$i] = $tempPages[$i][self::DESCRIPTION];
+                }
+            $indexName = $this->index_name;
+            $threshold = 10;
+            $word =
+            PhraseParser::getSimilarWords($original_query,$indexName,
+                         $lang,$threshold);
+            $results['WORDNET_SIMILAR_WORDS'] = $word;
+            if(!empty($word)){
+                $scoreBM25 = BM25Calculator::getScore($tempSummary,$word);
+                //Store the BM25 score for each page in result array
+                for($i=0;$i<count($scoreBM25);$i++){
+                    $tempPages[$i][self::BM25_SCORE] = $scoreBM25[$i];
+                    orderCallback($tempPages[$i], $tempPages[$i],
+                    self::BM25_SCORE);
+                }
+                if(array_sum($scoreBM25)!=0){
+                    usort($tempPages, "orderCallback");
+                }
+                $results['PAGES'] = $tempPages;
+                //Code added to sort the pages as per the BM25 score
+            }
+        }
         return $results;
     }
 
diff --git a/models/profile_model.php b/models/profile_model.php
index b96696f..91e2a90 100755
--- a/models/profile_model.php
+++ b/models/profile_model.php
@@ -59,7 +59,8 @@ class ProfileModel extends Model
         'ROBOT_INSTANCE', 'RSS_ACCESS', 'SERVER_ALPHA', 'SIGNIN_LINK',
         'SIMILAR_LINK', 'SUBSEARCH_LINK', 'TITLE_WEIGHT', 'TOR_PROXY',
         'USE_FILECACHE', 'USE_MAIL_PHP', 'USE_MEMCACHE', 'USE_PROXY',
-        'USER_AGENT_SHORT', 'WEB_URI', 'WEB_ACCESS', 'WORD_SUGGEST'
+        'USER_AGENT_SHORT', 'WEB_URI', 'WEB_ACCESS', 'WORD_SUGGEST',
+        'USE_WORDNET', 'WORDNET_DIR'
         );
     /**
      *  {@inheritdoc}
diff --git a/views/elements/pageoptions_element.php b/views/elements/pageoptions_element.php
index 677c8b2..fca3797 100644
--- a/views/elements/pageoptions_element.php
+++ b/views/elements/pageoptions_element.php
@@ -305,8 +305,28 @@ class PageOptionsElement extends Element
                 e("checked='checked'");}?>
             /></td>
         </tr>
+        <tr><td><label for="use-wordnet"><?php
+            e(tl('pageoption_element_wordnet_feature')); ?></label></td>
+        <td><input id='use-wordnet' type="checkbox"
+            name="USE_WORDNET" value="true"
+            onclick="document.getElementById('wordnet-dir')
+            .disabled=!this.checked;"
+            <?php if(isset($data['USE_WORDNET']) && $data['USE_WORDNET']){
+                e("checked='checked'");}?>
+            /></td>
+        </tr>
         </table></td>
         </tr></table>
+        <?php
+        if(PHP_OS=='WINNT'){
+        ?>
+        <h4><label for="wordnet-dir"><?php
+            e(tl('pageoption_element_wordnet_dir')); ?></label>
+        <br><input type="text" id="wordnet-dir"
+            name="WORDNET_DIR" class="extra-wide-field"
+            value="<?php  e($data['WORDNET_DIR']); ?>" /></br>
+        </h4>
+        <?php } ?>
         <h2><?php e(tl('pageoptions_element_ranking_factors'))?></h2>
         <table class="weights-table" >
         <tr><th><label for="title-weight"><?php
diff --git a/views/search_view.php b/views/search_view.php
index 1cba33a..64baf53 100755
--- a/views/search_view.php
+++ b/views/search_view.php
@@ -193,6 +193,22 @@ class SearchView extends View implements CrawlConstants
                 $data['TOTAL_ROWS']));
             }
         ?></h2>
+        <div id="wordnet-words" class="wordnet">
+        <?php
+            $similar_words = $data['WORDNET_SIMILAR_WORDS'];
+            if(count($similar_words)>0){
+                e(tl('search_view_wordnet_results'));
+                foreach($similar_words as $word){
+                    e("<br>");
+                    ?><span><a href="?<?php e(CSRF_TOKEN."=".$data[CSRF_TOKEN]);
+                    ?>&amp;its=<?php e($data['its']);?>
+                    &amp;q=<?php e($word);?>"><?php
+                        e($word); ?></a></span>
+                    <?php
+                }
+            }
+            ?>
+        </div>
         <div class="serp-results">
             <?php
             foreach($data['PAGES'] as $page) {
@@ -357,6 +373,8 @@ class SearchView extends View implements CrawlConstants
                 if(MOBILE && $aux_link_flag) {e("<br />");}
                 if(isset($page[self::SCORE])) {
                     ?><span title="<?php
+                    e(tl('search_view_bm25score',
+                        number_format($page[self::BM25_SCORE], 2))."\n");
                     e(tl('search_view_rank',
                         number_format($page[self::DOC_RANK], 2))."\n");
                     e(tl('search_view_relevancy',
-- 
1.8.4.msysgit.0

