_binary From 232809e6923ed538dfaaf9f8d4e49124ea07e1b6 Mon Sep 17 00:00:00 2001
From: Shailesh Padave <shaileshpadave49@gmail.com>
Date: Tue, 13 May 2014 22:50:12 -0700
Subject: [PATCH] Added WordNet Feature

---
 configs/config.php                                |   2 +
 configs/configure_tool.php                        |   2 +-
 controllers/components/crawl_component.php        |   3 +-
 controllers/search_controller.php                 |   3 +
 css/search.css                                    |  19 +-
 lib/calculate_bm25.php                            | 208 +++++++++++++++++
 lib/crawl_constants.php                           |   2 +
 lib/phrase_parser.php                             | 116 ++++++++++
 lib/process_wordnet.php                           | 263 ++++++++++++++++++++++
 locale/en-US/resources/part_of_speech_tagging.php | 174 ++++++++++++++
 models/phrase_model.php                           |  61 ++++-
 models/profile_model.php                          |   6 +-
 views/elements/pageoptions_element.php            |  16 ++
 views/search_view.php                             |  24 +-
 14 files changed, 893 insertions(+), 6 deletions(-)
 create mode 100644 lib/calculate_bm25.php
 create mode 100644 lib/process_wordnet.php
 create mode 100644 locale/en-US/resources/part_of_speech_tagging.php

diff --git a/configs/config.php b/configs/config.php
index 8e32647..cdf0e39 100755
--- a/configs/config.php
+++ b/configs/config.php
@@ -193,6 +193,8 @@ if(file_exists(WORK_DIRECTORY.PROFILE_FILE_NAME)) {
     define('SIMILAR_LINK', true);
     define('IN_LINK', true);
     define('IP_LINK', true);
+    define('USE_WORDNET', false);
+    define('WORDNET_EXEC', '');
     define('SIGNIN_LINK', true);
     define('NEWS_MODE', 'news_off');
     /** BM25F weight for title text */
diff --git a/configs/configure_tool.php b/configs/configure_tool.php
index a54cde0..aab0192 100644
--- a/configs/configure_tool.php
+++ b/configs/configure_tool.php
@@ -342,7 +342,7 @@ class ConfigureTool
             "SUBSEARCH_LINK"  => "Subsearch Links",
             "SIGNIN_LINK" => "Sign-in Links", "CACHE_LINK" => "Cache Link",
             "SIMILAR_LINK" => "Similar Link", "IN_LINK" => "Inlinks",
-            "IP_LINK"=> "IP Links");
+            "IP_LINK"=> "IP Links", "USE_WORDNET"=> "WordNet");
         $items = array();
         foreach($settings as $setting => $setting_string) {
             $toggle = ($data[$setting]) ? "On" : "Off";
diff --git a/controllers/components/crawl_component.php b/controllers/components/crawl_component.php
index 40b4902..960aa76 100644
--- a/controllers/components/crawl_component.php
+++ b/controllers/components/crawl_component.php
@@ -967,7 +967,8 @@ class CrawlComponent extends Component implements CrawlConstants
         } else {
             $parent->updateProfileFields($data, $profile,
                 array('IP_LINK','CACHE_LINK', 'SIMILAR_LINK', 'IN_LINK',
-                    'SIGNIN_LINK', 'SUBSEARCH_LINK','WORD_SUGGEST'));
+                    'SIGNIN_LINK', 'SUBSEARCH_LINK','WORD_SUGGEST',
+                    'USE_WORDNET', 'WORDNET_EXEC'));
         }
         $weights = array('TITLE_WEIGHT' => 4,
             'DESCRIPTION_WEIGHT' => 1, 'LINK_WEIGHT' => 2,
diff --git a/controllers/search_controller.php b/controllers/search_controller.php
index effdd21..c5e27b8 100755
--- a/controllers/search_controller.php
+++ b/controllers/search_controller.php
@@ -747,6 +747,9 @@ class SearchController extends Controller implements CrawlConstants
         $data['RAW'] = $raw;
         $data['PAGES'] = (isset($phrase_results['PAGES'])) ?
              $phrase_results['PAGES']: array();
+        $data['WORDNET_SIMILAR_WORDS'] =
+            (isset($phrase_results['WORDNET_SIMILAR_WORDS'])) ?
+             $phrase_results['WORDNET_SIMILAR_WORDS']: array();
         $data['SAVE_POINT'] = (isset($phrase_results["SAVE_POINT"])) ?
              $phrase_results["SAVE_POINT"]: array( 0 => 1);
         if(isset($phrase_results["HARD_QUERY"])) {
diff --git a/css/search.css b/css/search.css
index 818842e..a03405a 100755
--- a/css/search.css
+++ b/css/search.css
@@ -1140,6 +1140,14 @@ ul.in-list li
     width: 8in;
 }
 
+.html-ltr .wordnet-serp-results
+{
+    left: 2.2in;
+    position: relative;
+    top: -0.9in;
+    width: 8in;
+}
+
 .html-rtl .serp
 {
     position: relative;
@@ -1178,7 +1186,16 @@ ul.in-list li
     position: relative;
     top: -1in;
 }
-
+.html-ltr .wordnet
+{
+    color: #666;
+    font-size: 13pt;
+    font-weight: bold;
+    left: -0.2in;
+    position: relative;
+    top: -0.8in;
+    width: 8in;
+}
 .result
 {
     clear: both;
diff --git a/lib/calculate_bm25.php b/lib/calculate_bm25.php
new file mode 100644
index 0000000..a2ee8af
--- /dev/null
+++ b/lib/calculate_bm25.php
@@ -0,0 +1,208 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2014  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Shailesh Padave shaileshpadave49@gmail.com
+ * @package seek_quarry
+ * @subpackage library
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2014
+ * @filesource
+ */
+/**
+ * This class is used to rank the matching document from the generated summary
+ * during crawl using BM25 raking function.
+ * @package seek_quarry
+ * @subpackage library
+ */
+class BM25Calculator
+{
+    /**
+     * Gets array of BM25 score for given input array of summaries
+     * @param array $summary an array of summaries which is generated
+     *      during crawl time.
+     * @param array $similar_words an array of synonyms which is generated by
+     *      WordNet.
+     *@returns array of BM25 score for each document
+     */
+    static function getScore($summary, $similar_words)
+    {
+        $score = array();
+        //if there are no similar words then
+        if(empty($similar_words)) {
+            return;
+        } else {
+            for($i = 0; $i < count($similar_words); $i++) {
+                $query = $similar_words[$i];
+                $terms = explode(' ', $query);
+                $summary = self::changeCaseOfArray($summary);
+                $query = mb_strtolower($query);
+                $idf = self::calculateIDF($summary, $query, $terms);
+                $tf = self::calculateTF($summary, $query, $terms);
+                $summary_count = count($summary);
+                $count_terms = count($terms);
+                $bm25_result[$i] =
+                    self::calculateBM25($idf, $tf, $count_terms,
+                    $summary_count);
+            }
+            if ( count($bm25_result) == 1){
+                for($i = 0; $i < $summary_count; $i++) {
+                    $temp = 0;
+                    $temp = $bm25_result[0][$i];
+                    $score[$i] = $temp;
+                }
+            } else {
+                for($i = 0; $i < $summary_count; $i++) {
+                    $temp = 0;
+                    $temp = $bm25_result[0][$i] * (2/3) + $bm25_result[1][$i] * (1/3);
+                    $score[$i] = $temp;
+                }
+            }
+            return $score;
+            }
+        }
+    /**
+     * To make lower case for all the input variables
+     * @param array $summary an array of summaries
+     * @returns array of with strings converted to
+     * lower case
+     */
+    static function changeCaseOfArray($summary)
+    {
+        for($i = 0; $i < count($summary); $i++) {
+            $summary[$i] = mb_strtolower($summary[$i], 'UTF-8');
+        }
+        return $summary;
+    }
+    /**
+     * To get the total number of characters in input array
+     * @param array $summary an array of summaries
+     * @returns number of characters in input array
+     */
+    static function getlength($summary)
+    {
+        $summary_count = count($summary);
+        $str_len =0;
+        for($i = 0; $i < $summary_count; $i++) {
+            $str = $summary[$i];
+            $str_len = $str_len + strlen($str);
+        }
+        return $str_len;
+    }
+    /**
+     * To get the term frequency in given summary array
+     * @param array $summary an array of summaries
+     * @param array $query an array of words generated by wordnet output
+     * @returns array of term frequency for each query term
+     */
+    static function calculateTF($summary, $query, $terms)
+    {
+        $k1=1.5;
+        $b=0.75;
+        $tf_values = array();
+        $tfbm25 = array();
+        $doc_length = self::getlength($summary);
+        if(count($summary) != 0) {
+            $avg_length = $doc_length / count($summary);
+        } else {
+            $avg_length = 0;
+          }
+        $tf_values = self::calculateWordFreq($summary, $terms);
+        for($i = 0; $i < count($terms); $i++) {
+            for($j = 0; $j <count($summary); $j++) {
+                $frequency = $tf_values[$i][$j];
+                $tfbm25[$i][$j] =
+                ($frequency*($k1+1))/($frequency+$k1*((1-$b)+$b*($doc_length/$avg_length)));
+            }
+        }
+        return $tfbm25;
+    }
+    static function calculateWordFreq($summary, $query_array)
+    {
+        $tf_values = array();
+        for($i = 0; $i < count($query_array); $i++) {
+            for($j = 0; $j < count($summary); $j++) {
+                  $frequency = substr_count($summary[$j], $query_array[$i]);
+                  $tf_values[$i][$j] = $frequency;
+            }
+        }
+        return $tf_values;
+    }
+    /**
+     * To get the BM25 Score in given summary array
+     * @param array $idf inverse Doc frequency for given query array
+     * @param array $tf term frequency for given query array
+     * @param $count_terms count for number of queries
+     * @param $summary_count count for input summaries
+     * @returns array of BM25 score for each document
+     */
+    static function calculateBM25($idf, $tf, $count_terms, $summary_count)
+    {
+        $score = array();
+        for($i = 0; $i < $count_terms; $i++) {
+            for($j = 0; $j < $summary_count; $j++) {
+                $bm25_score[$i][$j] = $idf[$i] * $tf[$i][$j];
+            }
+        }
+        for($i = 0; $i < $summary_count; $i++) {
+            $val = 0;
+            for($j = 0; $j< $count_terms; $j++) {
+                $val += $bm25_score[$j][$i];
+            }
+            $score[$i] = $val;
+        }
+        return $score;
+    }
+    /**
+     * To get the Inverse Doc frequency in given summary array
+     * @param array $summary an array of summaries
+     * @param array $query an array of words generated by wordnet output
+     * @returns array of Inverse Doc frequency for each query term
+     */
+    static function calculateIDF($summary, $query, $terms)
+    {
+        $N = count($summary);
+        $Nt = array();
+        $term_count = 0;
+        for($i = 0; $i < count($terms); $i++) {
+            $cnt_Nt = 0;
+            $term_count++;
+            foreach($summary as $value)
+            {
+                $pattern = "/".$terms[$i]."/";
+                if(preg_match($pattern, $value)) {
+                    $cnt_Nt++;
+                }
+            }
+            $Nt[$i]=$cnt_Nt;
+            if($Nt[$i] != 0) {
+                $idf[$i] = log10($N / $Nt[$i]);
+            } else {
+                $idf[$i] = 0;
+            }
+        }
+        return $idf;
+    }
+}
+?>
\ No newline at end of file
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
index 61bf186..c26e6b0 100755
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -253,5 +253,7 @@ interface CrawlConstants
     const CENTROID_SUMMARIZER = 'dl';
     const SUMMARIZER_OPTION = 'dm';
     const WORD_CLOUD = 'dn';
+    const BM25_SCORE ='do';
+    const WORDNET_SIMILAR_WORDS='dp';
 }
 ?>
\ No newline at end of file
diff --git a/lib/phrase_parser.php b/lib/phrase_parser.php
index 1082082..e6a456e 100755
--- a/lib/phrase_parser.php
+++ b/lib/phrase_parser.php
@@ -71,8 +71,16 @@ require_once BASE_DIR."/lib/index_manager.php";
  */
 require_once BASE_DIR."/lib/suffix_tree.php";
 
+/**
+ * Get the part of speech tagging for given sentence
+ */
+require_once LOCALE_DIR . "/en-US/resources/part_of_speech_tagging.php";
 
 /**
+ * Get the similar words from wordnet for given query
+ */
+require_once BASE_DIR . "/lib/process_wordnet.php";
+/**
  * Library of functions used to manipulate words and phrases
  *
  * @author Chris Pollett
@@ -1172,4 +1180,112 @@ vaffanculo fok hoer kut lul やりまん 打っ掛け
         $score = $num_unsafe_terms * $unsafe_count/($len + 1);
         return $score;
     }
+    /**
+     *  Extracts similar words from wordnet output for given query.
+     *  Part of speech tagging processed on input, output is given to wordnet
+     *  processing. Wordnet function provides list of words as per ranking.
+     *  For those words, count will be calculated from posting list and top 2
+     *  words are selected.
+     *  @param string $orig_query input query from user
+     *  @param string $index_name selected index for search engine
+     *  @param string $lang language
+     *  @param integer $threshold once count in posting list for any word
+     *      reaches to threshold then return the number
+     *  @return array of top two words
+     */
+    static function getSimilarWords($orig_query, $index_name, $lang,
+            $threshold = 10)
+    {
+        $replacements = array();
+        $num_non_zero_score = array();
+        $num_zero_score = array();
+        $string = PartOfSpeechTagger::getPartOfSpeechTagging($orig_query);
+        $replacements =
+        WordNetProcessor::processWordNetOutput(trim($orig_query),trim($string));
+        $i = 0;
+        $cnt = 0;
+        foreach($replacements as $replacement) {
+            $cnt =
+            self::compute_score($replacement, $threshold, $index_name, $lang);
+            if($cnt){
+                $num_non_zero_score[$replacement] = $cnt;
+            } else {
+                $num_zero_score[$replacement] = $cnt;
+            }
+            $i++;
+        }
+        $result = array();
+        if(count($num_non_zero_score) >= 2) {
+            $i = 0;
+            arsort($num_non_zero_score);
+            foreach($num_non_zero_score as $k => $v) {
+                $result[$i] = $k;
+                $i++;
+                if($i == 2)
+                    break;
+            }
+        } elseif (count($num_non_zero_score) == 1) {
+            $i = 0;
+            foreach($num_non_zero_score as $k=>$v) {
+                $result[$i] = $k;
+                $i++;
+                if($i == 1)
+                    break;
+            }
+            foreach($num_zero_score as $k=>$v) {
+                $result[$i] = $k;
+                $i++;
+                if($i == 2)
+                    break;
+            }
+        } else {
+            $i = 0;
+            foreach($num_zero_score as $k=>$v) {
+                $result[$i] = $k;
+                $i++;
+                if($i == 2){
+                    break;
+                }
+            }
+        }
+    return $result;
+    }
+    /**
+     *  Computes count for each word in posting list
+     *  @param string $replacement input query part of speech tagging
+     *  @param integer $threshold once count in posting list for any word
+     *      reaches to threshold then return the number
+     *  @param string $index_name selected index for search engine
+     *  @param string $lang language
+     *  @return integer $count_whole_phrase in posting list for given word
+     */
+    static function compute_score($replacement, $threshold, $index_name, $lang)
+    {
+        $string = $replacement;
+        if (isset(self::$programming_language_map[$lang])) {
+            $control_word =
+            self::$programming_language_map[$lang].self::CONTROL_WORD_INDICATOR;
+            $string = trim(substr($string, strlen($control_word) + 1));
+        } else {
+            self::canonicalizePunctuatedTerms($string, $lang);
+        }
+        $terms = self::stemCharGramSegment($string, $lang);
+        $num   = count($terms);
+        /*  Changed as term size will not be zero.
+         *  if ($index_name == NULL || $num <= 1) {
+         */
+        if ($index_name == NULL) {
+            return $terms;
+        }
+        if (count($terms) > MAX_QUERY_TERMS) {
+            $first_terms  = array_slice($terms, 0, MAX_QUERY_TERMS);
+            $whole_phrase = implode(" ", $first_terms);
+        } else {
+            $whole_phrase = implode(" ", $terms);
+            $first_terms =& $terms;
+        }
+        $count_whole_phrase =
+        IndexManager::numDocsTerm($whole_phrase, $index_name, $threshold);
+        return $count_whole_phrase;
+    }
 }
\ No newline at end of file
diff --git a/lib/process_wordnet.php b/lib/process_wordnet.php
new file mode 100644
index 0000000..13a3ba0
--- /dev/null
+++ b/lib/process_wordnet.php
@@ -0,0 +1,263 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2014  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Shailesh Padave shaileshpadave49@gmail.com
+ * @package seek_quarry
+ * @subpackage library
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2014
+ * @filesource
+ */
+class WordNetProcessor
+{
+    /**
+     *  Extracts the similar words from WordNet output generated on command line
+     *  It is stored in text file and used for parsing similar words and
+     *  sentences.
+     *  Sentences are then used in cosine ranking along with query string.
+     *  According to score, synonyms are selected.
+     *  @param String $query : Query entered by user
+     *  @param String $pos_query : Same query processed after Part of speech
+     *               tagging
+     *  @return String array $result with all possible replacements for
+     *             given query.
+     */
+    static function processWordNetOutput($query, $pos_query)
+    {
+        $replacements = array();
+        $result = array();
+        // Get the query from user and do PartofSpeech Tagging on it
+        $terms = explode(" ", $query);
+        $output = $pos_query;
+        // Find out exact meaning of NN,JJ, in terms of Noun, Verb
+        $output_words = preg_split("/\s+/", $output, -1, PREG_SPLIT_NO_EMPTY);
+        $output_words_cnt = count($output_words);
+        $word_type = null;
+        $noun_arr = array("NN", "NNS", "NNP", "NNPS", "PRP", "PRP$",
+                "WP", "WP$");
+        $verb_arr = array("VB", "VBD", "VBG", "VBN", "VBP", "VBZ");
+        $adj_arr = array("AJ", "JJ", "JJR", "JJS");
+        $adv_arr = array("AV", "RB", "RBR", "RBS", "WRB");
+        $similar_words = array();
+        for ($i = 0; $i < $output_words_cnt; $i++) {
+            $wordnet_op = "";
+            $pos = strpos($output_words[$i], '~');
+            $substring_output_word = substr($output_words[$i], $pos + 1);
+            $current_word = substr($output_words[$i], 0, $pos);
+            if (in_array(trim($substring_output_word), $noun_arr) == 1) {
+                $word_type = "Noun";
+            } else if (in_array(trim($substring_output_word), $verb_arr) == 1) {
+                $word_type = "Verb";
+            } else if (in_array(trim($substring_output_word), $adj_arr) == 1) {
+                $word_type = "Adjective";
+            } else if (in_array(trim($substring_output_word), $adv_arr) == 1) {
+                $word_type = "Adverb";
+            } else {
+                $word_type = "NA";
+            }
+            if ($word_type != "NA") {
+                $os_name = PHP_OS;
+                $data = "";
+                if($os_name == 'WINNT') {
+                    exec(escapeshellarg(WORDNET_EXEC).'"/wn.exe" ' .
+                        $terms[$i] .' -over',$data);
+                } else {
+                    exec(escapeshellarg(WORDNET_EXEC).'"/wn" ' .
+                        $terms[$i] .' -over',$data);
+                }
+                $wordnet_op = implode(' ',$data);
+                /*  Calls to get an 2D array with matched words and respective
+                 *  cosine ranking score.
+                 */
+                $avg_score = self::splitWordNetOutput($word_type,
+                                    $query, $wordnet_op);
+                if ($avg_score == null) {
+                    continue;
+                }
+                if ($avg_score) {
+                    asort($avg_score[0]);
+                    $extracted_words = $avg_score[1];
+                    foreach ($avg_score[0] as $key => $value) {
+                        $sorted_score_key = $key;
+                        $sorted_score_value = $value;
+                    }
+                    $similar_words[$i] = $extracted_words[$sorted_score_key];
+                    //To remove similar words as query word
+                    $patterns =
+                    array('/^'.$current_word.'\,/','/\,'.$current_word.'\,/');
+                    $extracted_words[$sorted_score_key] =
+                    preg_replace($patterns,'',
+                        $extracted_words[$sorted_score_key]);
+                    $replacements +=
+                    array($current_word => $extracted_words[$sorted_score_key]);
+                }
+            }
+        }
+        foreach ($replacements as $words => $similar_word) {
+            $arr_similar_word = explode(",", trim($similar_word));
+            $arr_similar_word_cnt = count($arr_similar_word);
+            $arr_similar_word[$arr_similar_word_cnt - 1] =
+            trim(preg_replace('/[\-]+/', '',
+                $arr_similar_word[$arr_similar_word_cnt - 1]));
+            $replacements[$words] = $arr_similar_word;
+        }
+        $i = 0;
+        foreach ($replacements as $word => $rep) {
+            foreach ($rep as $replace) {
+                if (substr_count(trim($replace), ' ')) {
+                    $replace = preg_replace('/~[\w]+/','',$replace);
+                    $new_string =
+                    preg_replace('/' . $word . '/', trim($replace), $query);
+                } else {
+                    $new_string =
+                    preg_replace('/' . $word . '/', trim($replace), $query);
+                }
+                if(mb_strtolower($new_string) != mb_strtolower($query)){
+                    $result[$i] = $new_string;
+                    $i++;
+                }
+            }
+        }
+        return $result;
+    }
+    /**
+     * This function is used to get similar words from WordNet as per word type.
+     * @param word_type is the type of word such as noun, verb, adjective,
+     * adverb
+     * @param query is query provided by user in search box
+     * @return array of similar words along with their score.
+     */
+    static function splitWordNetOutput($word_type, $query, $wordnet_op)
+    {
+        if ($wordnet_op != "") {
+            $paragraphs =
+            preg_split("/\n\r/", $wordnet_op, -1, PREG_SPLIT_NO_EMPTY);
+            $pos_para = array();
+            $pos_type_para = array();
+            if ($word_type == "Verb"){
+                $pos_type_para = preg_grep("/\bThe\sverb\s/", $paragraphs);
+            } else if ($word_type == "Noun") {
+                $pos_type_para = preg_grep("/\bThe\snoun\s/", $paragraphs);
+            } else if ($word_type == "Adjective") {
+                $pos_type_para = preg_grep("/\bThe\sadj\s/", $paragraphs);
+            } else if ($word_type == "Adverb") {
+                $pos_type_para = preg_grep("/\bThe\sadv\s/", $paragraphs);
+            }
+            if ($pos_type_para) {
+                $temp_para = array_values($pos_type_para);
+                $para = $temp_para[0];
+                $pos_para  =
+                preg_split("/\d\.\s/", $para, -1, PREG_SPLIT_NO_EMPTY);
+                $cnt_pos_para = count($pos_para);
+                $sentence = array();
+                $wordmatch = array();
+                $avg_score = array();
+                for ($i = 1; $i < $cnt_pos_para; $i++) {
+                    preg_match_all('/\"(.*?)\"/', $pos_para[$i], $match);
+                    // to seperate out the words
+                    preg_match('/[\w+\s\,\.\']+\s\-+/',
+                        $pos_para[$i], $matchword);
+                    $wordmatch[$i] = trim($matchword[0]);
+                    $sent_count = count($match[1]);
+                    $score = array();
+                    for ($j = 0; $j < $sent_count; $j++) {
+                        $sentence = $match[1][$j];
+                        $score[$j] =
+                            self::getCosineRank(explode(' ', $query),
+                                explode(' ', $match[1][$j]));
+                        /*  If Cosine similarity is zero then go for
+                         *  intersection similarity ranking
+                         */
+                        if($score[$j] == 0){
+                            $score[$j] =
+                            self::getIntersection($query,$match[1][$j]);
+                        }
+                        $cosine_score = $score[$j];
+                    }
+                    if ($sent_count) {
+                        $avg_score[$i] = array_sum($score) / $sent_count;
+                    } else {
+                        $avg_score[$i] = 0;
+                    }
+                }
+                return array(
+                    $avg_score,
+                    $wordmatch
+                );
+            } else {
+                return null;
+            }
+        } else {
+            return null;
+        }
+    }
+    /**
+     *  This function is used to get the score by Cosine-similarity
+     *  ranking method between two sentences.
+     *  @param array tokenA, tokenB as input sentences
+     *  @return score by Cosine-similarity ranking method
+     */
+    static function getCosineRank(array $sentenceA, array $sentenceB)
+    {
+        $result = $cnt_sentA = $cnt_sentB = 0;
+        $arrayA = $arrayB = array();
+        $x = 0;
+        $mergedWords = array_unique(array_merge($sentenceA, $sentenceB));
+        foreach ($sentenceA as $word) {
+            $arrayA[$word] = 0;
+        }
+        foreach ($sentenceB as $word) {
+            $arrayB[$word] = 0;
+        }
+        foreach ($mergedWords as $word) {
+            $resA = isset($arrayA[$word]) ? 1 : 0;
+            $resB = isset($arrayB[$word]) ? 1 : 0;
+            $result += $resA * $resB;
+            $cnt_sentA += $x;
+            $cnt_sentB += $resB;
+        }
+        $score = ($cnt_sentA * $cnt_sentB) != 0 ?
+                    $result / sqrt($normA * $cnt_sentB) : 0;
+        return $score;
+    }
+    /**
+     *  This function is used to get the score by intersection
+     *  method between two sentences.
+     *  @param s1, s2 are the input sentences
+     *  @return score by intersection method
+     */
+    static function getIntersection($s1, $s2)
+    {
+        $sentence1 = explode(" ", $s1);
+        $sentence2 = explode(" ", $s2);
+        if ((count($sentence1) + count($sentence2)) == 0) {
+            return 0;
+        }
+        $count1 = count(array_intersect($sentence1, $sentence2));
+        $count2 = ((count($sentence1) + count($sentence2)) / 2);
+        return $count1 / $count2;
+    }
+}
+?>
\ No newline at end of file
diff --git a/locale/en-US/resources/part_of_speech_tagging.php b/locale/en-US/resources/part_of_speech_tagging.php
new file mode 100644
index 0000000..25d1b75
--- /dev/null
+++ b/locale/en-US/resources/part_of_speech_tagging.php
@@ -0,0 +1,174 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2014  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Shailesh Padave based on Ian Barber's Code
+ * @package seek_quarry
+ * @subpackage locale
+ * @license http://www.gnu.org/licenses/ GPLv3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2014
+ * @filesource http://phpir.com/part-of-speech-tagging
+ */
+/**
+ * This class is used to get part of speech tagging for given input.
+ * @package seek_quarry
+ * @subpackage locale
+ */
+class PartOfSpeechTagger
+{
+    private $dictionary;
+    public function __construct($lexicon)
+    {
+        $fh = fopen($lexicon, 'r');
+        while($line = fgets($fh)) {
+                $tags = explode(' ', $line);
+                $this->dictionary[strtolower(array_shift($tags))] = $tags;
+        }
+        fclose($fh);
+    }
+    public function tag($text)
+    {
+        preg_match_all("/[\w\d]+/", $text, $matches);
+        $nouns = array('NN', 'NNS');
+        $result = array();
+        $i = 0;
+        foreach($matches[0] as $token) {
+            // default to a common noun
+            $result[$i] = array('token' => $token, 'tag' => 'NN');
+            // remove trailing full stops
+            if(substr($token, -1) == '.') {
+                    $token = preg_replace('/\.+$/', '', $token);
+            }
+            // get from dictionary if set
+            if(isset($this->dictionary[strtolower($token)])) {
+                    $result[$i]['tag'] =
+                        $this->dictionary[strtolower($token)][0];
+            }
+            // Converts verbs after 'the' to nouns
+            if($i > 0) {
+                    if($result[$i - 1]['tag'] == 'DT' &&
+                             in_array($result[$i]['tag'],
+                                            array('VBD', 'VBP', 'VB'))) {
+                            $result[$i]['tag'] = 'NN';
+                    }
+            }
+            // Convert noun to number if . appears
+            if($result[$i]['tag'][0] == 'N' && strpos($token, '.') !== false) {
+                    $result[$i]['tag'] = 'CD';
+            }
+            // Convert noun to past particle if ends with 'ed'
+            if($result[$i]['tag'][0] == 'N' && substr($token, -2) == 'ed') {
+                    $result[$i]['tag'] = 'VBN';
+            }
+            // Anything that ends 'ly' is an adverb
+            if(substr($token, -2) == 'ly') {
+                    $result[$i]['tag'] = 'RB';
+            }
+            // Common noun to adjective if it ends with al
+            if(in_array($result[$i]['tag'], $nouns)
+                                    && substr($token, -2) == 'al') {
+                    $result[$i]['tag'] = 'JJ';
+            }
+            // Noun to verb if the word before is 'would'
+            if($i > 0) {
+                    if($result[$i]['tag'] == 'NN'
+                            && strtolower($result[$i-1]['token']) == 'would') {
+                            $result[$i]['tag'] = 'VB';
+                    }
+            }
+            // Convert noun to plural if it ends with an s
+            if($result[$i]['tag'] == 'NN' && substr($token, -1) == 's') {
+                    $result[$i]['tag'] = 'NNS';
+            }
+            // Convert common noun to gerund
+            if(in_array($result[$i]['tag'], $nouns)
+                            && substr($token, -3) == 'ing') {
+                    $result[$i]['tag'] = 'VBG';
+            }
+            /* If we get noun noun, and the second can be a verb,
+             * convert to verb
+             */
+            if($i > 0) {
+                if(in_array($result[$i]['tag'], $nouns)
+                        && in_array($result[$i-1]['tag'], $nouns)
+                        && isset($this->dictionary[strtolower($token)])) {
+                    if(in_array('VBN', $this->dictionary[strtolower($token)])) {
+                            $result[$i]['tag'] = 'VBN';
+                    } else if(in_array('VBZ',
+                            $this->dictionary[strtolower($token)])) {
+                            $result[$i]['tag'] = 'VBZ';
+                    }
+                }
+            }
+            $i++;
+        }
+        return $result;
+    }
+    /* Function for print the results
+     * @param $tags input string on which Part of speech processing is performed
+     * @return $output_string with customized part of speech for query input
+     */
+    static function printTag($tags)
+    {
+        $output_string = null;
+        $noun_arr = array("NN","NNS","NNP","NNPS","PRP","PRP$","WP","WP");
+        $verb_arr = array("VB","VBD","VBG","VBN","VBP","VBZ");
+        $adj_arr = array("JJ","JJR","JJS");
+        $adv_arr = array("RB","RBR","RBS","WRB");
+        foreach($tags as $t) {
+            if(in_array(trim($t['tag']),$noun_arr)==1) {
+                $output_string =
+                    $output_string . $t['token'] . "~" . "NN" .  " ";
+            } else if(in_array(trim($t['tag']),$verb_arr)==1) {
+                $output_string =
+                    $output_string . $t['token'] . "~" . "VB" .  " ";
+            } else if(in_array(trim($t['tag']),$adj_arr)==1) {
+                $output_string =
+                    $output_string . $t['token'] . "~" . "AJ" .  " ";
+            } else if(in_array(trim($t['tag']),$adv_arr)==1) {
+                $output_string =
+                    $output_string . $t['token'] . "~" . "AV" .  " ";
+            } else {
+                $output_string =
+                $output_string . $t['token'] . "~" . $t['tag'] .  " ";
+            }
+        }
+        return $output_string;
+    }
+    /* Function for part of speech for given query
+     * @param $string input string from user
+     * @return $output_string with part of speech processed on query input
+     */
+    static function getPartOfSpeechTagging($string)
+    {
+        $lang = guessLocaleFromString($string);
+        // Initialize the directory named as lexicon.txt
+        $tagger = new PartOfSpeechTagger(LOCALE_DIR . "/".
+                $lang ."/resources/lexicon.txt");
+        $tags = $tagger->tag($string);
+        $output_string  = self::printTag($tags);
+        return $output_string;
+    }
+}
+?>
\ No newline at end of file
diff --git a/models/phrase_model.php b/models/phrase_model.php
index 16c710d..b580dc3 100755
--- a/models/phrase_model.php
+++ b/models/phrase_model.php
@@ -52,6 +52,10 @@ require_once BASE_DIR."/lib/phrase_parser.php";
 require_once(BASE_DIR."/lib/file_cache.php");
 
 /**
+ * Load Calculate BM25 class  in case used
+ */
+require_once(BASE_DIR."/lib/calculate_bm25.php");
+/**
  * Load iterators to get docs out of index archive
  */
 foreach(glob(BASE_DIR."/lib/index_bundle_iterators/*_iterator.php")
@@ -426,6 +430,10 @@ class PhraseModel extends ParallelModel
         if(isset($out_results['HARD_QUERY'])) {
             $results['HARD_QUERY'] = $out_results['HARD_QUERY'];
         }
+        if(isset($out_results['WORDNET_SIMILAR_WORDS'])){
+            $results['WORDNET_SIMILAR_WORDS'] =
+                    $out_results['WORDNET_SIMILAR_WORDS'];
+        }
         if(count($results) == 0) {
             $results = NULL;
         }
@@ -461,6 +469,10 @@ class PhraseModel extends ParallelModel
         if($raw == 0) {
             $output = $this->formatPageResults($results, $format_words,
                 $description_length);
+             if(USE_WORDNET){
+                $output['WORDNET_SIMILAR_WORDS'] =
+                    $results['WORDNET_SIMILAR_WORDS'];
+             }
         } else {
             $output = $results;
         }
@@ -1344,9 +1356,56 @@ class PhraseModel extends ParallelModel
         if(USE_CACHE && $save_timestamp_name == "") {
             $CACHE->set($summary_hash, $results);
         }
+        $lang = guessLocaleFromString($original_query);
+        if(USE_WORDNET && (WORDNET_EXEC != "") && $lang == "en-US") {
+            $results_wordnet_score = self::sortByWordNetScore($results,
+                $original_query, $lang);
+            $results = $results_wordnet_score;
+        }
+        return $results;
+    }
+    /**
+     * If user selects Wordnet feature in page options then only
+     * do WordNet processing. Also user has to specify the WordNet directory
+     *
+     * @param array $results document summaries
+     * @param String User original query
+     * @return array results document summaries sorted by wordnet score
+     */
+    function sortByWordNetScore($results, $original_query, $lang)
+    {
+        /* If user selects Wordnet feature in page options then only
+         * do WordNet processing. Also user has to specify the WordNet directory
+         */
+        //Code added to sort the pages as per the BM25 score
+        $temp_pages = array();
+        $temp_summary = array();
+        $temp_pages = $results['PAGES'];
+            for($i = 0; $i < count($temp_pages); $i++) {
+                $temp_summary[$i] = $temp_pages[$i][self::DESCRIPTION];
+            }
+        $index_name = $this->index_name;
+        $threshold = 10;
+        $word =
+        PhraseParser::getSimilarWords($original_query, $index_name,
+                     $lang, $threshold);
+        $results['WORDNET_SIMILAR_WORDS'] = $word;
+        if(!empty($word)) {
+            $score_bm25 = BM25Calculator::getScore($temp_summary, $word);
+            //Store the BM25 score for each page in result array
+            for($i = 0; $i < count($score_bm25); $i++) {
+                $temp_pages[$i][self::BM25_SCORE] = $score_bm25[$i];
+                orderCallback($temp_pages[$i], $temp_pages[$i],
+                    self::BM25_SCORE);
+            }
+            if(array_sum($score_bm25) != 0){
+                usort($temp_pages, "orderCallback");
+            }
+            $results['PAGES'] = $temp_pages;
+            //Code added to sort the pages as per the BM25 score
+        }
         return $results;
     }
-
     /**
      * Used to lookup summary info for the pages provided (using their)
      * self::SUMMARY_OFFSET field. If any of the lookup-ed summaries
diff --git a/models/profile_model.php b/models/profile_model.php
index 3019217..5f7c5af 100755
--- a/models/profile_model.php
+++ b/models/profile_model.php
@@ -59,7 +59,8 @@ class ProfileModel extends Model
         'ROBOT_INSTANCE', 'RSS_ACCESS', 'SERVER_ALPHA', 'SIGNIN_LINK',
         'SIMILAR_LINK', 'SUBSEARCH_LINK', 'TITLE_WEIGHT', 'TOR_PROXY',
         'USE_FILECACHE', 'USE_MAIL_PHP', 'USE_MEMCACHE', 'USE_PROXY',
-        'USER_AGENT_SHORT', 'WEB_URI', 'WEB_ACCESS', 'WORD_SUGGEST'
+        'USE_WORDNET', 'USER_AGENT_SHORT', 'WEB_URI', 'WEB_ACCESS',
+        'WORD_SUGGEST', 'WORDNET_EXEC'
         );
     /**
      *  {@inheritdoc}
@@ -176,6 +177,9 @@ EOT;
                     $profile[$field] = "";
             }
 
+            if($field == "WORDNET_EXEC") {
+                $profile[$field] = str_replace('\\', '/', $profile[$field]);
+            }
             if($field == "NEWS_MODE" && $profile[$field] == "") {
                 $profile[$field] = "news_off";
             }
diff --git a/views/elements/pageoptions_element.php b/views/elements/pageoptions_element.php
index 545302c..5f8e4f1 100644
--- a/views/elements/pageoptions_element.php
+++ b/views/elements/pageoptions_element.php
@@ -301,8 +301,24 @@ class PageOptionsElement extends Element
                 e("checked='checked'");}?>
             /></td>
         </tr>
+        <tr><td><label for="use-wordnet"><?php
+            e(tl('pageoption_element_wordnet_feature')); ?></label></td>
+        <td><input id='use-wordnet' type="checkbox"
+            name="USE_WORDNET" value="true"
+            onclick="document.getElementById('wordnet-exec')
+            .disabled=!this.checked;"
+            <?php if(isset($data['USE_WORDNET']) && $data['USE_WORDNET']){
+                e("checked='checked'");}?>
+            /></td>
+        </tr>
         </table></td>
         </tr></table>
+        <h4><label for="wordnet-exec"><?php
+            e(tl('pageoption_element_wordnet_exec')); ?></label>
+        <br><input type="text" id="wordnet-exec"
+            name="WORDNET_EXEC" class="extra-wide-field"
+            value="<?php e($data['WORDNET_EXEC']); ?>" /></br>
+        </h4>
         <h2><?php e(tl('pageoptions_element_ranking_factors'))?></h2>
         <table class="weights-table" >
         <tr><th><label for="title-weight"><?php
diff --git a/views/search_view.php b/views/search_view.php
index b93c168..7a2e1e6 100755
--- a/views/search_view.php
+++ b/views/search_view.php
@@ -196,7 +196,27 @@ class SearchView extends View implements CrawlConstants
                 $data['TOTAL_ROWS']));
             }
         ?></h2>
-        <div class="serp-results">
+        <div id="wordnet-words" class="wordnet">
+        <?php
+            $similar_words = $data['WORDNET_SIMILAR_WORDS'];
+            if(count($similar_words) > 0) {
+                e(tl('search_view_wordnet_results'));
+                foreach ($similar_words as $word) {
+                    e("<br/>");
+                    ?><span><a href="?<?php e(CSRF_TOKEN."=".$data[CSRF_TOKEN]);
+                    ?>&amp;its=<?php e($data['its']);?>
+                    &amp;q=<?php e($word);?>"><?php
+                        e($word); ?></a></span>
+                    <?php
+                }
+            }
+            ?>
+        </div>
+        <?php if(count($similar_words) > 0) { ?>
+            <div class="wordnet-serp-results"> <?php
+        } else { ?>
+            <div class="serp-results">
+        <?php } ?>
             <?php
             foreach($data['PAGES'] as $page) {
                 if(isset($page[self::URL])) {
@@ -381,6 +401,8 @@ class SearchView extends View implements CrawlConstants
                 if(MOBILE && $aux_link_flag) {e("<br />");}
                 if(isset($page[self::SCORE])) {
                     ?><span title="<?php
+                    e(tl('search_view_bm25score',
+                        number_format($page[self::BM25_SCORE], 2))."\n");
                     e(tl('search_view_rank',
                         number_format($page[self::DOC_RANK], 2))."\n");
                     e(tl('search_view_relevancy',
-- 
1.8.4.msysgit.0

