_binary From c98192a9052e5af5d3ab35ed1026fc06d0f9aee0 Mon Sep 17 00:00:00 2001
From: Charles Bocage <charles.bocage@sjsu.edu>
Date: Thu, 26 Mar 2015 22:21:07 -0700
Subject: [PATCH] Adding the code changes to have the Graph Based Summarizer
 added and incorportated into the search engine

---
 bin/fetcher.php                            |   3 +-
 controllers/components/crawl_component.php |   3 +-
 lib/crawl_constants.php                    |   1 +
 lib/graph_based_summarizer.php             | 478 +++++++++++++++++++++++++++++
 lib/phrase_parser.php                      |  34 ++
 lib/processors/html_processor.php          |  10 +
 lib/processors/page_processor.php          |   6 +-
 lib/processors/text_processor.php          |   4 +
 locale/en-US/configure.ini                 |   3 +
 9 files changed, 538 insertions(+), 4 deletions(-)
 create mode 100644 lib/graph_based_summarizer.php

diff --git a/bin/fetcher.php b/bin/fetcher.php
index 1968672..f441b93 100755
--- a/bin/fetcher.php
+++ b/bin/fetcher.php
@@ -291,7 +291,8 @@ class Fetcher implements CrawlConstants
     var $crawl_order;
     /**
      * Stores the name of the summarizer used for crawling.
-     * Possible values are self::BASIC and self::CENTROID_SUMMARIZER
+* Possible values are self::BASIC, self::GRAPH_BASED_SUMMARIZER
+     * and self::CENTROID_SUMMARIZER
      * @var string
      */
     var $summarizer_option;
diff --git a/controllers/components/crawl_component.php b/controllers/components/crawl_component.php
index c0cf7c4..12b7ae5 100644
--- a/controllers/components/crawl_component.php
+++ b/controllers/components/crawl_component.php
@@ -853,7 +853,8 @@ class CrawlComponent extends Component implements CrawlConstants
             5000000=>5000000, 10000000=>10000000);
         $data['available_summarizers'] = array(
             self::BASIC_SUMMARIZER => tl('crawl_component_basic'),
-            self::CENTROID_SUMMARIZER =>  tl('crawl_component_centroid'));
+            self::CENTROID_SUMMARIZER =>  tl('crawl_component_centroid'),
+            self::GRAPH_BASED_SUMMARIZER =>  tl('crawl_component_graph_based'));
         if(!isset($seed_info["indexed_file_types"]["extensions"])) {
             $seed_info["indexed_file_types"]["extensions"] =
                 $INDEXED_FILE_TYPES;
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
index 76adf1a..c1d3d23 100755
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -227,5 +227,6 @@ interface CrawlConstants
     const IS_GOPHER_URL = "dp";
     const MINIMUM_FETCH_LOOP_TIME = "dq";
     const IMAGE_LINK = "dr";
+	 const GRAPH_BASED_SUMMARIZER = 'ds';
 }
 ?>
diff --git a/lib/graph_based_summarizer.php b/lib/graph_based_summarizer.php
new file mode 100644
index 0000000..07f15cb
--- /dev/null
+++ b/lib/graph_based_summarizer.php
@@ -0,0 +1,478 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Mangesh Dahale mangeshadahale@gmail.com
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+/**
+ * Reads in constants used as enums used for storing web sites
+ */
+require_once BASE_DIR."/lib/crawl_constants.php";
+/**
+ * Contains the max_description_length for the summary
+ */
+require_once BASE_DIR."/lib/processors/page_processor.php";
+/**
+ * Contains function getTokenizer to get the object of the language specified.
+ */
+require_once BASE_DIR."/lib/phrase_parser.php";
+
+
+/**
+ * Class which may be used by TextProcessors to get a summary for a text
+ * document that may later be used for indexing. 
+ *
+ * add more comments later
+ *
+ * @author Charles Bocage charles.bocage@sjsu.edu
+ * @package seek_quarry\library
+ */
+class GraphBasedSummarizer
+{
+    /**
+     * Number of distinct terms to use in generating summary
+     */
+    const MAX_DISTINCT_TERMS = 1000;
+    /**
+     * This is a graph based summarizer
+     *
+     * @param string $doc complete raw page to generate the summary from.
+     * @param string $lang language of the page to decide which stop words to
+     *     call proper tokenizer.php of the specified language.
+     *
+     * @return array array of summary and word cloud
+     */
+    static function getGraphBasedSummary($doc, $lang)
+    {
+        $doc = self::pageProcessing($doc);
+        $formatted_doc = self::formatDoc($doc);
+        //not filtering non-ascii characters
+        $sentences = self::getSentences($doc . " ", true);
+        $sentences = self::removeStopWords($sentences, $lang);
+        $sentences = self::removePunctuation($sentences);
+        $sentences = PhraseParser::stemTermsK($sentences, $lang, true);
+        $terms = self::getTerms($sentences, $lang);
+        $termFrequencies = self::getTermFrequencies($terms, $sentences);
+        $termFrequenciesNormalized =
+            self::normalizeTermFrequencies($termFrequencies);
+        $adjacency = self::computeAdjacency($termFrequenciesNormalized,
+            $sentences, $lang);
+        $p = self::getSentenceRanks($adjacency);
+        $sentencesWithPunctuation = self::getSentences($doc . " ", true);
+        $summary = self::getSummary($sentencesWithPunctuation, $p);
+        return $summary;
+    }
+    /*
+     * Get the summary from the sentences
+     * @param array $sentences the sentences in the doc
+     * @param array $p the sentence probabilities
+     */
+    static function getSummary($sentences, $p) {
+        $result = "";
+        $resultLength = 0;
+        $n = count($p);
+        for($i = 0; $i < $n; $i++ ) {
+            $index = self::findLargestIndex($p);
+            $p[$index] = -1;
+            $sentence = $sentences[$index];
+            if ($resultLength + strlen($sentence) > 
+                PageProcessor::$max_description_len) {
+            //    self::MAX_SUMMARY_LENGTH) {
+                break;
+            } else {
+                $resultLength += strlen($sentence);
+                if ($i == 0) {
+                    $result = $sentence;
+                } else {
+                    $result = $result . " " . $sentence;
+                }
+            }
+        }
+        return $result;
+    }
+    /*
+     * find the largest value in the array and return its
+     * @param array $v the array to search for the largest value
+     * @return double the largest value found in the array
+     */
+    static function findLargestIndex($v) {
+        $result = 0;
+        $n = count($v);
+        $lastValue = -1;
+        for($i = 0; $i < $n; $i++ ) {
+            if ($v[$i] > $lastValue) {
+                $lastValue = $v[$i];
+                $result = $i;
+            }
+        }
+        return $result;
+    }
+    /*
+     * compute the sentence ranks using a version of the famous 
+     * page ranking algorithm developed by the founder of Google.
+     * @param array $adjacency the adjacency matrix generated for the 
+     *      sentences
+     * @return array the sentence ranks
+     */
+    static function getSentenceRanks($adjacency) {
+        $n = count($adjacency);
+        $oldP = array();
+        $p = array();
+        for($i = 0; $i < $n; $i++ ) {
+            $p[$i] = 1 / $n;
+        }
+        for($i = 0; $i < 10; $i++ ) {
+            $p = self::mult_mut_vec($adjacency, $p);
+        }    
+        return $p;
+    }
+    /*
+     * Compute the difference of squares
+     * @param array $v the  minuend vector
+     * @param array $m the subtrahend vector
+     * @result double the difference of the squares of vectors
+     */
+    static function squareDiff($v, $w) {
+        $result = 0;
+        $n = count($v);
+        for($i = 0; $i < $n; $i++ ) {
+            $subtaction = $v[$i] - $w[$i];
+            $result += $subtaction * $subtaction;
+        }
+        return $result;
+    }
+    /*
+     * Perform matrix multiplication on a matrix and a vector
+     * @param array $termFrequenciesNormalized the array of term frequencies
+     * @param array $sentences the sentences in the doc
+     * @return array the new vector after it has been multiplied
+     */
+    static function mult_mut_vec($mat, $vec) {
+        $result = array();
+        $n = count($vec);
+        for($i = 0; $i < $n; $i++ ) {
+            $result[$i] = 0;
+            for($j = 0; $j < $n; $j++ ) {
+                $result[$i] += $mat[$i][$j] * $vec[$j];
+            }
+        }
+        return $result;
+    }
+    /*
+     * Compute the adjacency matrix based on its distortion measure
+     * @param array $termFrequenciesNormalized the array of term frequencies
+     * @param array $sentences the sentences in the doc
+     * @param string $lang locale tag for stemming
+     * @return array the array of sentence adjacency
+     */
+    static function computeAdjacency($termFrequenciesNormalized, $sentences,
+        $lang) {
+        $result = array(array());
+        $n = count($sentences);
+        for($i = 0; $i < $n; $i++ ) {
+            $result[$i][$i] = 0;
+            for($j = $i + 1; $j < $n; $j++ ) {
+                $result[$i][$j] = $result[$j][$i] = 
+                    self::find_distortion($sentences[$i], $sentences[$j],
+                    $termFrequenciesNormalized, $lang);
+            }
+        }
+        return $result;
+    }
+    /*
+     * Remove punctuation
+     * @param array $sentences the sentences in the doc
+     * @return array the array of sentences with the punctuation removed
+     */
+     static function removePunctuation($sentences) {
+        $n = count($sentences);
+        for($i = 0; $i < $n; $i++ ) {
+            $sentences[$i] = trim(preg_replace('/[^a-z0-9]+/i', ' ', 
+                $sentences[$i]));
+        }
+        return $sentences;
+     }
+    /*
+     * Remove the stop words from the array of sentences
+     * @param array $sentences the sentences in the doc
+     * @param string $lang locale tag for stemming
+     * @return array the array of sentences with the stop words removed
+     */
+    static function removeStopWords($sentences, $lang) {
+        $n = count($sentences);
+        $stop_obj = PhraseParser::getTokenizer($lang);
+        if($stop_obj && method_exists($stop_obj, "stopwordsRemover")) {
+            for($i = 0; $i < $n; $i++ ) {
+                $sentences[$i] = $stop_obj->stopwordsRemover(
+                    self::formatDoc($sentences[$i]));
+             }
+        }
+        return $sentences;
+    }    
+    /*
+     * Calculate the term frequencies.
+     * @param array $terms the list of all terms in the doc
+     * @param array $sentences the sentences in the doc
+     * @return array a two dimensional array where the word is the key and
+     *      the frequency is the value
+     */
+    static function getTermFrequencies($terms, $sentences) {
+        $t = count($terms);
+        $n = count($sentences);
+        $nk = array();
+        $nk = array_fill(0, $t, 0);
+        $nt = array();
+        for($i = 0; $i < $n; $i++) {
+            for($j = 0; $j < $t; $j++) {
+                if(strpos($sentences[$i], $terms[$j]) !== false) {
+                    $nk[$j]++;
+                }
+            }
+        }
+        for($i = 0; $i <  count($nk); $i++ ) {
+            $termFrequencies[$terms[$i]] = $nk[$i];
+        }
+        return $termFrequencies;
+    }
+    /*
+     * Get the terms from an array of sentences
+     * @param array $sentences the sentences in the doc
+     * @param string $lang locale tag for stemming
+     * $return array an array of terms in the array of sentences
+     */       
+    static function getTerms($sentences, $lang) {
+        $terms = array();
+        foreach($sentences as $sentence) {
+            $terms = array_merge($terms,
+                PhraseParser::segmentSegment($sentence, $lang));
+        }
+        $terms = array_filter($terms);
+        $terms_counts = array_count_values($terms);
+        arsort($terms_counts);
+        $terms_counts = array_slice($terms_counts, 0, self::MAX_DISTINCT_TERMS);
+        $terms = array_unique(array_keys($terms_counts));
+        $t = count($terms);
+        if($t == 0) {
+            return array("", "");
+        }
+        return $terms;
+    }
+    /**
+     * Calculates how many sentences to put in the summary to match the
+     * MAX_DESCRIPTION_LEN.
+     *
+     * @param array $sentences sentences in doc in their original order
+     * @param array $sim associative array of sentence-number-in-doc =>
+     *      similarity score to centroid (sorted from highest to lowest score).
+     * @return int number of sentences
+     */
+    static function summarySentenceCount($sentences, $sim) {
+        $top = NULL;
+        $count = 0;
+        foreach($sim as $key => $value) {
+            if($count < PageProcessor::$max_description_len) {
+                $count += strlen($sentences[$key]);
+                $top++;
+            }
+        }
+        return $top;
+    }
+    /**
+     * Breaks any content into sentences by splitting it on spaces or carriage
+     *   returns
+     * @param string $content complete page.
+     * @param boolean $keepPunctuation whether to keep the punctuation or not.
+     * @return array array of sentences from that content.
+     */
+    static function getSentences($content, $keepPunctuation) {
+        if ($keepPunctuation) {
+            $sentences = 
+                preg_split('/(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|\!)/',
+                $content, 0, PREG_SPLIT_NO_EMPTY);
+            $n = count($sentences);
+            for($i = 0; $i < $n; $i++ ) {
+                $sentences[$i] = trim($sentences[$i]);
+            }
+            return $sentences;
+        } else {
+            return preg_split(
+                '/(\.|\||\!|\?|！|？|。)\s+|(\n|\r)(\n|\r)+|\s{5}/',
+                $content, 0, PREG_SPLIT_NO_EMPTY);
+        }
+    }
+    /*
+     * Normalize the term frequencies based on the sum of the squares.
+     * @param array $termFrequencies the array with the terms as the key
+     *      and its frequency as the value
+     * @return array array of term frequencies normalized
+     */
+    static function normalizeTermFrequencies($termFrequencies) {
+        $sumOfSquares = 0;
+        $resultSum = 0;
+        foreach ($termFrequencies as $k => $v) {
+            $sumOfSquares += ($v * $v);
+        }     
+        $squareRoot = sqrt($sumOfSquares);
+        foreach ($termFrequencies as $k => $v) {
+            $result[$k] = ($v / $squareRoot);
+        }
+        foreach ($result as $k => $v) {
+            $resultSum += $v;
+        }
+        return $result;   
+    }
+    /**
+     *   Calcluate the distortion measure.
+     *   1. Check each word in sentence1 to see if it exists in sentence2.
+     *   If the word X of sentence1 does not exist in sentence2,
+     *   square the score of word X and add to the sum
+     *   and increase the number of not-common words by one.
+     *   2. In case the word X is common between sentence1 and
+     *   sentence2, calculate its frequency in sentence2 and subtract
+     *   it from the score of word X, then square and add to
+     *   sum.
+     *   3. Then check the sentence2 to find its not-common words
+     *   with sentence1, in case the word Y is not in sentence1,
+     *   square the score of word Y and add to sum and increase
+     *   the number of not-common words by one.
+     *   4. At the end, calcualte the distortion between sentence1 and
+     *   sentence2 by dividing sum by the number of not-common
+     *   words.
+     * @param string $firstSentence the first sentence to compare
+     * @param string $secondSentence the second sentence to compare
+     * @param string $termFrequencies the term frequency of the sentences
+     * @param string $lang locale tag for stemming
+     */
+    static function find_distortion($firstSentence, $secondSentence, 
+        $termFrequencies, $lang) {
+        $result = 0;
+        $firstSentenceSplit = preg_split('/ +/', $firstSentence);
+        $secondSentenceSplit = preg_split('/ +/', $secondSentence);
+        $sum = 0;
+        $nonCommonWords = 0;
+        $n = count($firstSentenceSplit);
+        for($i = 0; $i < $n; $i++ ) {
+            $wordToSearchFor = trim($firstSentenceSplit[$i]);
+            if ($wordToSearchFor != "") {
+                preg_match_all("/ " . $wordToSearchFor . " /", 
+                    $secondSentence, $matches);
+                if (count($matches[0]) == 0) {
+                    $sum += ($termFrequencies[$wordToSearchFor] * 
+                        $termFrequencies[$wordToSearchFor]);
+                    $nonCommonWords++;
+                } else {
+                    $terms = self::getTerms(array($secondSentence), $lang);
+                    $tempTermFrequencies =
+                    self::getTermFrequencies($terms, array($secondSentence));
+                    $tempTermFrequenciesNormalized =
+                    self::normalizeTermFrequencies($tempTermFrequencies);
+                    $newTermFequency = $termFrequencies[$wordToSearchFor] - 
+                        $tempTermFrequenciesNormalized[$wordToSearchFor];
+                    $sum += ($newTermFequency * $newTermFequency);
+                }
+            }
+        }
+        $n = count($secondSentenceSplit);
+        for($i = 0; $i < $n; $i++ ) {
+            $wordToSearchFor = trim($secondSentenceSplit[$i]);
+            if ($wordToSearchFor != "") {
+                preg_match_all("/ " . trim($wordToSearchFor) . " /", 
+                    $firstSentence, $matches);
+                if (count($matches[0]) == 0) {
+                    $sum += ($termFrequencies[$wordToSearchFor] * 
+                        $termFrequencies[$wordToSearchFor]);
+                    $nonCommonWords++;
+                }
+            }
+        }
+        if ($nonCommonWords != 0) {
+            $result = $sum / $nonCommonWords;
+        }
+        return $result;        
+    }
+    /**
+     * Formats the sentences to remove all characters except words,
+     *   digits and spaces
+     * @param string $sent complete page.
+     * @return string formatted sentences.
+     */
+    static function formatSentence($sent) {
+        $sent = trim(preg_replace('/[^\p{L}\p{N}\s]+/u',
+            ' ', mb_strtolower($sent)));
+        return $sent;
+    }
+    /**
+     * Formats the document to remove carriage returns, hyphens and digits
+     * as we will not be using digits in word cloud.
+     * The formatted document generated by this function is only used to
+     * compute centroid.
+     * @param string $content formatted page.
+     * @return string formatted document.
+     */
+    static function formatDoc($content) {
+        $substitute = array('/[\n\r\-]+/', '/[^\p{L}\s\.]+/u', '/[\.]+/');
+        $content = preg_replace($substitute, ' ', mb_strtolower($content));
+        return $content;
+    }
+    /**
+     * This function does an additional processing on the page
+     * such as removing all the tags from the page
+     * @param string $page complete page.
+     * @return string processed page.
+     */
+    static function pageProcessing($page) {
+        $substitutions = array('@<script[^>]*?>.*?</script>@si',
+            '/\&nbsp\;|\&rdquo\;|\&ldquo\;|\&mdash\;/si',
+            '@<style[^>]*?>.*?</style>@si', '/[\^\(\)]/',
+            '/\[(.*?)\]/', '/\t\n/'
+        );
+        $page = preg_replace($substitutions, ' ', $page);
+        $page = preg_replace('/\s{2,}/', ' ', $page);
+        $new_page = preg_replace("/\<br\s*(\/)?\s*\>/", "\n", $page);
+        $changed = false;
+        if($new_page != $page) {
+            $changed = true;
+            $page = $new_page;
+        }
+        $page = preg_replace("/\<\/(h1|h2|h3|h4|h5|h6|table|tr|td|div|".
+            "p|address|section)\s*\>/", "\n\n", $page);
+        $page = preg_replace("/\<a/", " <a", $page);
+        $page = preg_replace("/\&\#\d{3}(\d?)\;|\&\w+\;/", " ", $page);
+        $page = preg_replace("/\</", " <", $page);
+        $page = strip_tags($page);
+
+        if($changed) {
+            $page = preg_replace("/(\r?\n[\t| ]*){2}/", "\n", $page);
+        }
+        $page = preg_replace("/(\r?\n[\t| ]*)/", "\n", $page);
+        $page = preg_replace("/\n\n\n+/", "\n\n", $page);
+        return $page;
+    }
+}
+?>
diff --git a/lib/phrase_parser.php b/lib/phrase_parser.php
index fb5fd6e..cb42347 100755
--- a/lib/phrase_parser.php
+++ b/lib/phrase_parser.php
@@ -736,6 +736,40 @@ class PhraseParser
         return $stems;
     }
     /**
+	* Splits supplied string based on white space, then stems each
+     * terms according to the stemmer for $lanf if exists
+     *
+     * @param mixed $string_or_array to extract stemmed terms from
+     * @param string $lang IANA tag to look up stemmer under
+     * @param string $keepEmpties whether to keep empty sentences or not
+     * @return array stemmed terms if stemmer; terms otherwise
+     */
+    static function stemTermsK($string_or_array, $lang, $keepEmpties)
+    {
+        if($string_or_array == array() ||
+            $string_or_array == "") { return array();}
+        if(is_array($string_or_array)) {
+            $terms = & $string_or_array;
+        } else {
+            $terms = mb_split("[[:space:]]", $string_or_array);
+        }
+        $stem_obj = self::getTokenizer($lang);
+        $stems = array();
+        if($stem_obj != NULL && $stem_obj->stem("a") !== false) {
+            foreach($terms as $term) {
+                if(trim($term) == "") {
+                    if (!$keepEmpties) {
+                        continue;
+                    }
+                }
+                $stems[] = $stem_obj->stem($term);
+            }
+        } else {
+            return $terms;
+        }
+        return $stems;
+    }
+    /**
      * Loads and instantiates a tokenizer object for a language if exists
      *
      * @param string $lang IANA tag to look up stemmer under
diff --git a/lib/processors/html_processor.php b/lib/processors/html_processor.php
index 0c62c4f..96dadda 100755
--- a/lib/processors/html_processor.php
+++ b/lib/processors/html_processor.php
@@ -52,6 +52,10 @@ require_once BASE_DIR."/lib/url_parser.php";
 */
 require_once BASE_DIR."/lib/centroid_summarizer.php";
 /**
+* Get the graph based summary
+*/
+require_once BASE_DIR."/lib/graph_based_summarizer.php";
+/**
  * For guessing language from charset
  */
 require_once BASE_DIR."/lib/locale_functions.php";
@@ -83,6 +87,7 @@ class HtmlProcessor extends TextProcessor
     {
         $summary = NULL;
         $is_centroid = $this->summarizer_option == self::CENTROID_SUMMARIZER;
+        $is_graph_based = $this->summarizer_option == self::GRAPH_BASED_SUMMARIZER;
         if(is_string($page)) {
             $page = preg_replace('/\&nbsp\;|\&rdquo\;|\&ldquo\;|\&mdash\;/si',
                 ' ',$page);
@@ -105,6 +110,11 @@ class HtmlProcessor extends TextProcessor
                     $summary[self::DESCRIPTION] = $summary_cloud[0];
                     $summary[self::WORD_CLOUD] = $summary_cloud[1];
                     crawlLog("..Using Centroid Summarizer");
+                } elseif($is_graph_based) {
+                    $summary[self::DESCRIPTION] = 
+                        GraphBasedSummarizer::getGraphBasedSummary($dom_page,
+                        $summary[self::LANG]);
+                    crawlLog("..Using Graph Based Summarizer");
                 } else {
                     $summary[self::DESCRIPTION] = self::description($dom,
                         $dom_page);
diff --git a/lib/processors/page_processor.php b/lib/processors/page_processor.php
index dcd83f4..216027e 100644
--- a/lib/processors/page_processor.php
+++ b/lib/processors/page_processor.php
@@ -57,7 +57,8 @@ abstract class PageProcessor implements CrawlConstants
     var $plugin_instances;
     /**
      * Stores the name of the summarizer used for crawling.
-     * Possible values are self::BASIC and self::CENTROID_SUMMARIZER
+	 * Possible values are self::BASIC, self::GRAPH_BASED_SUMMARIZER
+     * and self::CENTROID_SUMMARIZER
      * @var string
      */
      var $summarizer_option;
@@ -76,7 +77,8 @@ abstract class PageProcessor implements CrawlConstants
      *     processor
      * @param int $max_description_len maximal length of a page summary
      * @param int $summarizer_option CRAWL_CONSTANT specifying what kind
-     *      of summarizer to use self::BASIC_SUMMARIZER or
+     *      of summarizer to use self::BASIC_SUMMARIZER,
+     *      self::GRAPH_BASED_SUMMARIZER and self::CENTROID_SUMMARIZER
      *      self::CENTROID_SUMMARIZER
      */
     function __construct($plugins = array(), $max_description_len = NULL,
diff --git a/lib/processors/text_processor.php b/lib/processors/text_processor.php
index 245c6e5..0d3b251 100755
--- a/lib/processors/text_processor.php
+++ b/lib/processors/text_processor.php
@@ -52,6 +52,10 @@ require_once BASE_DIR."/lib/processors/page_processor.php";
  */
 require_once BASE_DIR."/lib/centroid_summarizer.php";
 /**
+ * Get the graph based summary
+ */
+require_once BASE_DIR."/lib/graph_based_summarizer.php";
+/**
  * So can extract parts of the URL if need to guess lang
  */
 require_once BASE_DIR."/lib/url_parser.php";
diff --git a/locale/en-US/configure.ini b/locale/en-US/configure.ini
index 0cdb46f..7b3e23f 100755
--- a/locale/en-US/configure.ini
+++ b/locale/en-US/configure.ini
@@ -417,6 +417,9 @@ crawl_component_basic = "Basic"
 ; crawl_component.php line: 855
 crawl_component_centroid = "Centroid"
 ;
+; crawl_component.php line: 857
+crawl_component_graph_based = "Graph Based"
+;
 ; crawl_component.php line: 1141
 crawl_component_page_options_updated = "Page Options Updated!"
 ;
-- 
1.9.5.github.0

