_binary From f35b1ee4ad651385d589ddcc61c8a95e578e6fd5 Mon Sep 17 00:00:00 2001
From: diesal9 <charles.bocage@sjsu.edu>
Date: Sun, 6 Mar 2016 10:01:41 -0800
Subject: [PATCH] I have added a method to compress sentence during
 summarization.  I also updated the TextProcessor file to use all of the
 available summarizers

---
 src/configs/Config.php                             |  26 ++--
 src/library/PhraseParser.php                       |  64 ++++++---
 src/library/processors/HtmlProcessor.php           |   8 +-
 src/library/processors/TextProcessor.php           |  78 +++++++++--
 src/library/summarizers/CentroidSummarizer.php     |  11 +-
 .../summarizers/CentroidWeightedSummarizer.php     |  22 +--
 src/library/summarizers/GraphBasedSummarizer.php   |  10 +-
 src/library/summarizers/ScrapeSummarizer.php       |  12 +-
 src/locale/en_US/resources/Tokenizer.php           | 151 ++++++++++++++++-----
 9 files changed, 289 insertions(+), 93 deletions(-)

diff --git a/src/configs/Config.php b/src/configs/Config.php
index 0d02f8b..04acdf3 100755
--- a/src/configs/Config.php
+++ b/src/configs/Config.php
@@ -68,7 +68,7 @@ if (file_exists(__DIR__."/../../vendor/autoload.php")) {
         // use forward-slashes, add ./php
         $unixify_class_name = "/".str_replace('\\', '/', $relative_class) .
             '.php';
-        foreach($check_dirs as $dir) {
+        foreach ($check_dirs as $dir) {
             $file = $dir . $unixify_class_name;
             if (file_exists($file)) {
                 require $file;
@@ -108,7 +108,7 @@ function nsconddefine($constant, $value)
         define("seekquarry\\yioop\\configs\\" . $constant, $value);
     }
 }
-/** 
+/**
  * Version number for upgrade function
  * @var int
  */
@@ -128,7 +128,7 @@ function computeBaseUrl()
     $server_port = isset($_SERVER['HTTP_X_FORWARDED_PORT']) ?
         $_SERVER['HTTP_X_FORWARDED_PORT'] : (isset($_SERVER['SERVER_PORT']) ?
             $_SERVER['SERVER_PORT'] : 80);
-    $http = (!empty($_SERVER['HTTPS']) || $server_port == 443) ? 
+    $http = (!empty($_SERVER['HTTPS']) || $server_port == 443) ?
         "https://" : "http://";
     $port = ( ($http == "http://" && ($server_port != 80) ||
         ($http == "https://" && $server_port != 443))) ?
@@ -225,8 +225,8 @@ if (file_exists(BASE_DIR."/configs/LocalConfig.php")) {
 }
 /** setting Profile.php to something else in LocalConfig.php allows one to have
  *  two different yioop instances share the same work_directory but maybe have
- *  different configuration settings. This might be useful if one was production
- *  and one was more dev.
+ *  different configuration settings. This might be useful if one was
+ *  production and one was more dev.
  */
 nsconddefine('PROFILE_FILE_NAME', "/Profile.php");
 nsconddefine('MAINTENANCE_MESSAGE', <<<EOD
@@ -242,7 +242,7 @@ if (MAINTENANCE_MODE && $_SERVER["SERVER_ADDR"] != $_SERVER["REMOTE_ADDR"]) {
 nsdefine('DEFAULT_WORK_DIRECTORY', PARENT_DIR . "/work_directory");
 
 if (!nsdefined('WORK_DIRECTORY')) {
-/*+++ The next block of code is machine edited, change at 
+/*+++ The next block of code is machine edited, change at
 your own risk, please use configure web page instead +++*/
 nsdefine('WORK_DIRECTORY', DEFAULT_WORK_DIRECTORY);
 /*++++++*/
@@ -281,7 +281,7 @@ nsdefine('EMAIL_AND_QUESTIONS_RECOVERY', 2);
  */
 nsconddefine('FIAT_SHAMIR_ITERATIONS', 20);
 if (file_exists(WORK_DIRECTORY . PROFILE_FILE_NAME)) {
-    if((file_exists(WORK_DIRECTORY . "/locale/en-US") &&
+    if ((file_exists(WORK_DIRECTORY . "/locale/en-US") &&
         !file_exists(WORK_DIRECTORY . "/locale/en_US"))
         || (file_exists(WORK_DIRECTORY . "/app/locale/en-US") &&
         !file_exists(WORK_DIRECTORY . "/app/locale/en_US"))) {
@@ -522,8 +522,8 @@ defineMemoryProfile();
  */
 function defineMemoryProfile()
 {
-
-    $memory = 4000000000; //assume have at least 4GB on a Mac (could use vm_stat)
+    //assume have at least 4GB on a Mac(could use vm_stat)
+    $memory = 4000000000;
     if (strstr(PHP_OS, "WIN")) {
         if (function_exists("exec")) {
             exec('wmic memorychip get capacity', $memory_array);
@@ -654,9 +654,9 @@ nsconddefine("MAX_VIDEO_CONVERT_SIZE", 2000000000);
  */
 nsconddefine('MAX_FILE_TIMESTAMP_LIMIT', 600);
 /**
- * This mail timestamp limit allows mail server to create a new file 
- * and write next mailer batch in the new file. Otherwise, new mailer 
- * batch will be written in old file. For eg. new file will be created every 
+ * This mail timestamp limit allows mail server to create a new file
+ * and write next mailer batch in the new file. Otherwise, new mailer
+ * batch will be written in old file. For eg. new file will be created every
  * 5 minutes as per below value.
  */
 nsconddefine('MAX_MAIL_TIMESTAMP_LIMIT', 300);
@@ -882,3 +882,5 @@ nsconddefine('ALLOW_FREE_ROOT_CREDIT_PURCHASE', false);
 nsconddefine('AD_DATE_FORMAT','Y-m-d');
 /** advertisement logo*/
 nsconddefine('AD_LOGO','resources/adv-logo.png');
+/** sentence compression enabled or not*/
+nsconddefine('SENTENCE_COMPRESSION_ENABLED', true);
\ No newline at end of file
diff --git a/src/library/PhraseParser.php b/src/library/PhraseParser.php
index 769d5fd..8d45afc 100755
--- a/src/library/PhraseParser.php
+++ b/src/library/PhraseParser.php
@@ -257,7 +257,7 @@ class PhraseParser
         $string = preg_replace_callback($ampersand_pattern,
             function($matches) {
                 $result = mb_strtolower(
-                    mb_ereg_replace("\s*(\'n|\'N|\&)\s*", "_and_",$matches[0]));
+                    mb_ereg_replace("\s*(\'n|\'N|\&)\s*","_and_",$matches[0]));
                 return $result;
             },
             $string);
@@ -298,7 +298,7 @@ class PhraseParser
         $pos_lists = [];
         $maximal_phrases = [];
         $terms = self::stemCharGramSegment($string, $lang);
-        if ($terms == []) { 
+        if ($terms == []) {
             return [];
         }
         $suffix_tree = new SuffixTree($terms);
@@ -367,7 +367,8 @@ class PhraseParser
         $identifiers = "($id_start)($repeat)*";
         //Keywords
         $keywords_part1 = "abstract|assert|boolean|break|byte|case|catch|char";
-        $keywords_part2 = "class|const|continue|default|do|double|else|extends";
+        $keywords_part2 =
+            "class|const|continue|default|do|double|else|extends";
         $keywords_part3 = "final|finally|float|for|goto|if|implements|import";
         $keywords_part4 = "instanceof|int|interface|long|native|new|package";
         $keywords_part5 = "private|protected|public|return|short|static";
@@ -430,8 +431,8 @@ class PhraseParser
             $position = preg_match("/$tokens/", $string, $matches,
                 PREG_OFFSET_CAPTURE);
             if (isset($matches[0][0])) {
-                $text = preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/", "\n",
-                    $matches[0][0]);
+                $text = preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/",
+                    "\n", $matches[0][0]);
                 $lines = explode("\n", trim($text));
                 $line = implode(' ', $lines);
                 $data = preg_replace("/[\t\s]+/", ' ', trim($line));
@@ -527,11 +528,13 @@ class PhraseParser
         $single_quoted_element2 = "($graphic|$esc_u|\\s|\\t|\")*";
         $single_quoted_string1 = "(\"$single_quoted_element1\")";
         $single_quoted_string2 = "(\'$single_quoted_element2\')";
-        $single_quoted_string = "$single_quoted_string1|$single_quoted_string2";
+        $single_quoted_string =
+            "$single_quoted_string1|$single_quoted_string2";
         $triple_quoted_element = "$text_chars|$esc_u";
         $triple_quoted_string1 = "(\"\"\"($triple_quoted_element)*?\"\"\")";
         $triple_quoted_string2 = "(\'\'\'($triple_quoted_element)*?\'\'\')";
-        $triple_quoted_string = "$triple_quoted_string1|$triple_quoted_string2";
+        $triple_quoted_string =
+            "$triple_quoted_string1|$triple_quoted_string2";
         $string_literal_part1 = "($raw_opt)?($triple_quoted_string)";
         $string_literal_part2 = "($raw_opt)?($single_quoted_string)";
         $string_literal = "$string_literal_part1|$string_literal_part2";
@@ -564,8 +567,8 @@ class PhraseParser
             $position = preg_match("/$tokens/", $string, $matches,
                 PREG_OFFSET_CAPTURE);
             if (isset($matches[0][0])) {
-                $text = preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/", "\n",
-                    $matches[0][0]);
+                $text = preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/",
+                    "\n", $matches[0][0]);
                 $lines = explode("\n", trim($text));
                 $line = implode(' ', $lines);
                 $data = preg_replace("/[\t\s]+/", ' ', trim($line));
@@ -622,7 +625,8 @@ class PhraseParser
     /**
      * Returns the characters n-grams for the given terms where n is the length
      * Yioop uses for the language in question. If a stemmer is used for
-     * language then n-gramming is not done and this just returns an empty array
+     * language then n-gramming is not done and this just returns an empty
+     * array
      *
      * @param array $terms the terms to make n-grams for
      * @param string $lang locale tag to determine n to be used for n-gramming
@@ -743,10 +747,10 @@ class PhraseParser
         mb_internal_encoding("UTF-8");
         $lower_lang = strtolower($lang); //try to avoid case sensitivity issues
         $lang_parts = explode("-", $lang);
-        if(!isset($lang_parts[1])) {
+        if (!isset($lang_parts[1])) {
             $tokenizer_list = glob(C\LOCALE_DIR .
                 "/$lang*/resources/Tokenizer.php");
-            if(isset($tokenizer_list[0])) {
+            if (isset($tokenizer_list[0])) {
                 $tag = substr($tokenizer_list[0], strlen(C\LOCALE_DIR) + 1,
                     - strlen("/resources/Tokenizer.php"));
             } else {
@@ -757,7 +761,7 @@ class PhraseParser
             if (!file_exists(C\LOCALE_DIR . "/$tag/resources/Tokenizer.php")) {
                 $tokenizer_list = glob(C\LOCALE_DIR .
                     "/{$lang_parts[0]}*/resources/Tokenizer.php");
-                if(isset($tokenizer_list[0])) {
+                if (isset($tokenizer_list[0])) {
                     $tag = substr($tokenizer_list[0], strlen(C\LOCALE_DIR) + 1,
                         - strlen("/resources/Tokenizer.php"));
                 } else {
@@ -1124,7 +1128,7 @@ class PhraseParser
             $cur_pos--;
             $char_added =  mb_substr($segment, $cur_pos, 1);
             $is_space = trim($char_added) == "";
-            if ($is_space && $was_space) { 
+            if ($is_space && $was_space) {
                 continue;
             } else if ($is_space) {
                 $was_space = true;
@@ -1154,8 +1158,8 @@ class PhraseParser
                 }
             }
             if (!$is_suffix) {
-                if(mb_strlen($word_guess) > 1 &&
-                    !self::oneWord($word_guess, $locale, $additional_regexes)) {
+                if (mb_strlen($word_guess) > 1 &&
+                    !self::oneWord($word_guess, $locale, $additional_regexes)){
                     $out_segment .= " ".strrev(mb_substr($word_guess, 1));
                     $word_guess = $char_added;
                 } else {
@@ -1209,7 +1213,8 @@ XXX sex slut nymphomaniac MILF lolita lesbian sadomasochism
 bondage fisting erotic vagina Tribadism penis facial hermaphrodite
 transsexual tranny bestiality snuff boob fondle tit
 blowjob lap cock dick hardcore pr0n fuck pussy penetration ass
-cunt bisexual prostitution screw ass masturbation clitoris clit suck whore bitch
+cunt bisexual prostitution screw ass masturbation clitoris clit suck whore
+bitch
 bellaco cachar chingar shimar chinquechar chichar clavar coger culear hundir
 joder mámalo singar cojon carajo caray bicho concha chucha chocha
 chuchamadre coño panocha almeja culo fundillo fundío puta puto teta
@@ -1254,4 +1259,29 @@ vaffanculo fok hoer kut lul やりまん 打っ掛け
         $score = $num_unsafe_terms * $unsafe_count/($len + 1);
         return $score;
     }
+    /**
+     * Call the appropriate tokenizer sentence compression method
+     *
+     * @param string $sentence_to_compress the sentence to compress
+     * @param string $lang locale tag for stemming
+     * @return the compressed sentence
+     */
+    public static function compressSentence($sentence_to_compress,
+        $lang = null)
+    {
+        $result = $sentence_to_compress;
+        if (C\SENTENCE_COMPRESSION_ENABLED) {
+            if ($lang != null) {
+                $segment_obj = self::getTokenizer($lang);
+            } else {
+                $segment_obj = null;
+            }
+            if ($segment_obj != null && method_exists($segment_obj,
+                "compressSentence")) {
+                $result =
+                    $segment_obj->compressSentence($sentence_to_compress);
+            }
+        }
+        return $result;
+    }
 }
diff --git a/src/library/processors/HtmlProcessor.php b/src/library/processors/HtmlProcessor.php
index ee0e768..364f652 100755
--- a/src/library/processors/HtmlProcessor.php
+++ b/src/library/processors/HtmlProcessor.php
@@ -67,7 +67,8 @@ class HtmlProcessor extends TextProcessor
     public function __construct($plugins = [], $max_description_len = null,
         $summarizer_option = self::BASIC_SUMMARIZER)
     {
-        parent::__construct($plugins, $max_description_len, $summarizer_option);
+        parent::__construct($plugins, $max_description_len,
+            $summarizer_option);
         /** Register File Types We Handle*/
         $add_extensions = ["asp", "aspx", "cgi", "cfm", "cfml", "do", "htm",
             "html", "jsp", "php", "pl", "py", "shtml"];
@@ -144,7 +145,8 @@ class HtmlProcessor extends TextProcessor
                     L\crawlLog("..Using Graph Based Summarizer");
                 } else {
                     $summary[self::DESCRIPTION] =
-                        ScrapeSummarizer::getBasicSummary($dom, $dom_page);
+                        ScrapeSummarizer::getBasicSummary($dom, $dom_page,
+                        $summary[self::LANG]);
                     L\crawlLog("..Using Basic Summarizer");
                 }
                 $crude = false;
@@ -614,4 +616,4 @@ class HtmlProcessor extends TextProcessor
         $text = preg_replace('/\</', ' <', $text);
         return strip_tags($text);
     }
-}
+}
\ No newline at end of file
diff --git a/src/library/processors/TextProcessor.php b/src/library/processors/TextProcessor.php
index 703d69d..fcdc9d7 100755
--- a/src/library/processors/TextProcessor.php
+++ b/src/library/processors/TextProcessor.php
@@ -32,9 +32,10 @@ namespace seekquarry\yioop\library\processors;
 
 use seekquarry\yioop\configs as C;
 use seekquarry\yioop\library as L;
+use seekquarry\yioop\library\UrlParser;
 use seekquarry\yioop\library\summarizers\CentroidSummarizer;
 use seekquarry\yioop\library\summarizers\GraphBasedSummarizer;
-use seekquarry\yioop\library\UrlParser;
+use seekquarry\yioop\library\summarizers\ScrapeSummarizer;
 use seekquarry\yioop\library\summarizers\CentroidWeightedSummarizer;
 
 /**
@@ -65,7 +66,8 @@ class TextProcessor extends PageProcessor
     public function __construct($plugins = [], $max_description_len = null,
         $summarizer_option = self::BASIC_SUMMARIZER)
     {
-        parent::__construct($plugins, $max_description_len, $summarizer_option);
+        parent::__construct($plugins, $max_description_len,
+            $summarizer_option);
         /** Register File Types We Handle*/
         $add_extensions = ["csv", "tab", "tsv", "txt"];
         self::$indexed_file_types = array_merge(self::$indexed_file_types,
@@ -90,19 +92,41 @@ class TextProcessor extends PageProcessor
     public function process($page, $url)
     {
         $summary = null;
+        $is_centroid = $this->summarizer_option == self::CENTROID_SUMMARIZER;
+        $is_graph_based =
+            $this->summarizer_option == self::GRAPH_BASED_SUMMARIZER;
+        $is_centroid_weighted =
+            $this->summarizer_option == self::CENTROID_WEIGHTED_SUMMARIZER;
         if (is_string($page)) {
+            $dom_page = preg_replace('@<style[^>]*?>.*?</style>@si', ' ',
+                $page);
+            $dom = self::dom($dom_page);
             $summary[self::TITLE] = "";
-            $lang = self::calculateLang($page);
-            if ($this->summarizer_option == self::CENTROID_SUMMARIZER) {
-                $summary_cloud =
-                    CentroidSummarizer::getCentroidSummary($page, $lang);
+            $summary[self::LANG] = self::calculateLang($page);
+            if ($is_centroid) {
+                $summary_cloud = CentroidSummarizer::getCentroidSummary(
+                    $page, $summary[self::LANG]);
                 $summary[self::DESCRIPTION] = $summary_cloud[0];
                 $summary[self::WORD_CLOUD] = $summary_cloud[1];
+                L\crawlLog("..Using Centroid Summarizer");
+            } elseif ($is_centroid_weighted) {
+                $summary_cloud =
+                    CentroidWeightedSummarizer::getCentroidWeightedSummary(
+                    $page, $summary[self::LANG]);
+                $summary[self::DESCRIPTION] = $summary_cloud[0];
+                $summary[self::WORD_CLOUD] = $summary_cloud[1];
+                L\crawlLog("..Using Centroid Weighed Summarizer");
+            } elseif ($is_graph_based) {
+                $summary[self::DESCRIPTION] =
+                    GraphBasedSummarizer::getGraphBasedSummary($page,
+                    $summary[self::LANG]);
+                L\crawlLog("..Using Graph Based Summarizer");
             } else {
-                $summary[self::DESCRIPTION] = mb_substr($page, 0,
-                    self::$max_description_len);
+                $summary[self::DESCRIPTION] =
+                    ScrapeSummarizer::getBasicSummary($dom, $page,
+                    $summary[self::LANG]);
+                L\crawlLog("..Using Basic Summarizer");
             }
-            $summary[self::LANG] = $lang;
             $summary[self::LINKS] = self::extractHttpHttpsUrls($page);
             $summary[self::PAGE] = "<html><body><div><pre>".
                 strip_tags($page)."</pre></div></body></html>";
@@ -221,4 +245,38 @@ class TextProcessor extends PageProcessor
             }
         }
     }
-}
+    /**
+     * Return a document object based on a string containing the contents of
+     * a web page
+     *
+     * @param string $page   a web page
+     *
+     * @return object  document object
+     */
+    public static function dom($page)
+    {
+        /*
+             first do a crude check to see if we have at least an <html> tag
+             otherwise try to make a simplified html document from what we got
+         */
+        if (!stristr($page, "<html")) {
+            $head_tags = "<title><meta><base>";
+            $head = strip_tags($page, $head_tags);
+            $body_tags = "<frameset><frame><noscript><img><span><b><i><em>".
+                "<strong><h1><h2><h3><h4><h5><h6><p><div>".
+                "<a><table><tr><td><th><dt><dir><dl><dd>";
+            $body = strip_tags($page, $body_tags);
+            $page = "<html><head>$head</head><body>$body</body></html>";
+        }
+        $dom = new \DOMDocument();
+        //this hack modified from php.net
+        @$dom->loadHTML('<?xml encoding="UTF-8">' . $page);
+        foreach ($dom->childNodes as $item) {
+            if (isset($item->nodeType) && $item->nodeType == XML_PI_NODE) {
+                $dom->removeChild($item); // remove hack
+            }
+        }
+        $dom->encoding = "UTF-8"; // insert proper
+        return $dom;
+    }
+}
\ No newline at end of file
diff --git a/src/library/summarizers/CentroidSummarizer.php b/src/library/summarizers/CentroidSummarizer.php
index 2061760..8eed295 100644
--- a/src/library/summarizers/CentroidSummarizer.php
+++ b/src/library/summarizers/CentroidSummarizer.php
@@ -214,12 +214,17 @@ class CentroidSummarizer extends Summarizer
         sort($sum_array);
         $summary = '';
         foreach ($sum_array as $key) {
-            $summary .= $out_sentences[$key] . ". ";
+            $compressed_sentence =
+                PhraseParser::compressSentence($out_sentences[$key], $lang);
+            $summary .= $compressed_sentence . ". ";
         }
         if (self::OUTPUT_TO_FILE) {
             $output_file_contents = "";
             foreach ($sum_array as $key) {
-                $output_file_contents .= $out_sentences[$key] . ".\r\n";
+                $compressed_sentence =
+                    PhraseParser::compressSentence($out_sentences[$key],
+                    $lang);
+                $output_file_contents .= $compressed_sentence . ".\r\n";
             }
             file_put_contents(C\WORK_DIRECTORY . self::OUTPUT_FILE_PATH,
                 $output_file_contents);
@@ -347,4 +352,4 @@ class CentroidSummarizer extends Summarizer
         $page = preg_replace("/\n\n\n+/", "\n\n", $page);
         return $page;
     }
-}
+}
\ No newline at end of file
diff --git a/src/library/summarizers/CentroidWeightedSummarizer.php b/src/library/summarizers/CentroidWeightedSummarizer.php
index 20c3795..5144c5a 100644
--- a/src/library/summarizers/CentroidWeightedSummarizer.php
+++ b/src/library/summarizers/CentroidWeightedSummarizer.php
@@ -108,7 +108,7 @@ class CentroidWeightedSummarizer extends Summarizer
             $tf_average_sentence);
         usort($tf_dot_product_per_sentence, 'self::sortInAscendingOrder');
         $summary = self::getSummary($tf_dot_product_per_sentence,
-            $out_sentences);
+            $out_sentences, $lang);
         $n = count($out_sentences);
         $terms = array_filter($terms);
         $terms_counts = array_count_values($terms);
@@ -476,31 +476,35 @@ class CentroidWeightedSummarizer extends Summarizer
             product of each sentence.  It should be sorted from highest to
             lowest when it is passed to this method.
      * @param array $sentences the array of sentences to process
+     * @param string $lang language of the page to decide which stop words to
+     *     call proper tokenizer.php of the specified language.
      * @return string a string that represents the summary
      */
     public static function getSummary($tf_dot_product_per_sentence,
-            $sentences)
+            $sentences, $lang)
     {
         $result = "";
         $result_length = 0;
         $i = 0;
         foreach ($tf_dot_product_per_sentence as $k => $v) {
-            if ($result_length + strlen($sentences[$k]) >
+            $sentence = PhraseParser::compressSentence($sentences[$k],
+                $lang);
+            if ($result_length + strlen($sentence) >
                 PageProcessor::$max_description_len) {
                 break;
             } else {
-                $result_length += strlen($sentences[$k]);
+                $result_length += strlen($sentence);
                 if ($i == 0) {
                     $i = 1;
-                    $result = $sentences[$k] . ". ";
+                    $result = $sentence . ". ";
                     if (self::OUTPUT_TO_FILE) {
-                        $output_file_contents = $sentences[$k] . ". ";
+                        $output_file_contents = $sentence . ". ";
                     }
                 } else {
-                    $result .= " " . $sentences[$k] . ". ";
+                    $result .= " " . $sentence . ". ";
                     if (self::OUTPUT_TO_FILE) {
                         $output_file_contents = $output_file_contents .
-                            "\r\n" . $sentences[$k] . ". ";
+                            "\r\n" . $sentence . ". ";
                     }
                 }
             }
@@ -511,4 +515,4 @@ class CentroidWeightedSummarizer extends Summarizer
         }
         return $result;
     }
-}
+}
\ No newline at end of file
diff --git a/src/library/summarizers/GraphBasedSummarizer.php b/src/library/summarizers/GraphBasedSummarizer.php
index cf4b28c..1982b0d 100644
--- a/src/library/summarizers/GraphBasedSummarizer.php
+++ b/src/library/summarizers/GraphBasedSummarizer.php
@@ -84,15 +84,18 @@ class GraphBasedSummarizer extends Summarizer
             $sentences, $lang, $unmodified_doc);
         $p = self::getSentenceRanks($adjacency);
         $sentences_with_punctuation = self::getSentences($doc . " ", true);
-        $summary = self::getSummary($sentences_with_punctuation, $p);
+        $summary = self::getSummary($sentences_with_punctuation, $p, $lang);
         return $summary;
     }
     /**
      * Get the summary from the sentences
      * @param array $sentences the sentences in the doc
      * @param array $p the sentence probabilities
+     * @param string $lang language of the page to decide which stop words to
+     *     call proper tokenizer.php of the specified language.
+     * @return string a string that represents the summary
      */
-    public static function getSummary($sentences, $p)
+    public static function getSummary($sentences, $p, $lang)
     {
         $output_file_contents = "";
         $result = "";
@@ -101,7 +104,8 @@ class GraphBasedSummarizer extends Summarizer
         for ($i = 0; $i < $n; $i++ ) {
             $index = self::findLargestIndex($p);
             $p[$index] = -1;
-            $sentence = $sentences[$index];
+            $sentence = PhraseParser::compressSentence($sentences[$index],
+                $lang);
             if ($result_length + strlen($sentence) >
                 PageProcessor::$max_description_len) {
                 break;
diff --git a/src/library/summarizers/ScrapeSummarizer.php b/src/library/summarizers/ScrapeSummarizer.php
index 7dd89c0..66d4899 100644
--- a/src/library/summarizers/ScrapeSummarizer.php
+++ b/src/library/summarizers/ScrapeSummarizer.php
@@ -56,12 +56,14 @@ class ScrapeSummarizer extends Summarizer
      *
      * @param object $dom   a document object to extract a description from.
      * @param string $page original page string to extract description from
+     * @param string $lang language of the page to decide which stop words to
+     *     call proper tokenizer.php of the specified language.
      *
      * @return string the summary
      */
-    public static function getBasicSummary($dom, $dom_page)
+    public static function getBasicSummary($dom, $dom_page, $lang)
     {
-        return self::description($dom, $dom_page);
+        return self::description($dom, $dom_page, $lang);
     }
     /**
      * Returns descriptive text concerning a webpage based on its document
@@ -69,9 +71,11 @@ class ScrapeSummarizer extends Summarizer
      *
      * @param object $dom   a document object to extract a description from.
      * @param string $page original page string to extract description from
+     * @param string $lang language of the page to decide which stop words to
+     *     call proper tokenizer.php of the specified language.
      * @return string a description of the page
      */
-    public static function description($dom, $page)
+    public static function description($dom, $page, $lang)
     {
         $xpath = new \DOMXPath($dom);
         $metas = $xpath->evaluate("/html//meta");
@@ -134,6 +138,7 @@ class ScrapeSummarizer extends Summarizer
                 $first_len = $add_len;
             }
             foreach ($data as $datum) {
+                $datum = PhraseParser::compressSentence($datum, $lang);
                 $description .= " .. ". $datum;
                 if (self::OUTPUT_TO_FILE) {
                     if ($output_file_contents == "") {
@@ -184,4 +189,3 @@ class ScrapeSummarizer extends Summarizer
         return strip_tags($text);
     }
 }
-
diff --git a/src/locale/en_US/resources/Tokenizer.php b/src/locale/en_US/resources/Tokenizer.php
index a08ca70..2877d61 100755
--- a/src/locale/en_US/resources/Tokenizer.php
+++ b/src/locale/en_US/resources/Tokenizer.php
@@ -116,7 +116,8 @@ class Tokenizer
         return $pre_segment;
     }
     /**
-     * Computes similar words and scores from WordNet output based on word type.
+     * Computes similar words and scores from WordNet output based on word
+     * type.
      *
      * @param string $term term to find related thesaurus terms
      * @param string $word_type is the type of word such as "NN" (noun),
@@ -149,7 +150,8 @@ class Tokenizer
         $avg_scores = [];
         for ($i = 0; $i < $num_definitions; $i++) {
             //get sentence fragments examples of using that definition
-            preg_match_all('/\"(.*?)\"/', $definitions_for_sense[$i], $matches);
+            preg_match_all('/\"(.*?)\"/', $definitions_for_sense[$i],
+                $matches);
             // to separate out the words
             preg_match('/[\w+\s\,\.\']+\s\-+/', $definitions_for_sense[$i],
                 $match_word);
@@ -232,10 +234,10 @@ class Tokenizer
         'five','fix','followed','following','follows','for','former',
         'formerly','forth','found','four','from','further','furthermore',
         'g','gave','get','gets','getting','give','given','gives','giving','go',
-        'goes','gone','got','gotten','h','had','happens','hardly','has','hasnt',
-        'have','havent','having','he','hed','hence','her','here','hereafter',
-        'hereby','herein','heres','hereupon','hers','herself','hes','hi','hid',
-        'him','himself','his','hither','home','how','howbeit',
+        'goes','gone','got','gotten','h','had','happens','hardly','has',
+        'hasnt','have','havent','having','he','hed','hence','her','here',
+        'hereafter','hereby','herein','heres','hereupon','hers','herself',
+        'hes','hi','hid','him','himself','his','hither','home','how','howbeit',
         'however', 'http', 'https', 'hundred','i','id','ie','if','ill',
         'im','immediate','immediately',
         'importance','important','in','inc','indeed','index','information',
@@ -245,19 +247,19 @@ class Tokenizer
         'known','knows','l','largely','last','lately',
         'later','latter','latterly',
         'least','less','lest','let','lets','like','liked','likely','line',
-        'little','ll','look','looking','looks','ltd','m','made','mainly','make',
-        'makes','many','may','maybe','me','mean','means','meantime','meanwhile',
-        'merely','mg','might','million','miss','ml','more','moreover','most',
-        'mostly','mr','mrs','much','mug','must','my','myself','n','na','name',
-        'namely','nay','nd','near','nearly','necessarily','necessary','need',
-        'needs','neither','never','nevertheless','new','next',
-        'nine','ninety','no',
+        'little','ll','look','looking','looks','ltd','m','made','mainly',
+        'make','makes','many','may','maybe','me','mean','means','meantime',
+        'meanwhile','merely','mg','might','million','miss','ml','more',
+        'moreover','most','mostly','mr','mrs','much','mug','must','my',
+        'myself','n','na','name','namely','nay','nd','near','nearly',
+        'necessarily','necessary','need','needs','neither','never',
+        'nevertheless','new','next','nine','ninety','no',
         'nobody','non','none','nonetheless','noone',
         'nor','normally','nos','not',
         'noted','nothing','now','nowhere','o','obtain',
         'obtained','obviously','of',
-        'off','often','oh','ok','okay','old','omitted','on','once','one','ones',
-        'only','onto','or','ord','other','others',
+        'off','often','oh','ok','okay','old','omitted','on','once','one',
+        'ones','only','onto','or','ord','other','others',
         'otherwise','ought','our','ours',
         'ourselves','out','outside','over','overall','owing','own','p','page',
         'pages','part','particular','particularly',
@@ -297,21 +299,21 @@ class Tokenizer
         'thus','til','tip','to','together','too',
         'took','toward','towards','tried',
         'tries','truly','try','trying','ts','twice','two','u','un','under',
-        'unfortunately','unless','unlike','unlikely','until','unto','up','upon',
-        'ups','us','use','used','useful','usefully','usefulness','uses','using',
-        'usually','v','value','various','ve','very',
+        'unfortunately','unless','unlike','unlikely','until','unto','up',
+        'upon','ups','us','use','used','useful','usefully','usefulness',
+        'uses','using','usually','v','value','various','ve','very',
         'via','viz','vol','vols','vs',
         'w','want','wants','was','wasnt','way','we',
         'wed','welcome','well','went',
         'were','werent','weve','what','whatever',
         'whatll','whats','when','whence',
         'whenever','where','whereafter','whereas','whereby','wherein','wheres',
-        'whereupon','wherever','whether','which','while','whim','whither','who',
-        'whod','whoever','whole','wholl','whom','whomever','whos','whose','why',
-        'widely','willing','wish','with','within',
+        'whereupon','wherever','whether','which','while','whim','whither',
+        'who','whod','whoever','whole','wholl','whom','whomever','whos',
+        'whose','why','widely','willing','wish','with','within',
         'without','wont','words','world',
-        'would','wouldnt','www','x','y','yes','yet','you','youd','youll','your',
-        'youre','yours','yourself','yourselves','youve','z','zero'];
+        'would','wouldnt','www','x','y','yes','yet','you','youd','youll',
+        'your','youre','yours','yourself','yourselves','youve','z','zero'];
         $page = preg_replace('/\b('.implode('|',$stop_words).')\b/', '',
             mb_strtolower($page));
         return $page;
@@ -323,8 +325,8 @@ class Tokenizer
      * consisting of words from the Brown corpus together with a list of
      * part of speech tags that that word had in the Brown Corpus. These are
      * used to get an initial part of speech (in word was not present than
-     * we assume it is a noun). From this a fixed set of rules is used to modify
-     * the initial tag if necessary.
+     * we assume it is a noun). From this a fixed set of rules is used to
+     * modify the initial tag if necessary.
      *
      * @param string $phrase text to add parts speech tags to
      * @return string $tagged_phrase phrase where each term has ~part_of_speech
@@ -347,6 +349,7 @@ class Tokenizer
      * @param string $word the string to stem
      * @return string the stem of $words
      */
+
     public static function stem($word)
     {
         if (in_array($word, self::$no_stem_list)) {
@@ -490,8 +493,8 @@ class Tokenizer
         return true;
     }
     /**
-     * setto($s) sets (j+1),...k to the characters in the string $s, readjusting
-     * k.
+     * setto($s) sets (j+1),...k to the characters in the string $s,
+     * readjusting k.
      *
      * @param string $s string to modify the end of buffer with
      */
@@ -576,8 +579,8 @@ class Tokenizer
     }
     /**
      * step2() maps double suffices to single ones. so -ization ( = -ize plus
-     * -ation) maps to -ize etc.Note that the string before the suffix must give
-     * m() > 0.
+     * -ation) maps to -ize etc.Note that the string before the suffix must
+     * give m() > 0.
      */
     private static function step2()
     {
@@ -775,7 +778,7 @@ class Tokenizer
                 $current['tag'] = $tag_list[0];
             }
             // Converts verbs after 'the' to nouns
-            if ($previous['tag'] == 'DT' && in_array($current['tag'], $verbs)) {
+            if ($previous['tag'] == 'DT' && in_array($current['tag'], $verbs)){
                 $current['tag'] = 'NN';
             }
             // Convert noun to number if . appears
@@ -837,7 +840,7 @@ class Tokenizer
      * Takes an array of pairs (token, tag) that came from phrase
      * and builds a new phrase where terms look like token~tag.
      *
-     * @param array $tagged_tokens array of pairs as might come from tagTokenize
+     * @param array $tagged_tokens array pairs as might come from tagTokenize
      * @return $tagged_phrase a phrase with terms in the format token~tag
      */
     private static function taggedPartOfSpeechTokensToString($tagged_tokens)
@@ -858,5 +861,89 @@ class Tokenizer
             $tagged_phrase .= $t['token'] . "~" . $tag .  " ";
         }
         return $tagged_phrase;
+ }
+    /**
+     * Take in a sentence and try to compress it to a smaller version
+     * that "retains the most important information and remains grammatically
+     * correct" (Jing 2000).
+     *
+     * @param string $sentence_to_compress the sentence to compress
+     * @return the compressed sentence
+     */
+    public static function compressSentence($sentence_to_compress)
+    {
+        $result = $sentence_to_compress;
+        $result = self::numberTwo($result);
+        $result = self::numberThree($result);
+        $result = self::numberFour($result);
+        $result = self::numberSix($result);
+        return $result;
     }
-}
+    /**
+     * From Back to Basics: CLASSY 2006 page 3:
+     * 2. We remove many adverbs and all conjunctions, including phrases such
+     * as "As a matter of fact," and "At this point," that occur at the start
+     * of a sentence.
+     *
+     * @param string $sentence_to_compress the sentence to compress
+     * @return the compressed sentence
+     */
+    public static function numberTwo($sentence_to_compress)
+    {
+        $result = $sentence_to_compress;
+        $result = preg_replace("/^At this point,?/i", "", $result);
+        $result = preg_replace("/^As a matter of fact,?/i", "", $result);
+        //adverbs
+        $result = preg_replace("/^[a-zA-Z]*ly\s?/i", "", $result);
+        //conjunctions
+        $result = preg_replace("/(^and,?)|(^but,?)|(^for,?)|(^nor,?)|(^or,?)" .
+            "|(^so,?)|(^yet,?)/i", "", $result);
+        return $result;
+    }
+    /**
+     * From Back to Basics: CLASSY 2006 page 3:
+     * 3. We remove a small selections of words that occur in the middle of a
+     * sentence, such as ", however," and ", also," (not always requiring the
+     * commas).
+     *
+     * @param string $sentence_to_compress the sentence to compress
+     * @return the compressed sentence
+     */
+    public static function numberThree($sentence_to_compress)
+    {
+        $result = $sentence_to_compress;
+        $result = preg_replace("/,?\s?however,?/i", "", $result);
+        $result = preg_replace("/,?\s?also,?/i", "", $result);
+        return $result;
+    }
+    /**
+     * From Back to Basics: CLASSY 2006 page 3:
+     * 4. For DUC 2006, we added the removal of ages such as ", 51," or
+     * ", aged 24,".
+     *
+     * @param string $sentenceToCompress the sentence to compress
+     * @return the compressed sentence
+     */
+    public static function numberFour($sentence_to_compress)
+    {
+        $result = $sentence_to_compress;
+        $result = preg_replace("/,\s?\d{1,3},/i", "", $result);
+        $result = preg_replace("/,\s?aged\s?\d{1,3},/i", "", $result);
+        return $result;
+    }
+    /**
+     * From Back to Basics: CLASSY 2006 page 3:
+     * 6. We remove relative clause attributives (clauses beginning with
+     * "who(m)", "which", "when", and "where") wherever possible.
+     *
+     * @param string $sentence_to_compress the sentence to compress
+     * @return the compressed sentence
+     */
+    public static function numberSix($sentence_to_compress)
+    {
+        $result = $sentence_to_compress;
+        $result = preg_replace("/(,\s?whom?[^,]*,)|(,\s?which[^,]*,)|" .
+            "(,\s?when[^,]*,)|(,\s?where[^,]*,)/i", "", $result);
+        return $result;
+    }
+}
\ No newline at end of file
-- 
1.9.5.github.0

