_binary From 6fac2976c446274e6a720fb3567de8592fe1a4db Mon Sep 17 00:00:00 2001
From: diesal9 <charles.bocage@sjsu.edu>
Date: Tue, 5 May 2015 19:41:49 -0700
Subject: [PATCH] Reorganized the summarizers into their own folder and base
 class.  I added an additional weight method to the base class but it is not
 in use right now.  The basic summarizer in-line code has been pulled out and
 into the scrape_summarizer.php file.

---
 lib/centroid_summarizer.php                       | 339 ---------------
 lib/graph_based_summarizer.php                    | 474 ---------------------
 lib/processors/html_processor.php                 |  18 +-
 lib/processors/text_processor.php                 |   5 +-
 lib/summarizers/centroid_summarizer.php           | 355 ++++++++++++++++
 lib/summarizers/graph_based_summarizer.php        | 493 ++++++++++++++++++++++
 lib/summarizers/scrape_summarizer.php             | 176 ++++++++
 lib/summarizers/summarizer.php                    | 141 +++++++
 tests/summarizer_test.php                         | 202 +++++++++
 tests/test_files/summarizer/summarizer_input.txt  |  17 +
 tests/test_files/summarizer/summarizer_result.txt |  17 +
 11 files changed, 1414 insertions(+), 823 deletions(-)
 delete mode 100644 lib/centroid_summarizer.php
 delete mode 100644 lib/graph_based_summarizer.php
 create mode 100644 lib/summarizers/centroid_summarizer.php
 create mode 100644 lib/summarizers/graph_based_summarizer.php
 create mode 100644 lib/summarizers/scrape_summarizer.php
 create mode 100644 lib/summarizers/summarizer.php
 create mode 100644 tests/summarizer_test.php
 create mode 100644 tests/test_files/summarizer/summarizer_input.txt
 create mode 100644 tests/test_files/summarizer/summarizer_result.txt

diff --git a/lib/centroid_summarizer.php b/lib/centroid_summarizer.php
deleted file mode 100644
index 79d4a6c..0000000
--- a/lib/centroid_summarizer.php
+++ /dev/null
@@ -1,339 +0,0 @@
-<?php
-/**
- * SeekQuarry/Yioop --
- * Open Source Pure PHP Search Engine, Crawler, and Indexer
- *
- * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
- *
- * LICENSE:
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * END LICENSE
- *
- * @author Mangesh Dahale mangeshadahale@gmail.com
- * @license http://www.gnu.org/licenses/ GPL3
- * @link http://www.seekquarry.com/
- * @copyright 2009 - 2015
- * @filesource
- */
-if (!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
-/**
- * Reads in constants used as enums used for storing web sites
- */
-require_once BASE_DIR."/lib/crawl_constants.php";
-/**
- * Contains the max_description_length for the summary
- */
-require_once BASE_DIR."/lib/processors/page_processor.php";
-/**
- * Contains function getTokenizer to get the object of the language specified.
- */
-require_once BASE_DIR."/lib/phrase_parser.php";
-/**
- * Class which may be used by TextProcessors to get a summary for a text
- * document that may later be used for indexing. It does this by doing
- * centroid-based clustering. It also generates a word cloud for a document
- * @author Mangesh Dahale mangeshadahale@gmail.com
- * @package seek_quarry\library
- */
-class CentroidSummarizer
-{
-    /**
-     * Number of bytes in a sentence before it is considered long
-     * We use strlen rather than mbstrlen. This might actually be
-     * a better metric of the potential of a sentence to have info.
-     */
-    const LONG_SENTENCE_LEN = 50;
-    /**
-     * Number of sentences in a document before only consider longer
-     * sentences in centroid
-     */
-    const LONG_SENTENCE_THRESHOLD = 100;
-    /**
-     * Number of distinct terms to use in generating summary
-     */
-    const MAX_DISTINCT_TERMS = 1000;
-    /**
-     * Number of words in word cloud
-     */
-    const WORD_CLOUD_LEN = 5;
-    /**
-     * Number of nonzero centroid components
-     */
-    const CENTROID_COMPONENTS = 50;
-    /**
-     * Generates a centroid with which every sentence is ranked with cosine
-     * ranking method and also generates a word cloud.
-     * @param string $doc complete raw page to generate the summary from.
-     * @param string $lang language of the page to decide which stop words to
-     *     call proper tokenizer.php of the specified language.
-     *
-     * @return array array of summary and word cloud
-     */
-    static function getCentroidSummary($doc, $lang)
-    {
-        $doc = self::pageProcessing($doc);
-        /* Format the document to remove characters other than periods and
-           alphanumerics.
-        */
-        $formatted_doc = self::formatDoc($doc);
-        $stop_obj = PhraseParser::getTokenizer($lang);
-        /* Splitting into sentences */
-        $out_sentences = self::getSentences($doc);
-        $n = count($out_sentences);
-        $sentences = array();
-        if ($stop_obj && method_exists($stop_obj, "stopwordsRemover")) {
-            for ($i = 0; $i < $n; $i++ ) {
-                $sentences[$i] = $stop_obj->stopwordsRemover(
-                    self::formatDoc($out_sentences[$i]));
-             }
-        } else {
-            $sentences = $out_sentences;
-        }
-        /*  Splitting into terms */
-        $terms = array();
-        foreach ($sentences as $sentence) {
-            $terms = array_merge($terms,
-                PhraseParser::segmentSegment($sentence, $lang));
-        }
-        $terms = array_filter($terms);
-        $terms_counts = array_count_values($terms);
-        arsort($terms_counts);
-        $terms_counts = array_slice($terms_counts, 0, self::MAX_DISTINCT_TERMS);
-        $terms = array_unique(array_keys($terms_counts));
-        $t = count($terms);
-        if ($t == 0) {
-            return array("", "");
-        }
-        /* Initialize Nk array(Number of sentences the term occurs) */
-        $nk = array();
-        $nk = array_fill(0, $t, 0);
-        $nt = array();
-        /* Count TF for each word */
-        for ($i = 0; $i < $n; $i++) {
-            for ($j = 0; $j < $t; $j++) {
-                if (strpos($sentences[$i], $terms[$j]) !== false) {
-                    $nk[$j]++;
-                }
-            }
-        }
-        /* Calculate weights of each term for every sentence */
-        $w = array();
-        $idf = array();
-        $idf_temp = 0;
-        for ($k = 0; $k < $t; $k++) {
-            if ($nk[$k] == 0) {
-                $idf_temp = 0;
-                $tmp = 0;
-            } else {
-                $idf_temp = $n / $nk[$k];
-                $tmp = log($idf_temp);
-            }
-            $idf[$k] = $tmp;
-        }
-        /* Count TF for finding centroid */
-        $wc = array();
-        $max_nt = -1;
-        $b = "\b";
-        if (in_array($lang, array("zh-CN", "ja", "ko"))) {
-            $b = "";
-        }
-        for ($j = 0; $j < $t; $j++) {
-            $nt = @preg_match_all("/$b{$terms[$j]}$b/", $formatted_doc,
-                $matches); //$matches included for backwards compatibility
-            $wc[$j] = $nt * $idf[$j];
-            if (is_nan($wc[$j]) || is_infinite($wc[$j])) {
-                $wc[$j] = 0;
-            }
-        }
-        /* Calculate centroid */
-        arsort($wc);
-        $centroid = array_slice($wc, 0, self::CENTROID_COMPONENTS, true);
-        /* Initializing centroid weight array by 0 */
-        $wc = array_fill(0, $t, 0);
-        /* Word cloud */
-        $i = 0;
-        $word_cloud = array();
-        foreach ($centroid as $key => $value) {
-            $wc[$key] = $value;
-            if ($i < self::WORD_CLOUD_LEN) {
-                $word_cloud[$i] = $terms[$key];
-            }
-            $i++;
-        }
-        if (strlen($formatted_doc) < PageProcessor::$max_description_len
-            || $n == 1) {
-            //if input short only use above to get a word cloud
-            $formatted_doc = substr($formatted_doc, 0,
-                PageProcessor::$max_description_len);
-            return array($formatted_doc, $word_cloud);
-        }
-        ksort($wc);
-        /* Calculate similarity measure between centroid and each sentence */
-        $sim = array();
-        for ($i=0; $i < $n; $i++) {
-            $a = $b1 = $b2 = $c1 = $c2 = $d = 0;
-            for ($k = 0; $k < $t; $k++) {
-                    $wck = $wc[$k];
-                    $idfk = $idf[$k];
-                    $tmp = substr_count($sentences[$i], $terms[$k]);
-                    $wik = ($tmp > 0) ? $idfk * (1 + log($tmp)) : 0;
-                    $a += ($wik * $wck * $idfk);
-                    $b1 += ($wik * $wik);
-                    $c1 += ($wck * $wck);
-            }
-            $b2 = sqrt($b1);
-            $c2 = sqrt($c1);
-            $d = $b2 * $c2;
-            if ($d == 0) {
-                $sim[$i] = 0;
-            } else {
-                $sim[$i] = $a / $d;
-            }
-        }
-        arsort($sim);
-        /* Getting how many sentences should be there in summary */
-        $top = self::summarySentenceCount($out_sentences, $sim);
-        $sum_array = array();
-        $sum_array = array_slice($sim, 0, $top - 1, true);
-        ksort($sum_array);
-        /* Printing Summary */
-        $summary = '';
-        foreach ($sum_array as $key => $value) {
-            $summary .= "$out_sentences[$key]".". ";
-        }
-        /* Summary of text summarization */
-        return array($summary, $word_cloud);
-    }
-    /**
-     * Calculates how many sentences to put in the summary to match the
-     * MAX_DESCRIPTION_LEN.
-     *
-     * @param array $sentences sentences in doc in their original order
-     * @param array $sim associative array of sentence-number-in-doc =>
-     *      similarity score to centroid (sorted from highest to lowest score).
-     * @return int number of sentences
-     */
-    static function summarySentenceCount($sentences, $sim)
-    {
-        $top = null;
-        $count = 0;
-        foreach ($sim as $key => $value)
-        {
-            if ($count < PageProcessor::$max_description_len) {
-                $count += strlen($sentences[$key]);
-                $top++;
-            }
-        }
-        return $top;
-    }
-    /**
-     * Breaks any content into sentences by splitting it on spaces or carriage
-     *   returns
-     * @param string $content complete page.
-     * @return array array of sentences from that content.
-     */
-    static function getSentences($content)
-    {
-        $lines = preg_split(
-            '/(\.|\||\!|\?|！|？|。)\s+|(\n|\r)(\n|\r)+|\s{5}/',
-            $content, 0, PREG_SPLIT_NO_EMPTY);
-        $out = array();
-        $sentence = "";
-        $count = 0;
-        $theshold_factor = 1;
-        foreach ($lines as $line) {
-            $sentence .= " " . $line;
-            if (strlen($line) < 2) {
-                continue;
-            }
-            if ($count < self::LONG_SENTENCE_THRESHOLD ||
-                strlen($sentence) > $theshold_factor * self::LONG_SENTENCE_LEN){
-                $sentence = preg_replace("/\s+/ui", " ", $sentence);
-                $out[] = trim($sentence);
-                $count++;
-                $theshold_factor =
-                    pow(1.5, floor($count/self::LONG_SENTENCE_THRESHOLD));
-            }
-            $sentence = "";
-        }
-        if (trim($sentence) != "") {
-            $sentence = preg_replace("/\s+/ui", " ", $sentence);
-            $out[] = trim($sentence);
-        }
-        return $out;
-    }
-    /**
-     * Formats the sentences to remove all characters except words,
-     *   digits and spaces
-     * @param string $sent complete page.
-     * @return string formatted sentences.
-     */
-    static function formatSentence($sent)
-    {
-        $sent = trim(preg_replace('/[^\p{L}\p{N}\s]+/u',
-            ' ', mb_strtolower($sent)));
-        return $sent;
-    }
-    /**
-     * Formats the document to remove carriage returns, hyphens and digits
-     * as we will not be using digits in word cloud.
-     * The formatted document generated by this function is only used to
-     * compute centroid.
-     * @param string $content formatted page.
-     * @return string formatted document.
-     */
-    static function formatDoc($content)
-    {
-        $substitute = array('/[\n\r\-]+/', '/[^\p{L}\s\.]+/u', '/[\.]+/');
-        $content = preg_replace($substitute, ' ', mb_strtolower($content));
-        return $content;
-    }
-    /**
-     * This function does an additional processing on the page
-     * such as removing all the tags from the page
-     * @param string $page complete page.
-     * @return string processed page.
-     */
-    static function pageProcessing($page)
-    {
-        $substitutions = array('@<script[^>]*?>.*?</script>@si',
-            '/\&nbsp\;|\&rdquo\;|\&ldquo\;|\&mdash\;/si',
-            '@<style[^>]*?>.*?</style>@si', '/[\^\(\)]/',
-            '/\[(.*?)\]/', '/\t\n/'
-        );
-        $page = preg_replace($substitutions, ' ', $page);
-        $page = preg_replace('/\s{2,}/', ' ', $page);
-        $new_page = preg_replace("/\<br\s*(\/)?\s*\>/", "\n", $page);
-        $changed = false;
-        if ($new_page != $page) {
-            $changed = true;
-            $page = $new_page;
-        }
-        $page = preg_replace("/\<\/(h1|h2|h3|h4|h5|h6|table|tr|td|div|".
-            "p|address|section)\s*\>/", "\n\n", $page);
-        $page = preg_replace("/\<a/", " <a", $page);
-        $page = preg_replace("/\&\#\d{3}(\d?)\;|\&\w+\;/", " ", $page);
-        $page = preg_replace("/\</", " <", $page);
-        $page = strip_tags($page);
-
-        if ($changed) {
-            $page = preg_replace("/(\r?\n[\t| ]*){2}/", "\n", $page);
-        }
-        $page = preg_replace("/(\r?\n[\t| ]*)/", "\n", $page);
-        $page = preg_replace("/\n\n\n+/", "\n\n", $page);
-        return $page;
-    }
-}
diff --git a/lib/graph_based_summarizer.php b/lib/graph_based_summarizer.php
deleted file mode 100644
index c09d340..0000000
--- a/lib/graph_based_summarizer.php
+++ /dev/null
@@ -1,474 +0,0 @@
-<?php
-/**
- * SeekQuarry/Yioop --
- * Open Source Pure PHP Search Engine, Crawler, and Indexer
- *
- * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
- *
- * LICENSE:
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * END LICENSE
- *
- * @author Charles Bocage charles.bocage@sjsu.edu
- * @license http://www.gnu.org/licenses/ GPL3
- * @link http://www.seekquarry.com/
- * @copyright 2009 - 2015
- * @filesource
- */
-
-if (!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
-/**
- * Reads in constants used as enums used for storing web sites
- */
-require_once BASE_DIR."/lib/crawl_constants.php";
-/**
- * Contains the max_description_length for the summary
- */
-require_once BASE_DIR."/lib/processors/page_processor.php";
-/**
- * Contains function getTokenizer to get the object of the language specified.
- */
-require_once BASE_DIR."/lib/phrase_parser.php";
-
-
-/**
- * Class which may be used by TextProcessors to get a summary for a text
- * document that may later be used for indexing.
- *
- * add more comments later
- *
- * @author Charles Bocage charles.bocage@sjsu.edu
- * @package seek_quarry\library
- */
-class GraphBasedSummarizer
-{
-    /**
-     * Number of distinct terms to use in generating summary
-     */
-    const MAX_DISTINCT_TERMS = 1000;
-    /**
-     * This is a graph based summarizer
-     *
-     * @param string $doc complete raw page to generate the summary from.
-     * @param string $lang language of the page to decide which stop words to
-     *     call proper tokenizer.php of the specified language.
-     *
-     * @return array array of summary and word cloud
-     */
-    static function getGraphBasedSummary($doc, $lang)
-    {
-        $doc = self::pageProcessing($doc);
-        $formatted_doc = self::formatDoc($doc);
-        //not filtering non-ascii characters
-        $sentences = self::getSentences($doc . " ", true);
-        $sentences = self::removeStopWords($sentences, $lang);
-        $sentences = self::removePunctuation($sentences);
-        $sentences = PhraseParser::stemTermsK($sentences, $lang, true);
-        $terms = self::getTerms($sentences, $lang);
-        $term_frequencies = self::getTermFrequencies($terms, $sentences);
-        $term_frequencies_normalized =
-            self::normalizeTermFrequencies($term_frequencies);
-        $adjacency = self::computeAdjacency($term_frequencies_normalized,
-            $sentences, $lang);
-        $p = self::getSentenceRanks($adjacency);
-        $sentences_with_punctuation = self::getSentences($doc . " ", true);
-        $summary = self::getSummary($sentences_with_punctuation, $p);
-        return $summary;
-    }
-    /*
-     * Get the summary from the sentences
-     * @param array $sentences the sentences in the doc
-     * @param array $p the sentence probabilities
-     */
-    static function getSummary($sentences, $p)
-    {
-        $result = "";
-        $result_length = 0;
-        $n = count($p);
-        for ($i = 0; $i < $n; $i++ ) {
-            $index = self::findLargestIndex($p);
-            $p[$index] = -1;
-            $sentence = $sentences[$index];
-            if ($result_length + strlen($sentence) >
-                PageProcessor::$max_description_len) {
-                break;
-            } else {
-                $result_length += strlen($sentence);
-                if ($i == 0) {
-                    $result = $sentence;
-                } else {
-                    $result = $result . " " . $sentence;
-                }
-            }
-        }
-        return $result;
-    }
-    /*
-     * find the largest value in the array and return it
-     * @param array $v the array to search for the largest value
-     * @return double the largest value found in the array
-     */
-    static function findLargestIndex($v)
-    {
-        $result = 0;
-        $n = count($v);
-        $last_value = -1;
-        for ($i = 0; $i < $n; $i++ ) {
-            if ($v[$i] > $last_value) {
-                $last_value = $v[$i];
-                $result = $i;
-            }
-        }
-        return $result;
-    }
-    /*
-     * compute the sentence ranks using a version of the famous
-     * page ranking algorithm developed by the founder of Google.
-     * @param array $adjacency the adjacency matrix generated for the
-     *      sentences
-     * @return array the sentence ranks
-     */
-    static function getSentenceRanks($adjacency)
-    {
-        $n = count($adjacency);
-        $old_p = array();
-        $p = array();
-        for ($i = 0; $i < $n; $i++ ) {
-            $p[$i] = 1 / $n;
-        }
-        for ($i = 0; $i < 10; $i++ ) {
-            $p = self::multiplyMatrixVector($adjacency, $p);
-        }
-        return $p;
-    }
-    /*
-     * Compute the difference of squares
-     * @param array $v the  minuend vector
-     * @param array $m the subtrahend vector
-     * @result double the difference of the squares of vectors
-     */
-    static function squareDiff($v, $w)
-    {
-        $result = 0;
-        $n = count($v);
-        for ($i = 0; $i < $n; $i++ ) {
-            $subtraction = $v[$i] - $w[$i];
-            $result += $subtraction * $subtraction;
-        }
-        return $result;
-    }
-    /*
-     * Perform matrix multiplication on a matrix and a vector
-     * @param array $mat the matrix to multiply the probabilities to
-     * @param array $vec the probability vector
-     * @return array the new vector after it has been multiplied
-     */
-    static function multiplyMatrixVector($mat, $vec)
-    {
-        $result = array();
-        $n = count($vec);
-        for ($i = 0; $i < $n; $i++ ) {
-            $result[$i] = 0;
-            for ($j = 0; $j < $n; $j++ ) {
-                $result[$i] += $mat[$i][$j] * $vec[$j];
-            }
-        }
-        return $result;
-    }
-    /*
-     * Compute the adjacency matrix based on its distortion measure
-     * @param array $term_frequencies_normalized the array of term frequencies
-     * @param array $sentences the sentences in the doc
-     * @param string $lang locale tag for stemming
-     * @return array the array of sentence adjacency
-     */
-    static function computeAdjacency($term_frequencies_normalized, $sentences,
-        $lang)
-    {
-        $result = array(array());
-        $n = count($sentences);
-        for ($i = 0; $i < $n; $i++ ) {
-            $result[$i][$i] = 0;
-            for ($j = $i + 1; $j < $n; $j++ ) {
-                $result[$i][$j] = $result[$j][$i] =
-                    self::findDistortion($sentences[$i], $sentences[$j],
-                    $term_frequencies_normalized, $lang);
-            }
-        }
-        return $result;
-    }
-    /*
-     * Remove punctuation
-     * @param array $sentences the sentences in the doc
-     * @return array the array of sentences with the punctuation removed
-     */
-     static function removePunctuation($sentences)
-     {
-        $n = count($sentences);
-        for ($i = 0; $i < $n; $i++ ) {
-            $sentences[$i] = trim(preg_replace('/[^a-z0-9]+/iu', ' ',
-                $sentences[$i]));
-        }
-        return $sentences;
-     }
-    /*
-     * Remove the stop words from the array of sentences
-     * @param array $sentences the sentences in the doc
-     * @param string $lang locale tag for stemming
-     * @return array the array of sentences with the stop words removed
-     */
-    static function removeStopWords($sentences, $lang)
-    {
-        $n = count($sentences);
-        $stop_obj = PhraseParser::getTokenizer($lang);
-        if ($stop_obj && method_exists($stop_obj, "stopwordsRemover")) {
-            for ($i = 0; $i < $n; $i++ ) {
-                $sentences[$i] = $stop_obj->stopwordsRemover(
-                    self::formatDoc($sentences[$i]));
-             }
-        }
-        return $sentences;
-    }
-    /*
-     * Calculate the term frequencies.
-     * @param array $terms the list of all terms in the doc
-     * @param array $sentences the sentences in the doc
-     * @return array a two dimensional array where the word is the key and
-     *      the frequency is the value
-     */
-    static function getTermFrequencies($terms, $sentences)
-    {
-        $t = count($terms);
-        $n = count($sentences);
-        $nk = array();
-        $nk = array_fill(0, $t, 0);
-        $nt = array();
-        for ($i = 0; $i < $n; $i++) {
-            for ($j = 0; $j < $t; $j++) {
-                if (strpos($sentences[$i], $terms[$j]) !== false) {
-                    $nk[$j]++;
-                }
-            }
-        }
-        for ($i = 0; $i <  count($nk); $i++ ) {
-            $term_frequencies[$terms[$i]] = $nk[$i];
-        }
-        return $term_frequencies;
-    }
-    /*
-     * Get the terms from an array of sentences
-     * @param array $sentences the sentences in the doc
-     * @param string $lang locale tag for stemming
-     * $return array an array of terms in the array of sentences
-     */
-    static function getTerms($sentences, $lang)
-    {
-        $terms = array();
-        foreach ($sentences as $sentence) {
-            $terms = array_merge($terms,
-                PhraseParser::segmentSegment($sentence, $lang));
-        }
-        $terms = array_filter($terms);
-        $terms_counts = array_count_values($terms);
-        arsort($terms_counts);
-        $terms_counts = array_slice($terms_counts, 0, self::MAX_DISTINCT_TERMS);
-        $terms = array_unique(array_keys($terms_counts));
-        $t = count($terms);
-        if ($t == 0) {
-            return array("", "");
-        }
-        return $terms;
-    }
-    /**
-     * Breaks any content into sentences by splitting it on spaces or carriage
-     *   returns
-     * @param string $content complete page.
-     * @param boolean $keep_punctuation whether to keep the punctuation or not.
-     * @return array array of sentences from that content.
-     */
-    static function getSentences($content, $keep_punctuation)
-    {
-        if ($keep_punctuation) {
-            $sentences =
-                preg_split('/(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|\!)/u',
-                $content, 0, PREG_SPLIT_NO_EMPTY);
-            $n = count($sentences);
-            for ($i = 0; $i < $n; $i++ ) {
-                $sentences[$i] = trim($sentences[$i]);
-            }
-            return $sentences;
-        } else {
-            return preg_split(
-                '/(\.|\||\!|\?|！|？|。)\s+|(\n|\r)(\n|\r)+|\s{5}/u',
-                $content, 0, PREG_SPLIT_NO_EMPTY);
-        }
-    }
-    /*
-     * Normalize the term frequencies based on the sum of the squares.
-     * @param array $term_frequencies the array with the terms as the key
-     *      and its frequency as the value
-     * @return array array of term frequencies normalized
-     */
-    static function normalizeTermFrequencies($term_frequencies)
-    {
-        $sum_of_squares = 0;
-        $result_sum = 0;
-        foreach ($term_frequencies as $k => $v) {
-            $sum_of_squares += ($v * $v);
-        }
-        $square_root = sqrt($sum_of_squares);
-        foreach ($term_frequencies as $k => $v) {
-            $result[$k] = ($v / $square_root);
-        }
-        foreach ($result as $k => $v) {
-            $result_sum += $v;
-        }
-        return $result;
-    }
-    /**
-     *   Calcluate the distortion measure.
-     *   1. Check each word in sentence1 to see if it exists in sentence2.
-     *   If the word X of sentence1 does not exist in sentence2,
-     *   square the score of word X and add to the sum
-     *   and increase the number of not-common words by one.
-     *   2. In case the word X is common between sentence1 and
-     *   sentence2, calculate its frequency in sentence2 and subtract
-     *   it from the score of word X, then square and add to
-     *   sum.
-     *   3. Then check the sentence2 to find its not-common words
-     *   with sentence1, in case the word Y is not in sentence1,
-     *   square the score of word Y and add to sum and increase
-     *   the number of not-common words by one.
-     *   4. At the end, calcualte the distortion between sentence1 and
-     *   sentence2 by dividing sum by the number of not-common
-     *   words.
-     * @param string $first_sentence the first sentence to compare
-     * @param string $second_sentence the second sentence to compare
-     * @param string $term_frequencies the term frequency of the sentences
-     * @param string $lang locale tag for stemming
-     */
-    static function findDistortion($first_sentence, $second_sentence,
-        $term_frequencies, $lang)
-    {
-        $result = 0;
-        $first_sentence_split = preg_split('/ +/u', $first_sentence);
-        $second_sentence_split = preg_split('/ +/u', $second_sentence);
-        $sum = 0;
-        $non_common_words = 0;
-        $n = count($first_sentence_split);
-        for ($i = 0; $i < $n; $i++ ) {
-            $word_to_search_for = trim($first_sentence_split[$i]);
-            if ($word_to_search_for != "") {
-                preg_match_all("/ " . $word_to_search_for . " /",
-                    $second_sentence, $matches);
-                if (count($matches[0]) == 0) {
-                    $sum += ($term_frequencies[$word_to_search_for] *
-                        $term_frequencies[$word_to_search_for]);
-                    $non_common_words++;
-                } else {
-                    $terms = self::getTerms(array($second_sentence), $lang);
-                    $temp_term_frequencies = self::getTermFrequencies(
-                        $terms, array($second_sentence));
-                    $temp_term_frequencies_normalized =
-                        self::normalizeTermFrequencies(
-                        $temp_term_frequencies);
-                    $new_term_frequency =
-                        $term_frequencies[$word_to_search_for] -
-                        $temp_term_frequencies_normalized[$word_to_search_for];
-                    $sum += ($new_term_frequency * $new_term_frequency);
-                }
-            }
-        }
-        $n = count($second_sentence_split);
-        for ($i = 0; $i < $n; $i++ ) {
-            $word_to_search_for = trim($second_sentence_split[$i]);
-            if ($word_to_search_for != "") {
-                preg_match_all("/ " . trim($word_to_search_for) . " /",
-                    $first_sentence, $matches);
-                if (count($matches[0]) == 0) {
-                    $sum += ($term_frequencies[$word_to_search_for] *
-                        $term_frequencies[$word_to_search_for]);
-                    $non_common_words++;
-                }
-            }
-        }
-        if ($non_common_words != 0) {
-            $result = $sum / $non_common_words;
-        }
-        return $result;
-    }
-    /**
-     * Formats the sentences to remove all characters except words,
-     *   digits and spaces
-     * @param string $sent complete page.
-     * @return string formatted sentences.
-     */
-    static function formatSentence($sent)
-    {
-        $sent = trim(preg_replace('/[^\p{L}\p{N}\s]+/u',
-            ' ', mb_strtolower($sent)));
-        return $sent;
-    }
-    /**
-     * Formats the document to remove carriage returns, hyphens and digits
-     * as we will not be using digits in word cloud.
-     * The formatted document generated by this function is only used to
-     * compute centroid.
-     * @param string $content formatted page.
-     * @return string formatted document.
-     */
-    static function formatDoc($content)
-    {
-        $substitute = array('/[\n\r\-]+/', '/[^\p{L}\s\.]+/u', '/[\.]+/u');
-        $content = preg_replace($substitute, ' ', mb_strtolower($content));
-        return $content;
-    }
-    /**
-     * This function does an additional processing on the page
-     * such as removing all the tags from the page
-     * @param string $page complete page.
-     * @return string processed page.
-     */
-    static function pageProcessing($page)
-    {
-        $substitutions = array('@<script[^>]*?>.*?</script>@si',
-            '/\&nbsp\;|\&rdquo\;|\&ldquo\;|\&mdash\;/si',
-            '@<style[^>]*?>.*?</style>@si', '/[\^\(\)]/',
-            '/\[(.*?)\]/', '/\t\n/'
-        );
-        $page = preg_replace($substitutions, ' ', $page);
-        $page = preg_replace('/\s{2,}/u', ' ', $page);
-        $new_page = preg_replace("/\<br\s*(\/)?\s*\>/u", "\n", $page);
-        $changed = false;
-        if ($new_page != $page) {
-            $changed = true;
-            $page = $new_page;
-        }
-        $page = preg_replace("/\<\/(h1|h2|h3|h4|h5|h6|table|tr|td|div|".
-            "p|address|section)\s*\>/u", "\n\n", $page);
-        $page = preg_replace("/\<a/u", " <a", $page);
-        $page = preg_replace("/\&\#\d{3}(\d?)\;|\&\w+\;/u", " ", $page);
-        $page = preg_replace("/\</u", " <", $page);
-        $page = strip_tags($page);
-
-        if ($changed) {
-            $page = preg_replace("/(\r?\n[\t| ]*){2}/u", "\n", $page);
-        }
-        $page = preg_replace("/(\r?\n[\t| ]*)/u", "\n", $page);
-        $page = preg_replace("/\n\n\n+/u", "\n\n", $page);
-        return $page;
-    }
-}
diff --git a/lib/processors/html_processor.php b/lib/processors/html_processor.php
index b69d088..21eeba8 100755
--- a/lib/processors/html_processor.php
+++ b/lib/processors/html_processor.php
@@ -50,11 +50,15 @@ require_once BASE_DIR."/lib/url_parser.php";
 /**
 * Get the centroid summary
 */
-require_once BASE_DIR."/lib/centroid_summarizer.php";
+require_once BASE_DIR."/lib/summarizers/centroid_summarizer.php";
 /**
 * Get the graph based summary
 */
-require_once BASE_DIR."/lib/graph_based_summarizer.php";
+require_once BASE_DIR."/lib/summarizers/graph_based_summarizer.php";
+/**
+* Get the graph based summary
+*/
+require_once BASE_DIR."/lib/summarizers/scrape_summarizer.php";
 /**
  * For guessing language from charset
  */
@@ -117,8 +121,8 @@ class HtmlProcessor extends TextProcessor
                         $summary[self::LANG]);
                     crawlLog("..Using Graph Based Summarizer");
                 } else {
-                    $summary[self::DESCRIPTION] = self::description($dom,
-                        $dom_page);
+                    $summary[self::DESCRIPTION] =
+                        ScrapeSummarizer::getBasicSummary($dom, $dom_page);
                     crawlLog("..Using Basic Summarizer");
                 }
                 $crude = false;
@@ -150,7 +154,7 @@ class HtmlProcessor extends TextProcessor
                         $summary[self::LINKS][$location] = "location:".$url;
                         $summary[self::LOCATION] = true;
                         if (!$summary[self::DESCRIPTION]) {
-                            $summary[self::DESCRIPTION] .=$url." => ".$location;
+                            $summary[self::DESCRIPTION].=$url." => ".$location;
                         }
                         if (!$summary[self::TITLE]) {
                             $summary[self::TITLE] = $url;
@@ -394,7 +398,7 @@ class HtmlProcessor extends TextProcessor
                 if ($len > self::$max_description_len) { break 2;}
                 if (in_array($part, array("/html//p[1]", "/html//div[1]",
                     "/html//div[2]", "/html//p[2]", "/html//p[3]",
-                    "/html//div[3]", "/html//div[4]", "/html//p[4]"))) { break;}
+                    "/html//div[3]", "/html//div[4]", "/html//p[4]"))){ break;}
             }
         }
         krsort($para_data);
@@ -578,4 +582,4 @@ class HtmlProcessor extends TextProcessor
         $text = preg_replace('/\</', ' <', $text);
         return strip_tags($text);
     }
-}
+}
\ No newline at end of file
diff --git a/lib/processors/text_processor.php b/lib/processors/text_processor.php
index c698fcc..73c634b 100755
--- a/lib/processors/text_processor.php
+++ b/lib/processors/text_processor.php
@@ -50,11 +50,11 @@ require_once BASE_DIR."/lib/processors/page_processor.php";
 /**
  * Get the centroid summary
  */
-require_once BASE_DIR."/lib/centroid_summarizer.php";
+require_once BASE_DIR."/lib/summarizers/centroid_summarizer.php";
 /**
  * Get the graph based summary
  */
-require_once BASE_DIR."/lib/graph_based_summarizer.php";
+require_once BASE_DIR."/lib/summarizers/graph_based_summarizer.php";
 /**
  * So can extract parts of the URL if need to guess lang
  */
@@ -229,4 +229,3 @@ class TextProcessor extends PageProcessor
         }
     }
 }
-
diff --git a/lib/summarizers/centroid_summarizer.php b/lib/summarizers/centroid_summarizer.php
new file mode 100644
index 0000000..cf5b84a
--- /dev/null
+++ b/lib/summarizers/centroid_summarizer.php
@@ -0,0 +1,355 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Mangesh Dahale mangeshadahale@gmail.com
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+if (!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+/**
+ * Reads in constants used as enums used for storing web sites
+ */
+require_once BASE_DIR."/lib/crawl_constants.php";
+/**
+ * Contains the max_description_length for the summary
+ */
+require_once BASE_DIR."/lib/processors/page_processor.php";
+/**
+ * Contains function getTokenizer to get the object of the language specified.
+ */
+require_once BASE_DIR."/lib/phrase_parser.php";
+/**
+ * Contains the base methods for the summarizer.
+ */
+require_once BASE_DIR."/lib/summarizers/summarizer.php";
+/**
+ * Class which may be used by TextProcessors to get a summary for a text
+ * document that may later be used for indexing. It does this by doing
+ * centroid-based clustering. It also generates a word cloud for a document
+ * @author Mangesh Dahale mangeshadahale@gmail.com
+ * @package seek_quarry\library
+ */
+class CentroidSummarizer extends Summarizer
+{
+    /**
+     * Number of bytes in a sentence before it is considered long
+     * We use strlen rather than mbstrlen. This might actually be
+     * a better metric of the potential of a sentence to have info.
+     */
+    const LONG_SENTENCE_LEN = 50;
+    /**
+     * Number of sentences in a document before only consider longer
+     * sentences in centroid
+     */
+    const LONG_SENTENCE_THRESHOLD = 100;
+    /**
+     * Number of distinct terms to use in generating summary
+     */
+    const MAX_DISTINCT_TERMS = 1000;
+    /**
+     * Number of words in word cloud
+     */
+    const WORD_CLOUD_LEN = 5;
+    /**
+     * Number of nonzero centroid components
+     */
+    const CENTROID_COMPONENTS = 50;
+    /**
+     * Generates a centroid with which every sentence is ranked with cosine
+     * ranking method and also generates a word cloud.
+     * @param string $doc complete raw page to generate the summary from.
+     * @param string $lang language of the page to decide which stop words to
+     *     call proper tokenizer.php of the specified language.
+     *
+     * @return array array of summary and word cloud
+     */
+    static function getCentroidSummary($doc, $lang)
+    {
+        $unmodifiedDoc = strip_tags($doc,
+            '<a><h1><h2><h3><h4><h5><h6><b><em><i><u><dl><ol><ul><title>');
+        $doc = self::pageProcessing($doc);
+        /* Format the document to remove characters other than periods and
+           alphanumerics.
+        */
+        $formatted_doc = self::formatDoc($doc);
+        $stop_obj = PhraseParser::getTokenizer($lang);
+        /* Splitting into sentences */
+        $out_sentences = self::getSentences($doc);
+        $n = count($out_sentences);
+        $sentences = array();
+        if ($stop_obj && method_exists($stop_obj, "stopwordsRemover")) {
+            for ($i = 0; $i < $n; $i++ ) {
+                $sentences[$i] = $stop_obj->stopwordsRemover(
+                    self::formatDoc($out_sentences[$i]));
+             }
+        } else {
+            $sentences = $out_sentences;
+        }
+        /*  Splitting into terms */
+        $terms = array();
+        foreach ($sentences as $sentence) {
+            $terms = array_merge($terms,
+                PhraseParser::segmentSegment($sentence, $lang));
+        }
+        $terms = array_filter($terms);
+        $terms_counts = array_count_values($terms);
+        arsort($terms_counts);
+        $terms_counts = array_slice($terms_counts, 0,
+            self::MAX_DISTINCT_TERMS);
+        $terms = array_unique(array_keys($terms_counts));
+        $t = count($terms);
+        if ($t == 0) {
+            return array("", "");
+        }
+        /* Initialize Nk array(Number of sentences the term occurs) */
+        $nk = array();
+        $nk = array_fill(0, $t, 0);
+        $nt = array();
+        /* Count TF for each word */
+        for ($i = 0; $i < $n; $i++) {
+            for ($j = 0; $j < $t; $j++) {
+                if (strpos($sentences[$i], $terms[$j]) !== false) {
+                    $nk[$j]++;
+                }
+            }
+        }
+        /* Calculate weights of each term for every sentence */
+        $w = array();
+        $idf = array();
+        $idf_temp = 0;
+        for ($k = 0; $k < $t; $k++) {
+            if ($nk[$k] == 0) {
+                $idf_temp = 0;
+                $tmp = 0;
+            } else {
+                $idf_temp = $n / $nk[$k];
+                $tmp = log($idf_temp);
+            }
+            $idf[$k] = $tmp;
+        }
+        /* Count TF for finding centroid */
+        $wc = array();
+        $max_nt = -1;
+        $b = "\b";
+        if (in_array($lang, array("zh-CN", "ja", "ko"))) {
+            $b = "";
+        }
+        for ($j = 0; $j < $t; $j++) {
+            $nt = @preg_match_all("/$b{$terms[$j]}$b/", $formatted_doc,
+                $matches); //$matches included for backwards compatibility
+            $wc[$j] = $nt * $idf[$j];
+            if (is_nan($wc[$j]) || is_infinite($wc[$j])) {
+                $wc[$j] = 0;
+            }
+        }
+        /* Calculate centroid */
+        arsort($wc);
+        $centroid = array_slice($wc, 0, self::CENTROID_COMPONENTS, true);
+        /* Initializing centroid weight array by 0 */
+        $wc = array_fill(0, $t, 0);
+        /* Word cloud */
+        $i = 0;
+        $word_cloud = array();
+        foreach ($centroid as $key => $value) {
+            $wc[$key] = $value;
+            if ($i < self::WORD_CLOUD_LEN) {
+                $word_cloud[$i] = $terms[$key];
+            }
+            $i++;
+        }
+        if (strlen($formatted_doc) < PageProcessor::$max_description_len
+            || $n == 1) {
+            //if input short only use above to get a word cloud
+            $formatted_doc = substr($formatted_doc, 0,
+                PageProcessor::$max_description_len);
+            return array($formatted_doc, $word_cloud);
+        }
+        ksort($wc);
+        /* Calculate similarity measure between centroid and each sentence */
+        $sim = array();
+        for ($i=0; $i < $n; $i++) {
+            $a = $b1 = $b2 = $c1 = $c2 = $d = 0;
+            for ($k = 0; $k < $t; $k++) {
+                    $wck = $wc[$k];
+                    $idfk = $idf[$k];
+                    $tmp = substr_count($sentences[$i], $terms[$k]);
+                    $wik = ($tmp > 0) ? $idfk * (1 + log($tmp)) : 0;
+                    $a += ($wik * $wck * $idfk);
+                    $b1 += ($wik * $wik);
+                    $c1 += ($wck * $wck);
+            }
+            $b2 = sqrt($b1);
+            $c2 = sqrt($c1);
+            $d = $b2 * $c2;
+            if ($d == 0) {
+                $sim[$i] = 0;
+            } else {
+                $sim[$i] = $a / $d;
+            }
+        }
+        arsort($sim);
+        /* Getting how many sentences should be there in summary */
+        $top = self::summarySentenceCount($out_sentences, $sim);
+        $sum_array = array();
+        $sum_array = array_slice($sim, 0, $top - 1, true);
+        ksort($sum_array);
+        /* Printing Summary */
+        $summary = '';
+        $outputToFile = false;
+        $outputFileContents = "";
+        foreach($sum_array as $key => $value) {
+            $summary .= "$out_sentences[$key]".". ";
+            if ($outputToFile) {
+                $outputFileContents .= "$out_sentences[$key]".".\r\n";
+            }
+        }
+        if ($outputToFile) {
+            file_put_contents("C:/Temp/out.txt", $outputFileContents);
+        }
+        /* Summary of text summarization */
+        return array($summary, $word_cloud);
+    }
+    /**
+     * Calculates how many sentences to put in the summary to match the
+     * MAX_DESCRIPTION_LEN.
+     *
+     * @param array $sentences sentences in doc in their original order
+     * @param array $sim associative array of sentence-number-in-doc =>
+     *      similarity score to centroid (sorted from highest to lowest score).
+     * @return int number of sentences
+     */
+    static function summarySentenceCount($sentences, $sim)
+    {
+        $top = null;
+        $count = 0;
+        foreach ($sim as $key => $value)
+        {
+            if ($count < PageProcessor::$max_description_len) {
+                $count += strlen($sentences[$key]);
+                $top++;
+            }
+        }
+        return $top;
+    }
+    /**
+     * Breaks any content into sentences by splitting it on spaces or carriage
+     *   returns
+     * @param string $content complete page.
+     * @return array array of sentences from that content.
+     */
+    static function getSentences($content)
+    {
+        $lines = preg_split(
+            '/(\.|\||\!|\?|！|？|。)\s+|(\n|\r)(\n|\r)+|\s{5}/',
+            $content, 0, PREG_SPLIT_NO_EMPTY);
+        $out = array();
+        $sentence = "";
+        $count = 0;
+        $theshold_factor = 1;
+        foreach ($lines as $line) {
+            $sentence .= " " . $line;
+            if (strlen($line) < 2) {
+                continue;
+            }
+            if ($count < self::LONG_SENTENCE_THRESHOLD ||
+                strlen($sentence) > $theshold_factor *
+                    self::LONG_SENTENCE_LEN){
+                $sentence = preg_replace("/\s+/ui", " ", $sentence);
+                $out[] = trim($sentence);
+                $count++;
+                $theshold_factor =
+                    pow(1.5, floor($count/self::LONG_SENTENCE_THRESHOLD));
+            }
+            $sentence = "";
+        }
+        if (trim($sentence) != "") {
+            $sentence = preg_replace("/\s+/ui", " ", $sentence);
+            $out[] = trim($sentence);
+        }
+        return $out;
+    }
+    /**
+     * Formats the sentences to remove all characters except words,
+     *   digits and spaces
+     * @param string $sent complete page.
+     * @return string formatted sentences.
+     */
+    static function formatSentence($sent)
+    {
+        $sent = trim(preg_replace('/[^\p{L}\p{N}\s]+/u',
+            ' ', mb_strtolower($sent)));
+        return $sent;
+    }
+    /**
+     * Formats the document to remove carriage returns, hyphens and digits
+     * as we will not be using digits in word cloud.
+     * The formatted document generated by this function is only used to
+     * compute centroid.
+     * @param string $content formatted page.
+     * @return string formatted document.
+     */
+    static function formatDoc($content)
+    {
+        $substitute = array('/[\n\r\-]+/', '/[^\p{L}\s\.]+/u', '/[\.]+/');
+        $content = preg_replace($substitute, ' ', mb_strtolower($content));
+        return $content;
+    }
+    /**
+     * This function does an additional processing on the page
+     * such as removing all the tags from the page
+     * @param string $page complete page.
+     * @return string processed page.
+     */
+    static function pageProcessing($page)
+    {
+        $substitutions = array('@<script[^>]*?>.*?</script>@si',
+            '/\&nbsp\;|\&rdquo\;|\&ldquo\;|\&mdash\;/si',
+            '@<style[^>]*?>.*?</style>@si', '/[\^\(\)]/',
+            '/\[(.*?)\]/', '/\t\n/'
+        );
+        $page = preg_replace($substitutions, ' ', $page);
+        $page = preg_replace('/\s{2,}/', ' ', $page);
+        $new_page = preg_replace("/\<br\s*(\/)?\s*\>/", "\n", $page);
+        $changed = false;
+        if ($new_page != $page) {
+            $changed = true;
+            $page = $new_page;
+        }
+        $page = preg_replace("/\<\/(h1|h2|h3|h4|h5|h6|table|tr|td|div|".
+            "p|address|section)\s*\>/", "\n\n", $page);
+        $page = preg_replace("/\<a/", " <a", $page);
+        $page = preg_replace("/\&\#\d{3}(\d?)\;|\&\w+\;/", " ", $page);
+        $page = preg_replace("/\</", " <", $page);
+        $page = strip_tags($page);
+
+        if ($changed) {
+            $page = preg_replace("/(\r?\n[\t| ]*){2}/", "\n", $page);
+        }
+        $page = preg_replace("/(\r?\n[\t| ]*)/", "\n", $page);
+        $page = preg_replace("/\n\n\n+/", "\n\n", $page);
+        return $page;
+    }
+}
\ No newline at end of file
diff --git a/lib/summarizers/graph_based_summarizer.php b/lib/summarizers/graph_based_summarizer.php
new file mode 100644
index 0000000..daaf3c1
--- /dev/null
+++ b/lib/summarizers/graph_based_summarizer.php
@@ -0,0 +1,493 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Charles Bocage charles.bocage@sjsu.edu
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+
+if (!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+/**
+ * Reads in constants used as enums used for storing web sites
+ */
+require_once BASE_DIR."/lib/crawl_constants.php";
+/**
+ * Contains the max_description_length for the summary
+ */
+require_once BASE_DIR."/lib/processors/page_processor.php";
+/**
+ * Contains function getTokenizer to get the object of the language specified.
+ */
+require_once BASE_DIR."/lib/phrase_parser.php";
+/**
+ * Contains the base methods for the summarizer.
+ */
+require_once BASE_DIR."/lib/summarizers/summarizer.php";
+/**
+ * Class which may be used by TextProcessors to get a summary for a text
+ * document that may later be used for indexing.
+ *
+ * add more comments later
+ *
+ * @author Charles Bocage charles.bocage@sjsu.edu
+ * @package seek_quarry\library
+ */
+class GraphBasedSummarizer extends Summarizer
+{
+    /**
+     * Number of distinct terms to use in generating summary
+     */
+    const MAX_DISTINCT_TERMS = 1000;
+    /**
+     * This is a graph based summarizer
+     *
+     * @param string $doc complete raw page to generate the summary from.
+     * @param string $lang language of the page to decide which stop words to
+     *     call proper tokenizer.php of the specified language.
+     *
+     * @return string the summary
+     */
+    static function getGraphBasedSummary($doc, $lang)
+    {
+        $unmodifiedDoc = strip_tags($doc,
+            '<a><h1><h2><h3><h4><h5><h6><b><em><i><u><dl><ol><ul><title>');
+        $doc = self::pageProcessing($doc);
+        $formatted_doc = self::formatDoc($doc);
+        //not filtering non-ascii characters
+        $sentences = self::getSentences($doc . " ", true);
+        $sentences = self::removeStopWords($sentences, $lang);
+        $sentences = self::removePunctuation($sentences);
+        $sentences = PhraseParser::stemTermsK($sentences, $lang, true);
+        $terms = self::getTerms($sentences, $lang);
+        $term_frequencies = self::getTermFrequencies($terms, $sentences,
+            $unmodifiedDoc);
+        $term_frequencies_normalized =
+            self::normalizeTermFrequencies($term_frequencies);
+        $adjacency = self::computeAdjacency($term_frequencies_normalized,
+            $sentences, $lang, $unmodifiedDoc);
+        $p = self::getSentenceRanks($adjacency);
+        $sentences_with_punctuation = self::getSentences($doc . " ", true);
+        $summary = self::getSummary($sentences_with_punctuation, $p);
+        return $summary;
+    }
+    /*
+     * Get the summary from the sentences
+     * @param array $sentences the sentences in the doc
+     * @param array $p the sentence probabilities
+     */
+    static function getSummary($sentences, $p)
+    {
+        $outputToFile = false;
+        $outputFileContents = "";
+        $result = "";
+        $result_length = 0;
+        $n = count($p);
+        for($i = 0; $i < $n; $i++ ) {
+            $index = self::findLargestIndex($p);
+            $p[$index] = -1;
+            $sentence = $sentences[$index];
+            if ($result_length + strlen($sentence) >
+                PageProcessor::$max_description_len) {
+                break;
+            } else {
+                $result_length += strlen($sentence);
+                if ($i == 0) {
+                    $result = $sentence;
+                    if ($outputToFile) {
+                        $outputFileContents = $sentence;
+                    }
+                } else {
+                    $result = $result . " " . $sentence;
+                    if ($outputToFile) {
+                        $outputFileContents = $outputFileContents . "\r\n" .
+                            $sentence;
+                    }
+                }
+            }
+        }
+        if ($outputToFile) {
+            file_put_contents("C:/Temp/out.txt", $outputFileContents);
+        }
+        return $result;
+    }
+    /*
+     * find the largest value in the array and return it
+     * @param array $v the array to search for the largest value
+     * @return double the largest value found in the array
+     */
+    static function findLargestIndex($v)
+    {
+        $result = 0;
+        $n = count($v);
+        $last_value = -1;
+        for ($i = 0; $i < $n; $i++ ) {
+            if ($v[$i] > $last_value) {
+                $last_value = $v[$i];
+                $result = $i;
+            }
+        }
+        return $result;
+    }
+    /*
+     * compute the sentence ranks using a version of the famous
+     * page ranking algorithm developed by the founder of Google.
+     * @param array $adjacency the adjacency matrix generated for the
+     *      sentences
+     * @return array the sentence ranks
+     */
+    static function getSentenceRanks($adjacency)
+    {
+        $n = count($adjacency);
+        $old_p = array();
+        $p = array();
+        for ($i = 0; $i < $n; $i++ ) {
+            $p[$i] = 1 / $n;
+        }
+        for ($i = 0; $i < 10; $i++ ) {
+            $p = self::multiplyMatrixVector($adjacency, $p);
+        }
+        return $p;
+    }
+    /*
+     * Compute the difference of squares
+     * @param array $v the  minuend vector
+     * @param array $m the subtrahend vector
+     * @result double the difference of the squares of vectors
+     */
+    static function squareDiff($v, $w)
+    {
+        $result = 0;
+        $n = count($v);
+        for ($i = 0; $i < $n; $i++ ) {
+            $subtraction = $v[$i] - $w[$i];
+            $result += $subtraction * $subtraction;
+        }
+        return $result;
+    }
+    /*
+     * Perform matrix multiplication on a matrix and a vector
+     * @param array $mat the matrix to multiply the probabilities to
+     * @param array $vec the probability vector
+     * @return array the new vector after it has been multiplied
+     */
+    static function multiplyMatrixVector($mat, $vec)
+    {
+        $result = array();
+        $n = count($vec);
+        for ($i = 0; $i < $n; $i++ ) {
+            $result[$i] = 0;
+            for ($j = 0; $j < $n; $j++ ) {
+                $result[$i] += $mat[$i][$j] * $vec[$j];
+            }
+        }
+        return $result;
+    }
+    /*
+     * Compute the adjacency matrix based on its distortion measure
+     * @param array $term_frequencies_normalized the array of term frequencies
+     * @param array $sentences the sentences in the doc
+     * @param string $lang locale tag for stemming
+     * @param string $doc complete raw page to generate the summary from.
+     * @return array the array of sentence adjacency
+     */
+    static function computeAdjacency($term_frequencies_normalized, $sentences,
+        $lang, $doc)
+    {
+        $result = array(array());
+        $n = count($sentences);
+        for ($i = 0; $i < $n; $i++ ) {
+            $result[$i][$i] = 0;
+            for ($j = $i + 1; $j < $n; $j++ ) {
+                $result[$i][$j] = $result[$j][$i] =
+                    self::findDistortion($sentences[$i], $sentences[$j],
+                    $term_frequencies_normalized, $lang, $doc);
+            }
+        }
+        return $result;
+    }
+    /*
+     * Remove punctuation
+     * @param array $sentences the sentences in the doc
+     * @return array the array of sentences with the punctuation removed
+     */
+     static function removePunctuation($sentences)
+     {
+        $n = count($sentences);
+        for ($i = 0; $i < $n; $i++ ) {
+            $sentences[$i] = trim(preg_replace('/[^a-z0-9]+/iu', ' ',
+                $sentences[$i]));
+        }
+        return $sentences;
+     }
+    /*
+     * Remove the stop words from the array of sentences
+     * @param array $sentences the sentences in the doc
+     * @param string $lang locale tag for stemming
+     * @return array the array of sentences with the stop words removed
+     */
+    static function removeStopWords($sentences, $lang)
+    {
+        $n = count($sentences);
+        $stop_obj = PhraseParser::getTokenizer($lang);
+        if ($stop_obj && method_exists($stop_obj, "stopwordsRemover")) {
+            for ($i = 0; $i < $n; $i++ ) {
+                $sentences[$i] = $stop_obj->stopwordsRemover(
+                    self::formatDoc($sentences[$i]));
+             }
+        }
+        return $sentences;
+    }
+    /*
+     * Calculate the term frequencies.
+     * @param array $terms the list of all terms in the doc
+     * @param array $sentences the sentences in the doc
+     * @param string $doc complete raw page to generate the summary from.
+     * @return array a two dimensional array where the word is the key and
+     *      the frequency is the value
+     */
+    static function getTermFrequencies($terms, $sentences, $doc)
+    {
+        $t = count($terms);
+        $n = count($sentences);
+        $nk = array();
+        $nk = array_fill(0, $t, 0);
+        $nt = array();
+       for($j = 0; $j < $t; $j++) {
+            for($i = 0; $i < $n; $i++) {
+                $nk[$j] += preg_match_all("/\b" . $terms[$j] . "\b/iu",
+                    $sentences[$i], $matches);
+            }
+        }
+        for ($i = 0; $i <  count($nk); $i++ ) {
+            $term_frequencies[$terms[$i]] = $nk[$i];
+        }
+        return $term_frequencies;
+    }
+    /*
+     * Get the terms from an array of sentences
+     * @param array $sentences the sentences in the doc
+     * @param string $lang locale tag for stemming
+     * $return array an array of terms in the array of sentences
+     */
+    static function getTerms($sentences, $lang)
+    {
+        $terms = array();
+        foreach ($sentences as $sentence) {
+            $terms = array_merge($terms,
+                PhraseParser::segmentSegment($sentence, $lang));
+        }
+        $terms = array_filter($terms);
+        $terms_counts = array_count_values($terms);
+        arsort($terms_counts);
+        $terms_counts = array_slice($terms_counts, 0,
+            self::MAX_DISTINCT_TERMS);
+        $terms = array_unique(array_keys($terms_counts));
+        $t = count($terms);
+        if ($t == 0) {
+            return array("", "");
+        }
+        return $terms;
+    }
+    /**
+     * Breaks any content into sentences by splitting it on spaces or carriage
+     *   returns
+     * @param string $content complete page.
+     * @param boolean $keep_punctuation whether to keep the punctuation or not.
+     * @return array array of sentences from that content.
+     */
+    static function getSentences($content, $keep_punctuation)
+    {
+        if ($keep_punctuation) {
+            $sentences =
+                preg_split('/(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|\!)/u',
+                $content, 0, PREG_SPLIT_NO_EMPTY);
+            $n = count($sentences);
+            for ($i = 0; $i < $n; $i++ ) {
+                $sentences[$i] = trim($sentences[$i]);
+            }
+            return $sentences;
+        } else {
+            return preg_split(
+                '/(\.|\||\!|\?|！|？|。)\s+|(\n|\r)(\n|\r)+|\s{5}/u',
+                $content, 0, PREG_SPLIT_NO_EMPTY);
+        }
+    }
+    /*
+     * Normalize the term frequencies based on the sum of the squares.
+     * @param array $term_frequencies the array with the terms as the key
+     *      and its frequency as the value
+     * @return array array of term frequencies normalized
+     */
+    static function normalizeTermFrequencies($term_frequencies)
+    {
+        $sum_of_squares = 0;
+        $result_sum = 0;
+        foreach ($term_frequencies as $k => $v) {
+            $sum_of_squares += ($v * $v);
+        }
+        $square_root = sqrt($sum_of_squares);
+        foreach ($term_frequencies as $k => $v) {
+            $result[$k] = ($v / $square_root);
+        }
+        foreach ($result as $k => $v) {
+            $result_sum += $v;
+        }
+        return $result;
+    }
+    /**
+     *   Calcluate the distortion measure.
+     *   1. Check each word in sentence1 to see if it exists in sentence2.
+     *   If the word X of sentence1 does not exist in sentence2,
+     *   square the score of word X and add to the sum
+     *   and increase the number of not-common words by one.
+     *   2. In case the word X is common between sentence1 and
+     *   sentence2, calculate its frequency in sentence2 and subtract
+     *   it from the score of word X, then square and add to
+     *   sum.
+     *   3. Then check the sentence2 to find its not-common words
+     *   with sentence1, in case the word Y is not in sentence1,
+     *   square the score of word Y and add to sum and increase
+     *   the number of not-common words by one.
+     *   4. At the end, calcualte the distortion between sentence1 and
+     *   sentence2 by dividing sum by the number of not-common
+     *   words.
+     * @param string $first_sentence the first sentence to compare
+     * @param string $second_sentence the second sentence to compare
+     * @param string $term_frequencies the term frequency of the sentences
+     * @param string $lang locale tag for stemming
+     */
+    static function findDistortion($first_sentence, $second_sentence,
+        $term_frequencies, $lang, $doc)
+    {
+        $result = 0;
+        $first_sentence_split = preg_split('/ +/u', $first_sentence);
+        $second_sentence_split = preg_split('/ +/u', $second_sentence);
+        $sum = 0;
+        $non_common_words = 0;
+        $n = count($first_sentence_split);
+        for ($i = 0; $i < $n; $i++ ) {
+            $word_to_search_for = trim($first_sentence_split[$i]);
+            if ($word_to_search_for != "") {
+                preg_match_all("/ " . $word_to_search_for . " /",
+                    $second_sentence, $matches);
+                if (count($matches[0]) == 0) {
+                    $sum += ($term_frequencies[$word_to_search_for] *
+                        $term_frequencies[$word_to_search_for]);
+                    $non_common_words++;
+                } else {
+                    $terms = self::getTerms(array($second_sentence), $lang);
+                    $temp_term_frequencies = self::getTermFrequencies(
+                        $terms, array($second_sentence), $doc);
+                    $temp_term_frequencies_normalized =
+                        self::normalizeTermFrequencies(
+                        $temp_term_frequencies);
+                    $new_term_frequency =
+                        $term_frequencies[$word_to_search_for] -
+                        $temp_term_frequencies_normalized[$word_to_search_for];
+                    $sum += ($new_term_frequency * $new_term_frequency);
+                }
+            }
+        }
+        $n = count($second_sentence_split);
+        for ($i = 0; $i < $n; $i++ ) {
+            $word_to_search_for = trim($second_sentence_split[$i]);
+            if ($word_to_search_for != "") {
+                preg_match_all("/ " . trim($word_to_search_for) . " /",
+                    $first_sentence, $matches);
+                if (count($matches[0]) == 0) {
+                    $sum += ($term_frequencies[$word_to_search_for] *
+                        $term_frequencies[$word_to_search_for]);
+                    $non_common_words++;
+                }
+            }
+        }
+        if ($non_common_words != 0) {
+            $result = $sum / $non_common_words;
+        }
+        return $result;
+    }
+    /**
+     * Formats the sentences to remove all characters except words,
+     *   digits and spaces
+     * @param string $sent complete page.
+     * @return string formatted sentences.
+     */
+    static function formatSentence($sent)
+    {
+        $sent = trim(preg_replace('/[^\p{L}\p{N}\s]+/u',
+            ' ', mb_strtolower($sent)));
+        return $sent;
+    }
+    /**
+     * Formats the document to remove carriage returns, hyphens and digits
+     * as we will not be using digits in word cloud.
+     * The formatted document generated by this function is only used to
+     * compute centroid.
+     * @param string $content formatted page.
+     * @return string formatted document.
+     */
+    static function formatDoc($content)
+    {
+        $substitute = array('/[\n\r\-]+/', '/[^\p{L}\s\.]+/u', '/[\.]+/u');
+        $content = preg_replace($substitute, ' ', mb_strtolower($content));
+        return $content;
+    }
+    /**
+     * This function does an additional processing on the page
+     * such as removing all the tags from the page
+     * @param string $page complete page.
+     * @return string processed page.
+     */
+    static function pageProcessing($page)
+    {
+        $substitutions = array('@<script[^>]*?>.*?</script>@si',
+            '/\&nbsp\;|\&rdquo\;|\&ldquo\;|\&mdash\;/si',
+            '@<style[^>]*?>.*?</style>@si', '/[\^\(\)]/',
+            '/\[(.*?)\]/', '/\t\n/'
+        );
+        $page = preg_replace($substitutions, ' ', $page);
+        $page = preg_replace('/\s{2,}/u', ' ', $page);
+        $new_page = preg_replace("/\<br\s*(\/)?\s*\>/u", "\n", $page);
+        $changed = false;
+        if ($new_page != $page) {
+            $changed = true;
+            $page = $new_page;
+        }
+        $page = preg_replace("/\<\/(h1|h2|h3|h4|h5|h6|table|tr|td|div|".
+            "p|address|section)\s*\>/u", "\n\n", $page);
+        $page = preg_replace("/\<a/u", " <a", $page);
+        $page = preg_replace("/\&\#\d{3}(\d?)\;|\&\w+\;/u", " ", $page);
+        $page = preg_replace("/\</u", " <", $page);
+        $page = strip_tags($page);
+
+        if ($changed) {
+            $page = preg_replace("/(\r?\n[\t| ]*){2}/u", "\n", $page);
+        }
+        $page = preg_replace("/(\r?\n[\t| ]*)/u", "\n", $page);
+        $page = preg_replace("/\n\n\n+/u", "\n\n", $page);
+        return $page;
+    }
+}
\ No newline at end of file
diff --git a/lib/summarizers/scrape_summarizer.php b/lib/summarizers/scrape_summarizer.php
new file mode 100644
index 0000000..23f70d6
--- /dev/null
+++ b/lib/summarizers/scrape_summarizer.php
@@ -0,0 +1,176 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Charles Bocage charles.bocage@sjsu.edu
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+/**
+ * Reads in constants used as enums used for storing web sites
+ */
+require_once BASE_DIR."/lib/crawl_constants.php";
+/**
+ * Contains the max_description_length for the summary
+ */
+require_once BASE_DIR."/lib/processors/page_processor.php";
+/**
+ * Contains function getTokenizer to get the object of the language specified.
+ */
+require_once BASE_DIR."/lib/phrase_parser.php";
+/**
+ * Contains the base methods for the summarizer.
+ */
+require_once BASE_DIR."/lib/summarizers/summarizer.php";
+/**
+ * Class which may be used by TextProcessors to get a summary for a text
+ * document that may later be used for indexing.
+ *
+ * @author Charles Bocage charles.bocage@sjsu.edu
+ * @package seek_quarry\library
+ */
+class ScrapeSummarizer extends Summarizer
+{
+    /**
+     * This is a basic summarizer
+     *
+     * @param object $dom   a document object to extract a description from.
+     * @param string $page original page string to extract description from
+     *
+     * @return string the summary
+     */
+    static function getBasicSummary($dom, $dom_page)
+    {
+        return self::description($dom, $dom_page);
+    }
+    /**
+     * Returns descriptive text concerning a webpage based on its document
+     * object
+     *
+     * @param object $dom   a document object to extract a description from.
+     * @param string $page original page string to extract description from
+     * @return string a description of the page
+     */
+    static function description($dom, $page)
+    {
+        $xpath = new DOMXPath($dom);
+        $metas = $xpath->evaluate("/html//meta");
+        $description = "";
+        //look for a meta tag with a description
+        foreach($metas as $meta) {
+            if(stristr($meta->getAttribute('name'), "description")) {
+                $description .= " .. ".$meta->getAttribute('content');
+            }
+        }
+        if(PageProcessor::$max_description_len > 2 * MAX_DESCRIPTION_LEN) {
+            /* if don't need to summarize much, take meta description
+               from above code, then concatenate body of doc
+               after stripping tags, return result
+             */
+            $description .= "\n".self::crudeDescription($page);
+            return $description;
+        }
+        /*
+          concatenate the contents of then additional dom elements up to
+          the limit of description length. Choose tags in order of likely
+          importance to this doc
+        */
+        $page_parts = array("/html//p[1]",
+            "/html//div[1]", "/html//p[2]", "/html//div[2]", "/html//p[3]",
+            "/html//div[3]", "/html//p[4]", "/html//div[4]",
+            "/html//td", "/html//li", "/html//dt", "/html//dd",
+            "/html//pre", "/html//a", "/html//article",
+            "/html//section", "/html//cite");
+        $para_data = array();
+        $len = 0;
+        foreach($page_parts as $part) {
+            $doc_nodes = $xpath->evaluate($part);
+            foreach($doc_nodes as $node) {
+                if($part == "/html//a") {
+                    $content = $node->getAttribute('href')." = ";
+                    $add_len  = min(PageProcessor::$max_description_len / 2,
+                        mb_strlen($content));
+                    $para_data[$add_len][] = mb_substr($content, 0, $add_len);
+                }
+                $node_text = self::domNodeToString($node);
+                $add_len  = min(PageProcessor::$max_description_len / 2,
+                    mb_strlen($node_text));
+                $para_data[$add_len][] = mb_substr($node_text, 0, $add_len);
+                $len += $add_len;
+
+                if($len > PageProcessor::$max_description_len) { break 2;}
+                if(in_array($part, array("/html//p[1]", "/html//div[1]",
+                    "/html//div[2]", "/html//p[2]", "/html//p[3]",
+                    "/html//div[3]", "/html//div[4]", "/html//p[4]"))){ break;}
+            }
+        }
+        krsort($para_data);
+        foreach($para_data as $add_len => $data) {
+            if(!isset($first_len)) {
+                $first_len = $add_len;
+            }
+            foreach($data as $datum) {
+                $description .= " .. ". $datum;
+            }
+            if($first_len > 3 * $add_len) break;
+        }
+        $description = preg_replace("/(\s)+/u", " ",  $description);
+
+        return $description;
+    }
+    /**
+     * Returns summary of body of a web page based on crude regex matching
+     *     used as a fall back if dom parsing did not work.
+     *
+     * @param string $page to extract description from
+     * @return string  a title of the page
+     */
+    static function crudeDescription($page)
+    {
+        $body = parent::getBetweenTags($page, 0, "<body", "</body");
+        $body = preg_replace("/\</", " <", $body);
+        $body = strip_tags("<body".$body[1]."</body>");
+        if($body == "") { return $body; }
+        $body= preg_replace("/\s+/", " ", $body);
+        return mb_substr($body, 0, self::$max_description_len);
+    }
+    /**
+     * This returns the text content of a node but with spaces
+     * where tags were (unlike just using textContent)
+     *
+     * @param object $node a DOMNode
+     * @return string its text content with spaces
+     */
+    static function domNodeToString($node)
+    {
+        $text = $node->ownerDocument->saveHTML($node);
+        $text = html_entity_decode($text);
+        $text = preg_replace('/\</', ' <', $text);
+        return strip_tags($text);
+    }
+}
+?>
\ No newline at end of file
diff --git a/lib/summarizers/summarizer.php b/lib/summarizers/summarizer.php
new file mode 100644
index 0000000..2dddbdc
--- /dev/null
+++ b/lib/summarizers/summarizer.php
@@ -0,0 +1,141 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Charles Bocage charles.bocage@sjsu.edu
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/**
+ * Base class for all summarizers that will hold common methods and
+ * base functionality.
+ *
+ * @author Charles Bocage charles.bocage@sjsu.edu
+ * @package seek_quarry\library
+ */
+class Summarizer
+{
+    /**
+     * This function will return additional weights if the term is in
+     * certain tags of the document.  Each tag is grouped into 6 categories:
+     * A
+     * H1, H2
+     * H3, H4, H5, H6
+     * STRONG, B, EM, I, U, DL, OL, UL
+     * Title
+     * Plain Text = None of the above
+     *
+     * @param string $term the term to search for.
+     * @param string $doc complete raw page to generate the summary from.
+     *
+     * @return int the additional weight of the term
+     */
+    static function getAdditionalWeight($term, $doc)
+    {
+        $result = 0;
+        $classOneRegex = "/(?s)<a.*?>(.*?" . $term . ".*?)<\/a>/iu";
+        $classTwoRegex = "/(?s)<h1.*?>(.*?" . $term . ".*?)<\/h1>|" .
+            "<h2.*?>(.*?" . $term . ".*?)<\/h2>/iu";
+        $classThreeRegex = "/(?s)<h3.*?>(.*?" . $term . ".*?)<\/h3>|" .
+            "<h4.*?>(.*?" . $term . ".*?)<\/h4>|" .
+            "<h5.*?>(.*?" . $term . ".*?)<\/h5>|" .
+            "<h6.*?>(.*?" . $term . ".*?)<\/h6>/iu";
+        $classFourRegex = "/(?s)<strong.*?>(.*?" . $term . ".*?)<\/strong>|" .
+            "<b.*?>(.*?" . $term . ".*?)<\/b>|" .
+            "<em.*?>(.*?" . $term . ".*?)<\/em>|" .
+            "<i.*?>(.*?" . $term . ".*?)<\/i>|" .
+            "<u.*?>(.*?" . $term . ".*?)<\/u>|" .
+            "<dl.*?>(.*?" . $term . ".*?)<\/dl>|" .
+            "<ol.*?>(.*?" . $term . ".*?)<\/ol>|" .
+            "<ul.*?>(.*?" . $term . ".*?)<\/ul>/iu";
+        $classFiveRegex = "/(?s)<title.*?>(.*?" . $term . ".*?)<\/title>/iu";
+        $classOneWeight = 0;
+        $classTwoWeight = 0;
+        $classThreeWeight = 0;
+        $classFourWeight = 0;
+        $classFiveWeight = 0;
+        $classOneMatches = 0;
+        $classTwoMatches = 0;
+        $classThreeMatches = 0;
+        $classFourMatches = 0;
+        $classFiveMatches = 0;
+        $classOneMatchesCount = 0;
+        $classTwoMatchesCount = 0;
+        $classThreeMatchesCount = 0;
+        $classFourMatchesCount = 0;
+        $classFiveMatchesCount = 0;
+        if (strpos(mb_strtolower($doc),
+            mb_strtolower($term), 0) !== false)
+        {
+            preg_match_all($classOneRegex, $doc, $classOneMatches);
+            preg_match_all($classTwoRegex, $doc, $classTwoMatches);
+            preg_match_all($classThreeRegex, $doc, $classThreeMatches);
+            preg_match_all($classFourRegex, $doc, $classFourMatches);
+            preg_match_all($classFiveRegex, $doc, $classFiveMatches);
+
+            if (count($classOneMatches[0]) > 0) {
+                for($i = 1; $i < count($classOneMatches); $i++) {
+                    $classOneMatchesCount = $classOneMatchesCount +
+                        substr_count($classOneMatches[$i][0], $term);
+                }
+            }
+            if (count($classTwoMatches[0]) > 0) {
+                for($i = 1; $i < count($classTwoMatches); $i++) {
+                    $classTwoMatchesCount = $classTwoMatchesCount +
+                        substr_count($classTwoMatches[$i][0], $term);
+                }
+            }
+            if (count($classThreeMatches[0]) > 0) {
+                for($i = 1; $i < count($classThreeMatches); $i++) {
+                    $classThreeMatchesCount = $classThreeMatchesCount +
+                        substr_count($classThreeMatches[$i][0], $term);
+                }
+            }
+            if (count($classFourMatches[0]) > 0) {
+                for($i = 1; $i < count($classFourMatches); $i++) {
+                    $classFourMatchesCount = $classFourMatchesCount +
+                        substr_count($classFourMatches[$i][0], $term);
+                }
+            }
+            if (count($classFiveMatches[0]) > 0) {
+                for($i = 1; $i < count($classFiveMatches); $i++) {
+                    $classFiveMatchesCount = $classFiveMatchesCount +
+                        substr_count($classFiveMatches[$i][0], $term);
+                }
+            }
+            $result =
+                ($classOneWeight * $classOneMatchesCount) +
+                ($classTwoWeight * $classTwoMatchesCount) +
+                ($classThreeWeight * $classThreeMatchesCount) +
+                ($classFourWeight * $classFourMatchesCount) +
+                ($classFiveWeight * $classFiveMatchesCount);
+        }
+        return $result;
+    }
+}
+?>
\ No newline at end of file
diff --git a/tests/summarizer_test.php b/tests/summarizer_test.php
new file mode 100644
index 0000000..2d6f3d1
--- /dev/null
+++ b/tests/summarizer_test.php
@@ -0,0 +1,202 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+/**
+ * Load the run function
+ */
+require_once BASE_DIR.'lib/unit_test.php';
+/**
+ * Code used to test the summarizer algorithms.
+ *
+ * @author Chris Pollett
+ * @package seek_quarry\test
+ */
+class SummarizerTest extends UnitTest
+{
+    static $summarizerCodeFileContentsSnapshot = "";
+    static $summarizerCodeFileName = BASE_DIR .
+        "lib/summarizers/summarizer.php";
+    /**
+     * Each test we set up a new Summarizer object
+     */
+    function setUp()
+    {
+        self::$summarizerCodeFileContentsSnapshot =
+            file_get_contents(self::$summarizerCodeFileName);
+        $this->modifyWeights();
+        /**
+         * Load the summarizer here because it gets modified in the test
+         * setup
+         */
+        require_once BASE_DIR."/lib/summarizers/summarizer.php";
+        $this->test_objects['FILE1'] = new Summarizer;
+    }
+    /**
+     * Nothing done for unit test tear done
+     */
+    function tearDown()
+    {
+        file_put_contents(self::$summarizerCodeFileName,
+            self::$summarizerCodeFileContentsSnapshot);
+    }
+    /**
+     * Tests whether the stem funtion for the Dutch stemming algorithm
+     * stems words according to the rules of stemming. The function tests stem
+     * by calling stem with the words in $test_words and compares the results
+     * with the stem words in $stem_words
+     *
+     * $test_words is an array containing a set of words in Dutch provided in
+     * the snowball web page
+     * $stem_words is an array containing the stems for words in $test_words
+     */
+    function additionalWeightTestCase()
+    {
+        $stem_dir = BASE_DIR.'/tests/test_files/summarizer';
+        //Test word set from snowball
+        $test_words = file("$stem_dir/summarizer_input.txt");
+        //Stem word set from snowball for comparing results
+        $stem_words = file("$stem_dir/summarizer_result.txt");
+        /**
+         * check if function gets the correct additional weight with the words
+         * in $test_words by comparing results with stem words in $stem_words
+         */
+        for($i = 0; $i < count($test_words); $i++) {
+            $word = preg_split('/\|+/u', trim($test_words[$i]));
+            $stem = trim($stem_words[$i]);
+            $word_stem = $this->test_objects['FILE1']->
+                getAdditionalWeight($word[0], $word[1]);
+            if($stem != $word_stem) {
+                $cleanWord = htmlentities($word[1], ENT_QUOTES, "UTF-8");
+                echo "Calculating the weight of \"$word[0]\" in ".
+                    "$cleanWord to $word_stem should be $stem\n";
+                $this->tearDown();
+                exit();
+            }
+            $this->assertEqual($word_stem,
+                    $stem,"function getAdditionalWeight correctly weights
+                    \"$word[0]\" in $word[1] to $stem");
+        }
+    }
+    /**
+     * Modify the summarizer to have all of the weights set to 1 in order
+     * to get consitent weight for the unit tests
+     */
+    function modifyWeights()
+    {
+        $summarizerCodeFileContents = "";
+        $weightsToTry = 1;
+        $classOneWeightCodeToModify = '$classOneWeight = ';
+        $classTwoWeightCodeToModify = '$classTwoWeight = ';
+        $classThreeWeightCodeToModify = '$classThreeWeight = ';
+        $classFourWeightCodeToModify = '$classFourWeight = ';
+        $classFiveWeightCodeToModify = '$classFiveWeight = ';
+        $file = fopen(self::$summarizerCodeFileName, "r");
+        while(!feof($file)){
+            $line = fgets($file);
+            if ((strpos(mb_strtolower($line),
+                mb_strtolower($classOneWeightCodeToModify)) === false) &&
+                (strpos(mb_strtolower($line),
+                mb_strtolower($classTwoWeightCodeToModify)) === false) &&
+                (strpos(mb_strtolower($line),
+                mb_strtolower($classThreeWeightCodeToModify)) === false) &&
+                (strpos(mb_strtolower($line),
+                mb_strtolower($classFourWeightCodeToModify)) === false) &&
+                (strpos(mb_strtolower($line),
+                mb_strtolower($classFiveWeightCodeToModify)) === false)) {
+                if ($summarizerCodeFileContents == "") {
+                    $summarizerCodeFileContents = $line;
+                } else {
+                    $summarizerCodeFileContents = $summarizerCodeFileContents .
+                        $line;
+                }
+            } else if (strpos(mb_strtolower($line),
+                mb_strtolower($classOneWeightCodeToModify)) !== false) {
+                if ($summarizerCodeFileContents == "") {
+                    $summarizerCodeFileContents = "        " .
+                        $classOneWeightCodeToModify
+                        . $weightsToTry . ";\r\n";
+                } else {
+                    $summarizerCodeFileContents = $summarizerCodeFileContents .
+                        "        " . $classOneWeightCodeToModify .
+                        $weightsToTry . ";\r\n";
+                }
+            } else if (strpos(mb_strtolower($line),
+                mb_strtolower($classTwoWeightCodeToModify)) !== false) {
+                if ($summarizerCodeFileContents == "") {
+                    $summarizerCodeFileContents = "        " .
+                        $classTwoWeightCodeToModify
+                        . $weightsToTry . ";\r\n";
+                } else {
+                    $summarizerCodeFileContents = $summarizerCodeFileContents .
+                        "        " . $classTwoWeightCodeToModify .
+                        $weightsToTry . ";\r\n";
+                }
+            } else if (strpos(mb_strtolower($line),
+                mb_strtolower($classThreeWeightCodeToModify)) !== false) {
+                if ($summarizerCodeFileContents == "") {
+                    $summarizerCodeFileContents = "        " .
+                        $classThreeWeightCodeToModify
+                        . $weightsToTry . ";\r\n";
+                } else {
+                    $summarizerCodeFileContents = $summarizerCodeFileContents .
+                        "        " . $classThreeWeightCodeToModify .
+                        $weightsToTry . ";\r\n";
+                }
+            } else if (strpos(mb_strtolower($line),
+                mb_strtolower($classFourWeightCodeToModify)) !== false) {
+                if ($summarizerCodeFileContents == "") {
+                    $summarizerCodeFileContents = "        " .
+                        $classFourWeightCodeToModify
+                        . $weightsToTry . ";\r\n";
+                } else {
+                    $summarizerCodeFileContents = $summarizerCodeFileContents .
+                        "        " . $classFourWeightCodeToModify .
+                        $weightsToTry . ";\r\n";
+                }
+            } else if (strpos(mb_strtolower($line),
+                mb_strtolower($classFiveWeightCodeToModify)) !== false) {
+                if ($summarizerCodeFileContents == "") {
+                    $summarizerCodeFileContents = "        " .
+                        $classFiveWeightCodeToModify
+                        . $weightsToTry . ";\r\n";
+                } else {
+                    $summarizerCodeFileContents = $summarizerCodeFileContents .
+                        "        " . $classFiveWeightCodeToModify .
+                        $weightsToTry . ";\r\n";
+                }
+            }
+        }
+        fclose($file);
+        file_put_contents(self::$summarizerCodeFileName,
+            $summarizerCodeFileContents);
+    }
+}
+?>
\ No newline at end of file
diff --git a/tests/test_files/summarizer/summarizer_input.txt b/tests/test_files/summarizer/summarizer_input.txt
new file mode 100644
index 0000000..b5409f5
--- /dev/null
+++ b/tests/test_files/summarizer/summarizer_input.txt
@@ -0,0 +1,17 @@
+the|<a href="the.html">I went to the store but the hours were different</a>the<a></a>
+the|<h1>I went to the store but the hours were different</h1>the<h1></h1>
+the|<h2>I went to the store but the hours were different</h2>the<h2></h2>
+the|<h3>I went to the store but the hours were different</h3>the<h3></h3>
+the|<h4>I went to the store but the hours were different</h4>the<h4></h4>
+the|<h5>I went to the store but the hours were different</h5>the<h5></h5>
+the|<h6>I went to the store but the hours were different</h6>the<h6></h6>
+the|<strong>I went to the store but the hours were different</strong>the<strong></strong>
+the|<b>I went to the store but the hours were different</b>the<b></b>
+the|<em>I went to the store but the hours were different</em>the<em></em>
+the|<i>I went to the store but the hours were different</i>the<i></i>
+the|<u>I went to the store but the hours were different</u>the<u></u>
+the|<dl>I went to the store but the hours were different</dl>the<dl></dl>
+the|<ol>I went to the store but the hours were different</ol>the<ol></ol>
+the|<ul>I went to the store but the hours were different</ul>the<ul></ul>
+the|<a href="the.html">I went to the store but the hours were different</a>the in between<a></a><h1>h1I went to the store but the hours were differenth1</h1>the h1 tag<h1></h1>
+the|<html><body>I went to the 49ers game last week.<a>Here is the link</a></body></html>
\ No newline at end of file
diff --git a/tests/test_files/summarizer/summarizer_result.txt b/tests/test_files/summarizer/summarizer_result.txt
new file mode 100644
index 0000000..cf31695
--- /dev/null
+++ b/tests/test_files/summarizer/summarizer_result.txt
@@ -0,0 +1,17 @@
+2
+2
+2
+2
+2
+2
+2
+2
+2
+2
+2
+2
+2
+2
+2
+4
+1
\ No newline at end of file
-- 
1.9.5.github.0

