<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett (chris@pollett.org)
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\summarizers;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\processors\PageProcessor;
use seekquarry\yioop\library\data_structures\LinearAlgebra;
/**
* CentroidWeightedSummarizer is one of the summarizers TextProcessors and
* HtmlProcessor may use, through getSummary, to summarize a document for
* the index; a crawl picks it as its summarizer option. To generate a
* summary a normalized term frequency vector is computed for each
* sentence. An average vector is then computed by summing these and
* renormalizing the result.
* The computation of this average vector is biased by weighting earlier
* sentences vectors more when computing the sum of vectors. This is done
* using weight coming from a Zipf like distribution. Once an average
* sentence is obtained, then sentences are score against it using
* a residual cosine similarity score. I.e., the most important sentence
* is determined by cosine rank. Then the components of this sentence in
* the direction of the average sentence is deleted from the average sentence.
* and the next most important sentence is computed by ranking against this
* new average sentence vector and so on.
* @author Charles Bocage (charles.bocage@sjsu.edu)
* rewritten Chris Pollett (chris@pollett.org)
*/
class CentroidWeightedSummarizer extends Summarizer
{
/**
* $step_times holds, for the last call of getSummary, the seconds it
* spent in each of its parts, named with what each part read: finding
* the sentences, with how many it found, the term frequencies of each
* sentence, the average sentence and word cloud, scoring the
* sentences against the average, and putting the summary together.
* HtmlProcessor adds them to its own step times, so a fetcher's slow
* page line says which part of a slow summary took the time.
* @var array
*/
public static $step_times = [];
/**
* getSummary generates a summary, word cloud, and summary scores based
* on the closeness of normalized term frequency vectors to an average
* term frequency vector for sentences.
*
* @param object $dom document object model of page to summarize
* @param string $page complete raw page to generate the summary from.
* @param string $lang language of the page to decide which stop words to
* call proper tokenizer.php of the specified language.
*
* @return array a triple (string summary, array word cloud, array
* of position => scores for positions within the summary)
*/
public static function getSummary($dom, $page, $lang)
{
self::$step_times = [];
$step_start = microtime(true);
$end_step = function ($name) use (&$step_start) {
$now = microtime(true);
self::$step_times[$name] = $now - $step_start;
$step_start = $now;
};
list($original_sentences, $sentences) =
self::getPunctuatedUnpunctuatedSentences($dom, $page, $lang);
$end_step("sentences, " . count($sentences) . " of them");
list($terms, $tf_per_sentence_normalized) =
self::computeTermFrequenciesPerSentence($sentences, $lang);
$vector_sizes = array_map('count', $tf_per_sentence_normalized);
$end_step("term frequencies, " . array_sum($vector_sizes) .
" in all, at most " . max([0, ...$vector_sizes]) .
" in a sentence");
$tf_average_sentence =
self::getAverageSentence($tf_per_sentence_normalized);
$word_cloud = self::wordCloudFromTermVector($tf_average_sentence);
$end_step("average and word cloud, " . count($tf_average_sentence) .
" terms");
$sorted_sentence_scores =
self::scoreSentencesVersusAverage($tf_per_sentence_normalized,
$tf_average_sentence, $original_sentences);
$end_step("scoring, " . count($sorted_sentence_scores) . " picked");
list($summary, $summary_scores) = self::getSummaryFromSentenceScores(
$sorted_sentence_scores, $original_sentences, $lang);
$end_step("putting together");
/* Summary of text summarization */
return [$summary, $word_cloud, $summary_scores];
}
/**
* In getAverageSentence, each sentence's normalized term frequency
* vector is weighted by a Zipf like distribution on the sentence's
* index, so earlier sentences count more, and the weighted vectors are
* added and the sum normalized, giving the page's average sentence
* @param array $term_frequencies_normalized the array with the terms as
* the key and its normalized frequency as the value
* @return array a normalized vector of term => weights
*/
public static function getAverageSentence($term_frequencies_normalized)
{
$average_sentence = [];
if (!empty($term_frequencies_normalized)) {
foreach ($term_frequencies_normalized as
$sentence_index => $term_frequencies_sentence) {
/* used to slightly favor earlier sentences according
to a Zipf like behavior
*/
$sentence_weight = 1.0/(1.0 + pow($sentence_index, 0.80));
foreach ($term_frequencies_sentence as
$term_index => $frequency) {
$frequency *= $sentence_weight;
$average_sentence[$term_index] =
(empty($average_sentence[$term_index])) ?
$frequency :
$average_sentence[$term_index] + $frequency;
}
}
$average_sentence = LinearAlgebra::normalize($average_sentence);
arsort($average_sentence);
}
return $average_sentence;
}
/**
* scoreSentencesVersusAverage picks a page's sentences one at a time,
* each the sentence whose normalized word vector has the largest inner
* product with the average sentence, and takes the picked sentence's
* direction out of the average before the next pick, so the summary
* does not say the same thing twice.
*
* @param array $sentence_vectors the array with the terms as
* the key and its normalized frequency as the value
* @param array $average_sentence an array of each words average
* frequency value
* @param array $sentences the sentences as they appear on the page, by
* the same index as $sentence_vectors. Scoring stops once the
* sentences picked are as long as
* PageProcessor::$max_description_len, since
* getSummaryFromSentenceScores takes no sentence past that
* length. Each sentence is scored once, and a pick then lowers
* only the scores of the sentences sharing its terms.
* With no sentences given, every sentence is scored.
* @return array array of sentence index => score pairs
*/
public static function scoreSentencesVersusAverage($sentence_vectors,
$average_sentence, $sentences = [])
{
/* Each sentence's score is worked out once. A pick only changes
the average on the picked sentence's terms, so afterward only
the sentences sharing one of those terms have their scores
lowered, by what that term gave them, found through a list of
the sentences each term is in. Before, every pick worked out
every sentence's score again. */
$scores = [];
$sentences_with_term = [];
foreach ($sentence_vectors as $sentence => $word_vector) {
$scores[$sentence] = LinearAlgebra::dot($average_sentence,
$word_vector);
foreach ($word_vector as $term => $weight) {
$sentences_with_term[$term][$sentence] = $weight;
}
}
$sentence_scores = [];
$max_sentence_score = 1;
$picked_length = 0;
while ($max_sentence_score > 0 && ($sentences == [] ||
$picked_length < PageProcessor::$max_description_len)) {
$max_sentence_score = -1;
$max_sentence_index = -1;
foreach ($scores as $sentence => $sentence_score) {
if ($max_sentence_score < $sentence_score) {
$max_sentence_score = $sentence_score;
$max_sentence_index = $sentence;
}
}
if ($max_sentence_score > -1) {
$sentence_scores[$max_sentence_index] = $max_sentence_score;
$picked_length += strlen($sentences[$max_sentence_index] ??
"");
unset($scores[$max_sentence_index]);
/* The picked sentence's direction, its vector being
normalized, comes out of the average. */
foreach ($sentence_vectors[$max_sentence_index] as
$term => $weight) {
$taken = ($average_sentence[$term] ?? 0) * $weight;
if (empty($taken)) {
continue;
}
$average_sentence[$term] -= $taken;
foreach ($sentences_with_term[$term] as
$sentence => $sentence_weight) {
if (isset($scores[$sentence])) {
$scores[$sentence] -= $taken * $sentence_weight;
}
}
}
}
}
return $sentence_scores;
}
}
- Yioop Repo - Seekquarry