- Yioop Repo - Seekquarry

<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\tests;

use seekquarry\yioop\library\summarizers\Summarizer;
use seekquarry\yioop\library\UnitTest;

/**
 * SummarizerTest checks how Summarizer counts the words of a page's text,
 * which a fetcher does for each page it processes.
 *
 * @author Chris Pollett
 */
class SummarizerTest extends UnitTest
{
    /**
     * setUp does nothing, since each case makes its own text.
     */
    public function setUp()
    {
    }
    /**
     * tearDown does nothing, since a case leaves nothing behind.
     */
    public function tearDown()
    {
    }
    /**
     * longRunsKeepTheirSentencesTestCase checks that getSentences keeps the
     * text of a block holding a long run of letters with no space, or a
     * long run of newlines. A sentence end comes after a word of at least
     * five letters, so "here." does not end one. A run longer than
     * Summarizer::MAX_SENTENCE_LEN comes back in pieces.
     */
    public function longRunsKeepTheirSentencesTestCase()
    {
        $run = str_repeat("a", self::LONG_RUN);
        $pieces = Summarizer::getSentences(
            "First words here. $run. Last words here. ");
        $this->assertEqual(str_replace(" ", "", implode("", $pieces)),
            "Firstwordshere.$run.Lastwordshere.",
            "a long run of letters keeps its text");
        $this->assertEqual(end($pieces), "Last words here. ",
            "a long run of letters ends its sentence");
        $this->assertEqual(Summarizer::getSentences(
            "Before the gap" . str_repeat("\n", self::LONG_RUN) .
            "after the gap."),
            ["Before the gap.", "after the gap."],
            "a long run of newlines ends a sentence");
    }
    /**
     * sentenceTermFrequenciesCountTheSentencesTermsTestCase checks the term
     * frequencies computeTermFrequenciesPerSentence gives, before they are
     * normalized, through their ratios. A Chinese sentence, written with no
     * spaces, counts each of its words, where searching the sentence for
     * each word as a whole word found none. A sentence of
     * NUM_TAGS_IN_SENTENCE terms that are not single words, such as #tag1,
     * is counted in under a second, where searching the sentence for each
     * took several seconds.
     */
    public function sentenceTermFrequenciesCountTheSentencesTermsTestCase()
    {
        list(, $frequencies) = Summarizer::computeTermFrequenciesPerSentence(
            ["韩国政府宣布韩国文化计划"], "zh-CN");
        $this->assertTrue(($frequencies[0]["韩国"] ?? 0) > 0,
            "a Chinese word is counted");
        $this->assertEqual(round(($frequencies[0]["韩国"] ?? 0) /
            (($frequencies[0]["政府"] ?? 0) ?: 1)), 2.0,
            "a Chinese word twice in the sentence counts twice");
        $tags = implode(" ", array_map(fn($i) => "#tag$i",
            range(1, self::NUM_TAGS_IN_SENTENCE))) . " #tag1";
        $start = microtime(true);
        list(, $frequencies) = Summarizer::computeTermFrequenciesPerSentence(
            [$tags], "en");
        $this->assertTrue(microtime(true) - $start < 1,
            "a sentence of many terms that are not words is quick");
        $this->assertEqual(round($frequencies[0]["#tag1"] /
            ($frequencies[0]["#tag2"] ?: 1)), 2.0, "such a term is counted");
    }
    /**
     * NUM_TAGS_IN_SENTENCE is how many terms that are not single words
     * sentenceTermFrequenciesCountTheSentencesTermsTestCase puts in one
     * sentence.
     */
    const NUM_TAGS_IN_SENTENCE = 20000;
    /**
     * LONG_RUN is how many letters, or newlines, longRunsKeepTheir
     * SentencesTestCase puts in one run. It is well past the few thousand
     * at which a pattern that rereads a run, or repeats a group once per
     * character, gives up.
     */
    const LONG_RUN = 50000;
    /**
     * longSentencesAreCutTestCase checks that a run with no sentence end
     * longer than Summarizer::MAX_SENTENCE_LEN comes back as pieces no
     * longer than that, cut at spaces, holding all its words in order,
     * and that a run of one script with no spaces is cut too
     */
    public function longSentencesAreCutTestCase()
    {
        $max = Summarizer::MAX_SENTENCE_LEN;
        $words = [];
        for ($i = 0; $i < 400; $i++) {
            $words[] = "word$i";
        }
        $text = implode(" ", $words);
        $pieces = Summarizer::getSentences($text);
        $this->assertTrue(count($pieces) > 1, "a long run is cut");
        $longest = max(array_map("mb_strlen", $pieces));
        $this->assertTrue($longest <= $max, "no piece is too long");
        $this->assertEqual(implode(" ", $pieces), $text,
            "the pieces hold the words in order, cut at spaces");
        $han = str_repeat("字", 3 * $max);
        $pieces = Summarizer::getSentences($han);
        $this->assertEqual(count($pieces), 3, "a run with no spaces is cut");
        $this->assertEqual(implode("", $pieces), $han,
            "the pieces hold the run");
        $this->assertEqual(Summarizer::getSentences("A short one."),
            ["A short one."], "a short sentence is kept as is");
    }
}