- Yioop Repo - Seekquarry

<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\tests;

use seekquarry\yioop\library\CrawlConstants as CC;
use seekquarry\yioop\library\indexing_plugins\WordfilterPlugin;
use seekquarry\yioop\library\UnitTest;

/**
 * Checks the rule presets of WordfilterPlugin, which an administrator loads
 * from the plugin's configure screen under Page Options > Crawl Time.
 *
 * @author Chris Pollett
 */
class WordfilterPluginTest extends UnitTest
{
    /**
     * The plugin under test, with no saved configuration read
     * @var WordfilterPlugin
     */
    public $plugin;
    /**
     * Makes a fresh plugin for each case
     */
    public function setUp()
    {
        $this->plugin = new WordfilterPlugin();
    }
    /**
     * Nothing is kept between cases
     */
    public function tearDown()
    {
        $this->plugin = null;
    }
    /**
     * factoryRulesDropBotCheckPagesTestCase loads the Factory Settings and
     * checks that the pages sites send in place of their content to a
     * crawler they take for a bot are dropped, while an ordinary page and an
     * article that only mentions captchas are kept. On a laptop crawl, many
     * press-and-hold pages were cached in place of the pages asked for.
     */
    public function factoryRulesDropBotCheckPagesTestCase()
    {
        $this->plugin->rules_string =
            $this->plugin->preset_rules["restore"];
        $this->plugin->setConfiguration($this->plugin->parseRules());
        $pages = [
            ["Access to this page has been denied",
                "Press & Hold to confirm you are a human (and not a bot).",
                false],
            ["Just a moment...",
                "Verifying you are human. This may take a few seconds.",
                false],
            ["example.com", "Please enable JS and disable any ad blocker",
                false],
            ["Access Denied", "You don't have permission to access " .
                "this page on this server. Reference #18.5c", false],
            ["Ars Technica", "News, reviews, and analysis", true],
            ["How captchas work", "A captcha asks a person to prove they " .
                "are human; some ask you to press and hold a button.", true],
        ];
        foreach ($pages as [$title, $description, $kept]) {
            $summary = [CC::TITLE => $title, CC::DESCRIPTION => $description];
            $this->plugin->pageSummaryProcessing($summary,
                "https://www.example.com/page");
            $this->assertEqual($summary !== false, $kept,
                ($kept ? "kept: " : "dropped: ") . $title);
        }
    }
    /**
     * demoRulesKeepTheTestCrawlRulesTestCase loads the Demo Canada Crawl
     * preset and checks it still holds the dummy rules the test crawl in
     * tests/test_files/word-filter-test-crawl.zip relies on, and that it
     * drops a Wikipedia page that does not mention Canada.
     */
    public function demoRulesKeepTheTestCrawlRulesTestCase()
    {
        $rules = $this->plugin->preset_rules["demo_canada"];
        $this->assertTrue(str_contains($rules, "term1:NOPROCESS"),
            "the test crawl rules are there");
        $this->plugin->rules_string = $rules;
        $this->plugin->setConfiguration($this->plugin->parseRules());
        $summary = [CC::TITLE => "Ottawa", CC::DESCRIPTION =>
            "A city in Ontario. term0"];
        $this->plugin->pageSummaryProcessing($summary,
            "https://en.wikipedia.org/wiki/Ottawa");
        $this->assertTrue($summary === false,
            "a Wikipedia page without canada or canadian is dropped");
    }
}