Last commit for src/library/processors/CompressedProcessor.php: 2addb500315b7393a90fe66431d7832b1e7386c7

Adjust copyrights years

Chris Pollett [2024-01-03 21:Jan:rd]
Adjust copyrights years
<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2023  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2023
 * @filesource
 */
namespace seekquarry\yioop\library\processors;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\PartialZipArchive;

/**
 * Used to create crawl summary information
 * for a gz compressed file whose uncompressed form has
 * a processor we index.
 *
 * @author Chris Pollett
 */
class CompressedProcessor extends PageProcessor
{
    /**
     * Set-ups the any indexing plugins associated with this page
     * processor
     *
     * @param array $plugins an array of indexing plugins which might
     *     do further processing on the data handles by this page
     *     processor
     * @param int $max_description_len maximal length of a page summary
     * @param int $max_links_to_extract maximum number of links to extract
     *      from a single document
     * @param string $summarizer_option CRAWL_CONSTANT specifying what kind
     *      of summarizer to use self::BASIC_SUMMARIZER,
     *      self::GRAPH_BASED_SUMMARIZER and self::CENTROID_SUMMARIZER
     *      self::CENTROID_SUMMARIZER
     */
    public function __construct($plugins = [], $max_description_len = null,
        $max_links_to_extract = null,
        $summarizer_option = self::BASIC_SUMMARIZER)
    {
        parent::__construct($plugins, $max_description_len,
            $max_links_to_extract, $summarizer_option);
        /** Register File Types We Handle*/
        self::$indexed_file_types[] = "gz";
        self::$indexed_file_types[] = "bz";
        self::$mime_processor["application/x-gzip"] = "CompressedProcessor";
        self::$mime_processor["application/x-bzip"] = "CompressedProcessor";
        self::$mime_processor["application/x-bzip2"] = "CompressedProcessor";
        self::$mime_processor["application/zip"] = "CompressedProcessor";
        self::$mime_processor["application/x-zip-compressed"] =
            "CompressedProcessor";
    }
    /**
     * Used to extract the title, description and links from
     * a string consisting of compressed file of some known indexed_file_type
     *
     * @param string $page   web-page contents
     * @param string $url   the url where the page contents came from,
     *    used to canonicalize relative links
     *
     * @return array  a summary of the contents of the page
     *
     */
    public function process($page, $url)
    {
        if (preg_match('/^(.+)\.(gz|bz|zip)$/', $url, $match) === false ||
            empty($match[2])) {
            return null;
        }
        $uncompress_url = $match[1];
        $compress_type = $match[2];
        $compress_open = $compress_type . "open";
        $compress_read = $compress_type . "read";
        $compress_close = $compress_type . "close";
        $mime_type = UrlParser::guessMimeTypeFromFileName($uncompress_url,
            "unknown");
        if ($mime_type == 'unknown' ||
            !isset(self::$mime_processor[$mime_type]) ) {
            return null;
        }
        $uncompress_page = "";
        if ($compress_type == 'zip') {
            $file_name = UrlParser::getDocumentFilename($url);
            $zip = new PartialZipArchive($page);
            if ($zip) {
                $uncompress_page = $zip->getFromName($file_name);
            }
        } else {
            $temp_dir = C\CRAWL_DIR . "/temp/";
            if (!file_exists($temp_dir)) {
                 mkdir($temp_dir);
            }
            if (!file_exists($temp_dir)) {
                return null;
            }
            $temp_file = $temp_dir .  L\crawlHash($url);
            file_put_contents($temp_file, $page);
            if (!file_exists($temp_file)) {
                return null;
            }
            $ch = $compress_open($temp_file, "r");
            $max_size = 10 * C\PAGE_RANGE_REQUEST;
            while ($block = $compress_read($ch, 8192)) {
                $uncompress_page .= $block;
                if (strlen($uncompress_page) > $max_size) {
                    break;
                }
            }
            $compress_close($ch);
            unlink($temp_file);
        }
        if (empty($uncompress_page)) {
            return null;
        }
        $processor_name = C\NS_PROCESSORS .
            PageProcessor::$mime_processor[$mime_type];
        $processor = new $processor_name([], self::$max_description_len,
            $this->summarizer_option);
        return $processor->process($uncompress_page, $url);
    }
    /**
     * Return a document object based on a string containing the contents of
     * an XML page
     *
     * @param string $page   a web page
     *
     * @return object  document object
     */
    public static function dom($page)
    {
        return L\getDomFromString($page);
    }
}
ViewGit