<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\processors;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\Library as L;
use seekquarry\yioop\library\ComputerVision;
use seekquarry\yioop\library\UrlParser;

/**
 * Used to create crawl summary information
 * for PDF files
 *
 * @author Chris Pollett
 */
class PdfProcessor extends TextProcessor
{
    /**
     * Set-ups the any indexing plugins associated with this page
     * processor
     *
     * @param array $plugins an array of indexing plugins which might
     *     do further processing on the data handles by this page
     *     processor
     * @param int $max_description_len maximal length of a page summary
     * @param int $max_links_to_extract maximum number of links to extract
     *      from a single document
     * @param string $summarizer_option CRAWL_CONSTANT specifying what kind
     *      of summarizer to use self::BASIC_SUMMARIZER,
     *      self::GRAPH_BASED_SUMMARIZER and self::CENTROID_SUMMARIZER
     *      self::CENTROID_SUMMARIZER
     */
    public function __construct($plugins = [], $max_description_len = null,
        $max_links_to_extract = null,
        $summarizer_option = self::BASIC_SUMMARIZER)
    {
        parent::__construct($plugins, $max_description_len,
            $max_links_to_extract,$summarizer_option);
        /** Register File Types We Handle*/
        self::$indexed_file_types[] = "pdf";
        self::$mime_processor["application/pdf"] = "PdfProcessor";
    }
    /**
     * Used to extract the title, description and links from
     * a string consisting of PDF data.
     *
     * @param $page  a string consisting of web-page contents
     * @param $url  the url where the page contents came from,
     *    used to canonicalize relative links
     *
     * @return a summary of the contents of the page
     *
     */
    public function process($page, $url)
    {
        $text = "";
        if (is_string($page)) {
            list($encoding, $title) = self::getEncodingTitle($page);
            $text =  self::getText($page, $url, $encoding);
        }
        if ($text == "") {
            $text = $url;
        }
        $summary = parent::process($text, $url);
        if ($title) {
            $summary[self::TITLE] = $title;
        }
        return $summary;
    }
    /**
     * Used to create an thumbnail file to a thumb folder from an
     * PDF file provided the image magick command convert exists.
     * For this method to do anything the constant IMAGE_MAGICK
     * must be set to the path to the "convert" command.
     *
     * @param string $folder with pdf in it
     * @param string $thumb_folder folder to generate
     * @param string $file_name of pdf file in $folder
     * @param int $width = width in pixels of thumb
     * @param int $height = height in pixels of thumb
     */
    public static function createThumb($folder, $thumb_folder, $file_name,
        $width = C\THUMB_DIM, $height = C\THUMB_DIM)
    {
        if(!function_exists("exec") || !C\nsdefined("IMAGE_MAGICK")) {
            return;
        }
        if (file_exists("$thumb_folder/$file_name.webp")) {
            @unlink("$thumb_folder/$file_name.webp");
        }
        $make_thumb = C\IMAGE_MAGICK . " \"$folder/$file_name"."[0]\" ".
            "-resize ". $width . "x" .$height .
            " \"$thumb_folder/$file_name.webp\"";
        exec($make_thumb);
        clearstatcache("$thumb_folder/$file_name.webp");
    }
    /**
     * Returns the first encoding format information found in the PDF document
     *
     * @param string $pdf_string a string representing the PDF document
     * @return array [encoding, title] which of the default (if any) PDF
     * encoding formats is being used: MacRomanEncoding, WinAnsiEncoding,
     * PDFDocEncoding, etc as well as a title for the document if found
     *
     */
    public static function getEncodingTitle($pdf_string)
    {
        $len = strlen($pdf_string);
        $cur_pos = 0;
        $out = "";
        $i = 0;
        set_error_handler(null);
        $encoding = "";
        $title = "";
        while($cur_pos < $len && (!$encoding || !$title)) {
            list($cur_pos, $object_string) =
                self::getNextObject($pdf_string, $cur_pos);
            $object_dictionary = self::getObjectDictionary($object_string);
            if (preg_match("/\/(\w+Encoding)\b/", $object_dictionary,
                $match) != false) {
                $encoding = $match[1];
            }
            if (preg_match("/\/Title\(([^\)]+)\)/", $object_dictionary,
                $match) != false) {
                $title = $match[1];
            }
        }
        restore_error_handler();
        return [$encoding, $title];
    }
    /**
     * Gets the text out of a PDF document
     *
     * @param string $pdf_string a string representing the PDF document
     * @param $url  the url where the page contents came from,
     *    used to canonicalize relative links
     * @param string $encoding which of the default (if any) PDF encoding
     *    formats is being used: MacRomanEncoding, WinAnsiEncoding,
     *    PDFDocEncoding, etc.
     * @return string text extracted from the document
     */
    public static function getText($pdf_string, $url, $encoding = "")
    {
        $len = strlen($pdf_string);
        $cur_pos = 0;
        $out = "";
        $i = 0;
        set_error_handler(null);
        $state = "text";
        $temp_dir = C\TEMP_DIR . "/";
        if (!file_exists($temp_dir)) {
             mkdir($temp_dir);
        }
        if (!file_exists($temp_dir)) {
            return null;
        }
        $lang = UrlParser::getLang($url);
        while($cur_pos < $len) {
            list($cur_pos, $object_string) =
                self::getNextObject($pdf_string, $cur_pos);
            $object_dictionary = self::getObjectDictionary($object_string);
            if (ComputerVision::ocrEnabled() &&
                self::objectDictionaryHas($object_dictionary, ["Image"]) &&
                self::objectDictionaryHas($object_dictionary, ["XObject"]) &&
                self::objectDictionaryHas($object_dictionary, ["Width"]) &&
                self::objectDictionaryHas($object_dictionary, ["Height"]) &&
                !self::objectDictionaryHas($object_dictionary, ["ImageMask"])) {
                $stream_data = ltrim(self::getObjectStream($object_string));
                preg_match("/\/Width\s+(\d+)\b/", $object_dictionary, $matches);
                $width = $matches[1] ?? 0;
                preg_match("/\/Height\s+(\d+)\b/", $object_dictionary,
                    $matches);
                $height = $matches[1] ?? 0;
                preg_match("/\/BitsPerComponent\s+(\d+)\b/", $object_dictionary,
                    $matches);
                $bits_per_component = $matches[1] ?? 8;
                preg_match("/\/ColorSpace\s+(Device)?(Gray|RGB|CMYK)\b/",
                    $object_dictionary, $matches);
                $color_space = $matches[2] ?? "RGB";
                $is_jpeg = preg_match("/\/Filter\s+\/DCTDecode\b/",
                    $object_dictionary);
                if (!$width || !$height || $color_space == "CMYK") {
                    continue;
                }
                $is_rgb = ($color_space == "RGB");
                if (self::objectDictionaryHas($object_dictionary,
                    ["FlateDecode"])) {
                    $stream_data = @gzuncompress($stream_data);
                }
                if ($is_jpeg) {
                    // if pdf corrupted this can also through an error
                    @$image  = imagecreatefromstring($stream_data);
                } else {
                    $image  = imagecreatetruecolor($width, $height);
                    $pix_loc = 0;
                    for($y = 0; $y < $height; $y++) {
                        for($x = 0; $x < $width; $x++) {
                            if ($is_rgb) {
                                $r = empty($stream_data[$pix_loc]) ? 255 :
                                    ord($stream_data[$pix_loc]);
                                $g = empty($stream_data[$pix_loc + 1]) ? 255 :
                                    ord($stream_data[$pix_loc + 1]);
                                $b = empty($stream_data[$pix_loc + 2]) ? 255 :
                                    ord($stream_data[$pix_loc + 2]);
                                $pix_loc += 3;
                            } else {
                                $r = empty($stream_data[$pix_loc]) ? 255 :
                                    ord($stream_data[$pix_loc]);
                                $g = $r;
                                $b = $r;
                                $pix_loc++;
                            }
                            $color = imagecolorallocate($image, $r, $g, $b);
                            imagesetpixel($image, $x, $y, $color);
                        }
                    }
                }
                $temp_file = $temp_dir . L\crawlHash($stream_data) . ".png";
                if ($image) {
                    @imagepng($image, $temp_file);
                    $ocr_data = ComputerVision::recognizeText($temp_file,
                        [$lang]);
                    if (!empty($ocr_data)) {
                        $out  .= $ocr_data;
                    }
                    @unlink($temp_file);
                }
            } else if (self::objectDictionaryHas(
                $object_dictionary, ["Type", "Font", "FontDescriptor"])) {
                $state = "font";
                continue;
            }
            if (!self::objectDictionaryHas(
                $object_dictionary, ["Image", "Catalog"])) {
                $stream_data =
                    rtrim(ltrim(self::getObjectStream($object_string)));
                if ($state == 'font') {
                    $state == 'text';
                    continue;
                }
                if (self::objectDictionaryHas(
                    $object_dictionary, ["FlateDecode"])) {
                    $stream_data = @gzuncompress($stream_data);
                    if (strpos($stream_data, "PS-AdobeFont")) {
                        $out .= $stream_data . "\n\n";
                    }
                    $text = self::parseText($stream_data, $encoding);
                    $out .= $text. "\n\n";
                } else {
                    $text = self::parseText($stream_data, $encoding);
                    if (strpos($stream_data, "PS-AdobeFont")){
                        $out .= $stream_data . "\n\n";
                    }
                    $out .= $text . "\n\n";
                }
            }
        }
        restore_error_handler();
        $font_pos = strpos($out, "PS-AdobeFont");
        if (!$font_pos) {
            $font_pos = strlen($out);
        }
        $out = substr($out, 0, $font_pos);
        return $out;
    }
    /**
     * Gets between an obj and endobj tag at the current position in a PDF
     * document
     *
     * @param string $pdf_string astring of a PDF document
     * @param int $cur_pos a integer position in that string
     * @return string the contents of the PDF object located at $cur_pos
     */
    public static function getNextObject($pdf_string, $cur_pos)
    {
        return self::getBetweenTags($pdf_string, $cur_pos, "obj", "endobj");
    }
    /**
     * Checks if the PDF object's object dictionary is in a list of types
     *
     * @param string $object_dictionary the object dictionary to check
     * @param array $type_array the list of types to check against
     * @return whether it is in or not
     */
    public static function objectDictionaryHas($object_dictionary, $type_array)
    {
        foreach ($type_array as $type) {
            if (strstr($object_dictionary, $type)) {
                return true;
            }
        }
        return false;
    }
    /**
     * Gets the object dictionary portion of the current PDF object
     * @param string $object_string represents the contents of a PDF object
     * @return string the object dictionary for the object
     */
    public static function getObjectDictionary($object_string)
    {
        list( , $object_dictionary) =
            self::getBetweenTags($object_string, 0, '<<', '>>');
        return $object_dictionary;
    }
    /**
     * Gets the object stream portion of the current PDF object
     *
     * @param string $object_string represents the contents of a PDF object
     * @return string the object stream for the object
     */
    public static function getObjectStream($object_string)
    {
        list( , $stream_data) =
            self::getBetweenTags($object_string, 0, 'stream', 'endstream');
        return $stream_data;
    }
    /**
     * Extracts text from PDF data, getting rid of non printable data,
     * square brackets and parenthesis and converting char codes to their
     * values.
     *
     * @param string $data source to extract character data from
     * @param string $encoding which of the default (if any) PDF encoding
     *    formats is being used: MacRomanEncoding, WinAnsiEncoding,
     *    PDFDocEncoding, etc.
     * @return string extracted text
     */
    public static function parseText($data, $encoding = "")
    {
        $cur_pos = 0;
        //replace ASCII codes in decimal with their value
        $data = preg_replace_callback('/\\\(\d{3})/',
            function($matches) {
                return chr(intval($matches[1]));
            },
            $data);
        //replace ASCII codes in hex with their value
        $data = preg_replace_callback('/\<([0-9A-F]{2})\>/',
            function($matches) {
                return chr(hexdec($matches[1]));
            },
            $data);
        $len = strlen($data);
        $out = "";
        $escape_flag = false;
        while($cur_pos < $len) {
            $cur_char = $data[$cur_pos];
            if ($cur_char == '[' && !$escape_flag) {
                list($cur_pos, $text) = self::parseBrackets($data, $cur_pos,
                    $encoding);
                $cur_pos--;
                $out .= " ". $text;
            }
            if ($cur_char == '\\') {
                $escape_flag = true;
            } else {
                $escape_flag = false;
            }
            $cur_pos++;
        }
        return $out;
    }
    /**
     * Extracts text till the next close brackets
     *
     * @param string $data source to extract character data from
     * @param int $cur_pos position to start in $data
     * @param string $encoding which of the default (if any) PDF encoding
     *    formats is being used: MacRomanEncoding, WinAnsiEncoding,
     *    PDFDocEncoding, etc.
     * @return array pair consisting of the final position in $data as well
     *     as extracted text
     */
    public static function parseBrackets($data, $cur_pos, $encoding = "")
    {
        $cur_pos++;
        $len = strlen($data);
        $out = "";
        $escape_flag =false;
        $cur_char = "";
        while($cur_pos < $len && ($cur_char != "]")) {
            $cur_char = $data[$cur_pos];
            if ($cur_char == '(') {
                list($cur_pos, $text) = self::parseParentheses($data, $cur_pos,
                    $encoding);
                $cur_pos --;
                $out .= $text;
            }
            $cur_pos++;
        }
        if (isset($data[$cur_pos]) && isset($data[$cur_pos + 1]) &&
            ord($data[$cur_pos]) == ord('T') &&
                ord($data[$cur_pos + 1]) == ord('J') ) {
            if (isset($data[$cur_pos + 3]) &&
                ord($data[$cur_pos + 3]) != ord('F')) {
                $out .= " ";
            } else {
                $out .= "\n";
            }
        }
        return [$cur_pos, $out];
    }
    /**
     * Extracts ASCII text till the next close parenthesis
     *
     * @param string $data source to extract character data from
     * @param int $cur_pos position to start in $data
     * @param string $encoding which of the default (if any) PDF encoding
     *    formats is being used: MacRomanEncoding, WinAnsiEncoding,
     *    PDFDocEncoding, etc.
     * @return array pair consisting of the final position in $data as well
     *     as extracted text
     */
    public static function parseParentheses($data, $cur_pos, $encoding)
    {
        $cur_pos++;
        $len = strlen($data);
        $out = "";
        $escape_flag =false;
        $cur_char = "";
        while($cur_pos < $len && ($cur_char != ")" || $escape_flag)) {
            $cur_char = $data[$cur_pos];
            if ($cur_char == '\\' && !$escape_flag) {
                $escape_flag = true;
            } else {
                if ($escape_flag || $cur_char !=")") {
                    $out .= self::convertChar($cur_char, $encoding);
                }
                $escape_flag = false;
            }
            $cur_pos++;
        }
        $check_positioning = substr($data, $cur_pos, 4);
        if (preg_match("/\-\d{3}/", $check_positioning) > 0 ) {
            $out .= " ";
        }
        return [$cur_pos, $out];
    }
    /**
     * Used to convert characters from one of the built in PDF
     * encodings to UTF-8
     * @param char $cur_char character to convert
     * @param string $encoding which of the default (if any) PDF encoding
     *    formats is being used: MacRomanEncoding, WinAnsiEncoding,
     *    PDFDocEncoding, etc.
     * @return string resultign converted string for character
     */
    public static function convertChar($cur_char, $encoding)
    {
        $ascii = ord($cur_char);
        if ((9 <= $ascii && $ascii <= 13) ||
            (32 <= $ascii && $ascii <= 126)) {
            return $cur_char;
        }
        if ($encoding == "MacRomanEncoding") {
            return match ($ascii) {
                190 => 'ae',
                198 => 'AE',
                206 => 'OE',
                207 => 'oe',
                222 => 'fi',
                223 => 'fl',
                default => "",
            };
        } else if ($encoding == "WinAnsiEncoding") {
            return match ($ascii) {
                140 => 'OE',
                156 => 'oe',
                198 => 'AE',
                230 => 'ae',
                default => "",
            };
        }
        return "";
    }
}
