_binary From e06b232e9b567c3042528aa4d3997353653ee276 Mon Sep 17 00:00:00 2001
From: Vijeth <vijeth.patil@gmail.com>
Date: Mon, 8 Aug 2011 07:19:23 -0700
Subject: [PATCH] added epub processor and test class changes made in config file accordingly

---
 configs/config.php                |    4 +-
 lib/processors/epub_processor.php |  211 +++++++++++++++++++++++++++++++++++++
 tests/epub_processor_test.php     |  133 +++++++++++++++++++++++
 3 files changed, 347 insertions(+), 1 deletions(-)
 create mode 100644 lib/processors/epub_processor.php
 create mode 100644 tests/epub_processor_test.php

diff --git a/configs/config.php b/configs/config.php
index f6d7c4a..0a94bc2 100755
--- a/configs/config.php
+++ b/configs/config.php
@@ -241,6 +241,7 @@ $INDEXED_FILE_TYPES =
             "cfml",
             "csv",
             "doc",
+            "epub",
             "gif",
             "html",
             "htm",
@@ -283,7 +284,8 @@ $PAGE_PROCESSORS = array(   "text/html" => "HtmlProcessor",
 
                             "application/msword" => "DocProcessor",
                             "application/vnd.ms-powerpoint" => "PptProcessor",
-
+                            "application/epub+zip" => "EpubProcessor",
+                            
                             "text/rtf" => "RtfProcessor",  
                             "text/plain" => "TextProcessor", 
                             "text/csv" => "TextProcessor",
diff --git a/lib/processors/epub_processor.php b/lib/processors/epub_processor.php
new file mode 100644
index 0000000..0de5967
--- /dev/null
+++ b/lib/processors/epub_processor.php
@@ -0,0 +1,211 @@
+<?php
+/** 
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009, 2010, 2011  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Vijeth Patil vijeth.patil@gmail.com
+ * @package seek_quarry
+ * @subpackage processor
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009, 2010, 2011
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/**
+ * Load base class, if needed.
+ */
+require_once BASE_DIR."/lib/processors/text_processor.php";
+
+/**
+ * If XML turns out to be RSS ...
+ */
+require_once BASE_DIR."/lib/processors/rss_processor.php";
+
+/**
+ * If XML turns out to be XHTML ...
+ */
+require_once BASE_DIR."/lib/processors/html_processor.php";
+
+/**
+ * If XML turns out to be OPF
+ */
+//require_once BASE_DIR."/lib/processors/epub_opf_filehandler.php";
+
+/**
+ * If XML turns out to be XML/XHTML
+ */
+//require_once BASE_DIR."/lib/processors/epub_xhtml_filehandler.php";
+
+/**
+ * Load so can parse urls
+ */
+require_once BASE_DIR."/lib/url_parser.php";
+
+ /**
+ * Used to create crawl summary information 
+ * for XML files (those served as application/epub+zip)
+ *
+ * @author Vijeth Patil
+ * @package seek_quarry
+ * @subpackage processor
+ */
+
+class EpubProcessor extends TextProcessor
+{
+    const MAX_DESCRIPTION_LEN = 2000;
+    
+    var $name;
+    var $attributes;    
+    var $content;
+    var $children;
+
+    /**
+     *  Used to extract the title, description and links from
+     *  a string consisting of ebook publication data.
+     *
+     *  @param string $page epub contents
+     *  @param string $url the url where the page contents came from,
+     *     used to canonicalize relative links
+     *
+     *  @return array  a summary of the contents of the page
+     *
+     */ 
+    function process($page, $url)
+    { 
+        $summary       = NULL;
+        $opf_pattern   = "/.opf$/i";
+        $html_pattern  = "/.html$/i";
+        $xhtml_pattern = "/.xhtml$/i";
+        $temp_filename = "epubzipfilename.zip";
+        $epub_url      = 0;
+        $epub_language = '';
+        file_put_contents($temp_filename,$page);
+        $zip = new ZipArchive;        
+        if ($zip->open($temp_filename))
+        {    
+            for($i = 0; $i < $zip->numFiles; $i++)
+            {
+                // get the content file names of .epub document
+                $filename[$i] = $zip->getNameIndex($i) ;
+                if(preg_match($opf_pattern,$filename[$i]))
+                {
+                    // Get the file data from zipped folder
+                    $opf_data = $zip->getFromName($filename[$i]);
+                    $opf_summary = $this->xml_to_object($opf_data);
+                    for($m = 0;$m <= 10;$m++)
+                    {
+                        for($n = 0;$n <= 10;$n++)
+                        {
+                            if(($opf_summary->children[$m]->children[$n]->
+                                name) == "dc:language")
+                            {
+                                $epub_language = $opf_summary->children[$m]->
+                                children[$n]->content ;
+                            }
+                            if( ($opf_summary->children[$m]->children[$n]->
+                                name) == "dc:title")
+                            {
+                                $epub_title = $opf_summary->children[$m]->
+                                children[$n]->content ;
+                            }
+                            if( ($opf_summary->children[$m]->children[$n]->
+                                name) == "dc:creator")
+                            {
+                                $epub_author = $opf_summary->children[$m]->
+                                children[$n]->content ;
+                            }
+                            if( ($opf_summary->children[$m]->children[$n]->
+                                name) == "dc:identifier")
+                            {
+                                $epub_unique_identifier = $opf_summary->
+                                children[$m]->children[$n]->content ;
+                            }
+                        }
+                     }
+                }else if((preg_match($html_pattern,$filename[$i])) ||
+                         (preg_match($xhtml_pattern,$filename[$i])))
+                {
+                    $html = new HtmlProcessor;
+                    $html_data = $zip->getFromName($filename[$i]);
+                    $description[$i] = $html->process($html_data,$url);
+                }
+            }
+        }
+        $summary[self::TITLE] = $epub_title;
+        $summary[self::DESCRIPTION] = $description;      
+        $summary[self::LANG] = $epub_language;
+        $summary[self::LINKS] = $epub_url;
+        $summary[self::PAGE] = $page;
+        unlink($temp_filename);    
+        return $summary;
+    }
+
+    /**
+     *  Used to extract the title, author, language and links from
+     *  a string consisting of ebook publication data.
+     *
+     *  @param string $page epub contents
+     *  @param string $url the url where the page contents came from,
+     *     used to canonicalize relative links
+     *
+     *  @return array  an information about the contents of the page
+     *
+     */ 
+    function xml_to_object($xml)
+    {
+        $parser = xml_parser_create();
+        xml_parser_set_option($parser, XML_OPTION_CASE_FOLDING, 0);
+        xml_parser_set_option($parser, XML_OPTION_SKIP_WHITE, 1);
+        xml_parse_into_struct($parser, $xml, $tags);
+        xml_parser_free($parser);
+
+        $elements = array();  // the currently filling [child] XmlElement array
+        $stack = array();
+        foreach ($tags as $tag)
+        {
+            $index = count($elements);
+            if ($tag['type'] == "complete" || $tag['type'] == "open")
+            {
+                $elements[$index] = new EpubProcessor;
+                $elements[$index]->name = $tag['tag'];
+                $elements[$index]->attributes = $tag['attributes'];
+                $elements[$index]->content = $tag['value'];
+                if ($tag['type'] == "open")
+                {  // push
+                    $elements[$index]->children = array();
+                    $stack[count($stack)] = &$elements;
+                    $elements = &$elements[$index]->children;
+                }
+            }
+            if ($tag['type'] == "close")
+            {  // pop
+                $elements = &$stack[count($stack) - 1];
+                unset($stack[count($stack) - 1]);
+            }
+        }
+        return $elements[0];  // the single top-level element
+    }
+}
+?>
diff --git a/tests/epub_processor_test.php b/tests/epub_processor_test.php
new file mode 100644
index 0000000..5615c6a
--- /dev/null
+++ b/tests/epub_processor_test.php
@@ -0,0 +1,133 @@
+<?php
+/** 
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009, 2010, 2011  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Vijeth Patil vijeth.patil@gmail.com
+ * @package seek_quarry
+ * @subpackage test
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009, 2010, 2011
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/**
+ *  Load search engine-wide configuration file 
+ */
+require_once BASE_DIR.'/configs/config.php';
+
+/** 
+ *  Load the EpubProcessor class we are going to test
+ */
+require_once BASE_DIR."/lib/processors/epub_processor.php"; 
+
+/**
+ *  Load the base unit test class
+ */
+require_once BASE_DIR."/lib/unit_test.php";
+
+/**
+ * Load the Crawl constants required for the summary
+ */
+ require_once BASE_DIR."/lib/crawl_constants.php";
+/**
+ * UnitTest for the EpubProcessor class. An EpubProcessor is used to process 
+ * a .epub (ebook publishing standard) file and extract summary from it. This 
+ * class tests the processing of an .epub file format by EpubProcessor.
+ * 
+ *
+ * @author Vijeth Patil
+ * @package seek_quarry
+ * @subpackage test
+ */
+ 
+class EpubProcessorTest extends UnitTest implements CrawlConstants
+{
+    /**
+     * Creates a new EpubProcessor object so that
+     * we can process an .epub format file.
+     */
+     
+    public function setUp()
+    {
+        $epub_object = new EpubProcessor;
+        $url = "http://www.epubbooks.com/book/582/conrad-almayers-folly.epub";
+        $filename="conrad-almayers-folly.epub";
+        $page = file_get_contents($filename);
+        $summary=$epub_object->process($page,$url);
+        $this->test_objects['summary'] = $summary;
+        $this->testEpubTitleTestCase();
+        $this->testEpubLangTestCase();
+        $this->testEpubDescriptionTestCase();
+    }
+      
+    /**
+     * Delete any files associated with our test on EpubProcessor
+     */
+    public function tearDown()
+    {
+        @unlink("");
+    }
+    
+    /**
+     * Test case to check whether the title of the epub document
+     * is retrieved correctly.
+     */ 
+    public function testEpubTitleTestCase()
+    {
+        $m = $this->test_objects['summary'] ;
+        $x = $m[self::TITLE];
+        $correct_title = "Almayer's Folly";
+        $description = "Test Passed with correct title";
+        $this->assertEqual($x,$correct_title,$description);
+    }
+    
+    /**
+     * Test case to check whether the language of the document is 
+     * retrieved correctly.
+     */
+    public function testEpubLangTestCase()
+    {
+        $m = $this->test_objects['summary'] ;
+        $x = $m[self::LANG];
+        $correct_language = "en-gb";
+        $description = "Test Passed with correct Language";
+        $this->assertEqual($x,$correct_language,$description);       
+    }
+    
+    /**
+     * Test case to check whether the description of the document is 
+     * not empty.
+     */
+    public function testEpubDescriptionTestCase()
+    {
+        $m = $this->test_objects['summary'] ;
+        $x = $m[self::DESCRIPTION];
+        $description = "Test Passed with Description information not empty";
+        $this->assertTrue($x,$description);     
+    }
+    
+       
+ }
-- 
1.7.4.1

