_binary From 8a47eba397e9db7d5808d5d49be43abb73939d54 Mon Sep 17 00:00:00 2001
From: unknown <Nakul@.(none)>
Date: Mon, 8 Aug 2011 21:20:29 -0700
Subject: [PATCH 1/2] Added pptx processor

---
 configs/config.php                |    7 +-
 lib/processors/pptx_processor.php |  265 +++++++++++++++++++++++++++++++++++++
 tests/pptx_processor_test.php     |  128 ++++++++++++++++++
 3 files changed, 398 insertions(+), 2 deletions(-)
 create mode 100644 lib/processors/pptx_processor.php
 create mode 100644 tests/pptx_processor_test.php

diff --git a/configs/config.php b/configs/config.php
index f6d7c4a..e741b57 100755
--- a/configs/config.php
+++ b/configs/config.php
@@ -259,7 +259,8 @@ $INDEXED_FILE_TYPES =
             "tab",
             "tsv",
             "txt",
-            "xml");
+            "xml",
+            "pptx");
 
 /**
  * @global array filetypes which should be considered images
@@ -293,7 +294,9 @@ $PAGE_PROCESSORS = array(   "text/html" => "HtmlProcessor",
                             "image/gif" => "GifProcessor", 
                             "image/png" => "PngProcessor",
                             "image/bmp" => "BmpProcessor",
-                            "image/svg+xml"=> "SvgProcessor"
+                            "image/svg+xml"=> "SvgProcessor",
+                            "application/vnd.openxmlformats-officedocument.
+                                presentationml.presentation"=> "PptxProcessor"
 );
 
 $INDEXING_PLUGINS = array("recipe");
diff --git a/lib/processors/pptx_processor.php b/lib/processors/pptx_processor.php
new file mode 100644
index 0000000..ad430f6
--- /dev/null
+++ b/lib/processors/pptx_processor.php
@@ -0,0 +1,265 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009, 2010, 2011  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Nakul Natu nakul.natu@gmail.com
+ * @package seek_quarry
+ * @subpackage processor
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009, 2010, 2011
+ * @filesource
+ */
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/**
+ * Load base class, if needed.
+ */
+require_once BASE_DIR."/lib/processors/text_processor.php";
+/**
+ * Load so can parse urls
+ */
+require_once BASE_DIR."/lib/url_parser.php";
+
+/**
+ * Used to create crawl summary information
+ * for PPTX files
+ *
+ * @author Nakul Natu
+ * @package seek_quarry
+ * @subpackage processor
+ */
+class PptxProcessor extends TextProcessor
+{
+    /**
+     * Constant for maximum description length
+     * @var number
+     */
+    const MAX_DESCRIPTION_LEN = 2000;
+
+    /**
+     *  Used to extract the title, description and links from
+     *  a pptx file consisting of xml data.
+     *
+     *  @param string $page pptx(zip) contents
+     *  @param string $url the url where the page contents came from,
+     *     used to canonicalize relative links
+     *
+     *  @return array  a summary of the contents of the page
+     *
+     */
+    function process($page, $url)
+    {
+        $summary = NULL;
+        $sites = array();
+        // Create a temporary pptx file
+        $filename=CRAWL_DIR . "/pptx.zip";
+
+        file_put_contents($filename, $page);
+        // Open a zip archive
+        $zip = new ZipArchive;
+        if ($zip->open($filename) === TRUE) {
+            $buf= $zip->getFromName("docProps/core.xml");
+            if ($buf){
+                $dom = self::dom($buf);
+                if($dom !== false) {
+                // Get the title
+                    $summary[self::TITLE] = self::title($dom);
+                }
+            }
+            $buf= $zip->getFromName("docProps/app.xml");
+            if($buf){
+            // Get number of slides present
+                $dom = self::dom($buf);
+                $slides=self::slides($dom);
+            }
+
+            $summary[self::DESCRIPTION]="";
+            $summary[self::LINKS]=$sites;
+            for ($i = 1; $i <= $slides; $i++) {
+                $buf=$zip->getFromName("ppt/slides/slide" . $i . ".xml");
+                if($buf){
+                /* Get description , language and url links asociated
+                 * with each slide*/
+                    $dom = self::dom($buf);
+                    $desc=self::description($dom);
+
+                    if(strlen($summary[self::DESCRIPTION]) 
+                        < self::MAX_DESCRIPTION_LEN) { 
+                            $summary[self::DESCRIPTION]=
+                                $summary[self::DESCRIPTION].$desc;
+                    }
+                    $lang1=self::lang($dom);
+                    if($lang1){
+                        $summary[self::LANG]=$lang1;
+                    }
+                    $summary[self::LINKS]=array_merge($summary[self::LINKS]
+                        ,self::links($dom,$url));
+                }
+            }
+            // Close the zip
+            $zip->close();
+            // Delete zip from the directory
+            deleteFileOrDir($filename);
+        }
+        else{
+        // If not pptx then process it as a text file
+            $summary=parent::process($page, $url);
+        }
+
+        return $summary;
+    }
+
+    /**
+     * Returns up to MAX_LINK_PER_PAGE many links from the supplied
+     * dom object where links have been canonicalized according to
+     * the supplied $site information.
+     *
+     * @param object $dom   a document object with links on it
+     * @param string $site   a string containing a url
+     *
+     * @return array   links from the $dom object
+     */
+    static function links($dom, $site)
+    {
+        $sites = array();
+
+        $xpath = new DOMXPath($dom);
+        $paras = $xpath->evaluate("/p:sld//p:cSld//p:spTree//p:sp//
+            p:txBody//a:p//a:r//a:rPr//a:hlinkClick");
+
+        $i=0;
+
+        foreach($paras as $para) {
+            if($i < MAX_LINKS_PER_PAGE) {
+                $hlink=$para->parentNode->parentNode->
+                    getElementsByTagName("t")->item(0)->nodeValue;
+
+                $url=UrlParser::canonicalLink(
+                    $hlink, $site);
+                if(!UrlParser::checkRecursiveUrl($url)  &&
+                    strlen($url) < MAX_URL_LENGTH) {
+                    if(isset($sites[$url])) {
+                        $sites[$url] .=" ".$hlink;
+                    } else {
+                        $sites[$url] = $hlink;
+                    }
+                }
+            }
+            $i++;
+        }
+
+        return $sites;
+    }
+    /**
+     * Return a document object based on a string containing the contents of
+     * a web page
+     *
+     *  @param string $page   xml document
+     *
+     *  @return object  document object
+     */
+    static function dom($page)
+    {
+        $dom = new DOMDocument();
+
+        @$dom->loadXML($page);
+
+        return $dom;
+    }
+
+    /**
+     *  Returns powerpoint head title of a pptx based on its document object
+     *
+     *  @param object $dom   a document object to extract a title from.
+     *  @return string  a title of the page
+     *
+     */
+    static function title($dom)
+    {
+        $coreProperties=$dom->getElementsByTagName("coreProperties");
+        foreach ($coreProperties as $property) {
+            $titles=$property->getElementsByTagName("title");
+            $title=$titles->item(0)->nodeValue;
+            return $title;
+        }
+    }
+
+    /**
+     *  Returns number of slides of  pptx based on its document object
+     *
+     *  @param object $dom   a document object to extract a title from.
+     *  @return number  number of slides
+     *
+     */
+    static function slides($dom)
+    {
+        $properties=$dom->getElementsByTagName("Properties");
+        foreach($properties as $property){
+            $slides=$property->getElementsByTagName("Slides");
+            $number=$slides->item(0)->nodeValue;
+            return $number;
+        }
+    }
+
+    /** 
+     *  Determines the language of the xml document by looking at the
+     *  language attribute of a tag.
+     *
+     *  @param object $dom  a document object to check the language of
+     *
+     *  @return string language tag for guessed language
+     */
+    static function lang($dom)
+    {
+        $xpath = new DOMXPath($dom);
+
+        $languages = $xpath->evaluate("/p:sld//p:cSld//p:spTree//
+            p:sp//p:txBody//a:p//a:r//a:rPr");
+        foreach ($languages as $language){
+            return $language->getAttribute("lang");
+        }
+    }
+
+    /**
+     * Returns descriptive text concerning a pptx slide based on its document
+     * object
+     *
+     * @param object $dom   a document object to extract a description from.
+     * @return string a description of the slide
+     */
+    static function description($dom)
+    {
+        $xpath = new DOMXPath($dom);
+
+        $titles = $xpath->evaluate("/p:sld//p:cSld//p:spTree
+            //p:sp//p:txBody//a:p//a:r//a:t");
+        $description="";
+        foreach ($titles as $title){
+            $description=$description.$title->nodeValue;
+        }
+        return $description;
+    }
+
+}
+?>
\ No newline at end of file
diff --git a/tests/pptx_processor_test.php b/tests/pptx_processor_test.php
new file mode 100644
index 0000000..86a59bd
--- /dev/null
+++ b/tests/pptx_processor_test.php
@@ -0,0 +1,128 @@
+<?php
+/** 
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009, 2010, 2011  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Nakul Natu nakul.natu@gmail.com
+ * @package seek_quarry
+ * @subpackage test
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009, 2010, 2011
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** Load search engine-wide configuration file */
+require_once BASE_DIR.'/configs/config.php';
+foreach(glob(BASE_DIR."/lib/processors/*_processor.php") as $filename) { 
+    require_once $filename;
+}
+ 
+
+/**
+ * UnitTest for the PptxProcessor class. It is used to process 
+ * pptx files which are xml based zip format 
+ *
+ * @author Nakul Natu
+ * @package seek_quarry
+ * @subpackage test
+ */
+class PptxProcessorTest extends UnitTest implements CrawlConstants
+{
+    /**
+     *  Creates a summary of pptx document to check
+     */
+    public function setUp()
+    {
+        $processors="PptxProcessor";
+        $processor= new $processors();
+        $filename=BASE_DIR . "/PC54.pptx";
+        $page=file_get_contents($filename);
+        $url="http://mschnlnine.vo.llnwd.net/d1/pdc08/PPTX/PC54.pptx";
+        $summary=array();
+        $summary=$processor->process($page,$url);
+        $this->test_objects['summary'] = $summary;
+    }
+
+    /**
+     * Test object is set to null
+     */
+    public function tearDown()
+    {
+        $this->test_objects=null;
+    }
+    /**
+     * Checks title of the pptx is correct or not 
+     */
+    public function checkTitleTestCase()
+    {
+        $objects = $this->test_objects['summary'];
+        $title="PC54: Mono And .NET";
+        $this->assertEqual
+            ($objects[self::TITLE], $title,"Correct Title Retrived");
+    }
+
+    /**
+     * Checks Language of pptx is correct or not 
+     */
+    public function checkLangTestCase()
+    {
+        $objects = $this->test_objects['summary'];
+        $lang="en-US";
+        $this->assertEqual(
+            $objects[self::LANG], $lang,"Correct Language Retrived");
+    }
+
+    /**
+     * Checks the links are correct or not 
+     */
+    public function checkLinksTestCase()
+    {
+        $objects = $this->test_objects['summary'];
+        $testLinks=array();
+        $testLinks[0]="http://www.mono-project.com";
+        $testLinks[1]="http://www.mono-project.com/Start";
+        $testLinks[2]="http://www.go-mono.com/monologue";
+        $testLinks[3]="http://tirania.org/blog";
+        $links=array();
+        $links=$objects[self::LINKS];
+        $i=0;
+        foreach ($links as $link){
+            $this->assertEqual(
+                $link, $testLinks[$i],"Correct Link Retrived");
+            $i++;
+        }
+    }
+
+    /**
+     * Checks if description is not null
+     */
+    Public function checkDescriptionTestCase()
+    {
+        $objects = $this->test_objects['summary'];
+        $this->assertTrue(
+            isset($objects[self::DESCRIPTION]),"Description is not null");
+    }
+}
+?>
\ No newline at end of file
-- 
1.7.6.msysgit.0


From 785a825c225b0a449a3221eecd67c26efb8f37b9 Mon Sep 17 00:00:00 2001
From: unknown <Nakul@.(none)>
Date: Mon, 8 Aug 2011 21:22:34 -0700
Subject: [PATCH 2/2] Added pptx processor

---
 PC54.pptx |  Bin 0 -> 470658 bytes
 1 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 PC54.pptx

diff --git a/PC54.pptx b/PC54.pptx
new file mode 100644
index 0000000000000000000000000000000000000000..af6813555f73fab0e6f9f5b20476854c2206de70
GIT binary patch
-- 
1.7.6.msysgit.0

