_binary From 99ef30887e3efabcfaeda39476f201c29716de1e Mon Sep 17 00:00:00 2001
From: unknown <tanmayee.4170@gmail.com>
Date: Sun, 9 Dec 2012 23:33:09 -0800
Subject: [PATCH] Invisible_Web_Patch

---
 controllers/admin_controller.php                   |  54 +++
 css/search.css                                     |  87 ++++
 .../database_archive_bundle_iterator.php           | 375 ++++++++++++++++++
 .../log_archive_bundle_iterator.php                | 438 +++++++++++++++++++++
 lib/crawl_constants.php                            |   3 +
 views/elements/crawloptions_element.php            | 240 +++++++++++
 6 files changed, 1197 insertions(+)
 create mode 100644 lib/archive_bundle_iterators/database_archive_bundle_iterator.php
 create mode 100644 lib/archive_bundle_iterators/log_archive_bundle_iterator.php

diff --git a/controllers/admin_controller.php b/controllers/admin_controller.php
index f04c9ce..7710536 100755
--- a/controllers/admin_controller.php
+++ b/controllers/admin_controller.php
@@ -788,6 +788,16 @@ class AdminController extends Controller implements CrawlConstants
                     $crawl_params[self::META_WORDS] = 
                         isset($seed_info['meta_words']) ?
                         $seed_info['meta_words'] : array();
+
+                    $crawl_params[self::LOG_RECORDS] = 
+                        isset($seed_info['log_records']) ?
+                        $seed_info['log_records'] : array();
+
+                    $crawl_params[self::DATABASE_CONNECTION_DETAILS] = 
+                        isset($seed_info['database_connection_details']) ?
+                        $seed_info['database_connection_details'] : array();
+
+
                     if(isset($seed_info['indexing_plugins']['plugins'])) {
                         $crawl_params[self::INDEXING_PLUGINS] =
                             $seed_info['indexing_plugins']['plugins'];
@@ -1058,6 +1068,50 @@ class AdminController extends Controller implements CrawlConstants
                     } else if(isset($seed_info['meta_words'])){
                             $data['META_WORDS'] = $seed_info['meta_words'];
                     }
+
+                    $data['LOG_RECORDS'] = array();
+                    if(!$no_further_changes) {
+                        if(isset($_REQUEST["LOG_RECORDS"])){
+                            foreach($_REQUEST["LOG_RECORDS"] as $triplet) {
+                                list($field, $field_name,$field_type) = 
+                                                         array_values($triplet);
+                                $field = $this->clean($field, "string");
+                                $field_name = 
+                                          $this->clean($field_name, "string");
+                                $field_type = 
+                                           $this->clean($field_type,"string");
+                                $field_nt = $field_name."::".$field_type;
+                                if(trim($field) != "" &&trim($field_nt) !=""){
+                                      $data['LOG_RECORDS'][$field] = $field_nt;
+                                }
+                            }
+                            $seed_info['log_records'] = $data['LOG_RECORDS'];
+                            $update_flag = true;
+                        } else if(isset($seed_info['log_records'])){
+                            $data['LOG_RECORDS'] = $seed_info['log_records'];
+                        }
+                    } else if(isset($seed_info['log_records'])){
+                            $data['LOG_RECORDS'] = $seed_info['log_records'];
+                    }
+
+                    $data['DATABASE_CONNECTION_DETAILS'] = array();
+                    if(!$no_further_changes) {
+                        if(isset($_REQUEST["DATABASE_CONNECTION_DETAILS"])){
+                            $data['DATABASE_CONNECTION_DETAILS']=
+                                       $_REQUEST["DATABASE_CONNECTION_DETAILS"];
+                            $seed_info['database_connection_details'] = 
+                                           $data['DATABASE_CONNECTION_DETAILS'];
+                            $update_flag = true;
+                        } else if(isset(
+                               $seed_info['database_connection_details'])){
+                            $data['DATABASE_CONNECTION_DETAILS'] = 
+                                     $seed_info['database_connection_details'];
+                        }
+                    } else if(isset($seed_info['database_connection_details'])){
+                            $data['DATABASE_CONNECTION_DETAILS'] = 
+                                     $seed_info['database_connection_details'];
+                    }
+
                     $data['INDEXING_PLUGINS'] = array();
                     $included_plugins = array();
                     if(!$no_further_changes && isset($_REQUEST["posted"])) {
diff --git a/css/search.css b/css/search.css
index 0e44976..491ad4e 100755
--- a/css/search.css
+++ b/css/search.css
@@ -1711,6 +1711,93 @@ ul.in-list li
     width: 97%;
 }
 
+.logrecordstable
+{
+   width:100%;
+}
+
+.logrecordstable,
+.logrecordstable th,
+.logrecordstable td
+{
+    border: 1px ridge black;
+}
+
+.logrecordstable th
+{
+    text-align: center;
+    padding: .03in;
+}
+.logrecordstable td.input-field
+{
+    width: 1.3in;
+}
+.logrecordstable td.input-field input
+{
+    width: 1.5in;
+    margin:.05in;
+}
+.logrecordstable td.input-field-name
+{
+    margin:.03in;
+    width: 100%;
+}
+.logrecordstable td.input-field-name input
+{
+    margin:.05in;
+    width: 97%;
+}
+.logrecordstable td.input-field-type
+{
+    margin:.03in;
+    width: 100%;
+}
+.logrecordstable td.input-field-type input
+{
+    margin:.05in;
+    width: 97%;
+}
+.logrecordnewfield
+{
+    text-align: right;
+}
+
+.databaseconnectiondetailstable
+{
+   width:100%;
+}
+
+.databaseconnectiondetailstable,
+.databaseconnectiondetailstable td
+{
+    border: 1px ridge black;
+}
+
+.databaseconnectiondetailstable td.input-name
+{
+    width: 1.3in;
+}
+.databaseconnectiondetailstable td.input-data
+{
+    margin:.03in;
+    width: 100%;
+}
+.databaseconnectiondetailstable td.input-data input
+{
+    margin:.05in;
+    width: 98%;
+}
+.databaseconnectiondetailssubmit
+{
+    text-align: right;
+}
+
+.databaseresultstable
+{
+    width:100%;
+}
+
+
 .indexingplugintable
 {
    width:100%;
diff --git a/lib/archive_bundle_iterators/database_archive_bundle_iterator.php b/lib/archive_bundle_iterators/database_archive_bundle_iterator.php
new file mode 100644
index 0000000..214294a
--- /dev/null
+++ b/lib/archive_bundle_iterators/database_archive_bundle_iterator.php
@@ -0,0 +1,375 @@
+<?php
+/** 
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2012  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2012
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** 
+ *Loads base class for iterating
+ */
+require_once BASE_DIR.
+    '/lib/archive_bundle_iterators/archive_bundle_iterator.php';
+
+/**
+ * Used to iterate through the records of a database stored in a 
+ * DatabaseArchiveBundle folder. Database is a collection of tables with 
+ * various rows in each table. Iteration would be for the purpose of making 
+ * an index of each of these records.
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @see WebArchiveBundle
+ */
+class DatabaseArchiveBundleIterator extends ArchiveBundleIterator 
+    implements CrawlConstants
+{
+    /**
+     * The path to the directory containing the archive partitions to be 
+     * iterated over.
+     * @var string
+     */
+    var $iterate_dir;
+    /**
+     * The path to the directory where the iteration status is stored.
+     * @var string
+     */
+    var $result_dir;
+    /**
+     * The path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $index_dir;
+
+    /**
+     * The part of the path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $path_for_html_files;
+    /**
+     *  current number of record in the current database 
+     *  @var int
+     */
+    var $current_page_num;
+    /**
+     * The number of database records in this database archive bundle
+     *  @var int
+     */
+    var $num_of_records;
+    /**
+     *  Array of database records according to the query specified by the user
+     *  @var array
+     */
+    var $records;
+    /**
+     *  Array of fields of database record specified by the user
+     *  @var array
+     */
+    var $fields;
+    /**
+     *  Array of field names in database record specified by the user
+     *  @var array
+     */
+    var $field_names;
+    /**
+     *  Array of database fieldtypes specified by the user
+     *  @var array
+     */
+    var $field_types;
+    /**
+     *  Database handle for a database
+     *  @var resource
+     */
+    var $db_handle;
+    /**
+     *  Whether database exists or not
+     *  @var resource
+     */
+    var $db_found;
+    /**
+     *  Array of database connection details to connect to database
+     *  @var array
+     */
+    var $databaseConnectionsArray; 
+    /**
+     *  Host name for the database
+     *  @var string
+     */
+    var $host; 
+    /**
+     *  User Name for the localhost
+     *  @var string
+     */
+    var $user_name;
+    /**
+     *  Password for the connection
+     *  @var string
+     */
+    var $password;
+    /**
+     *  Name of the database for the records
+     *  @var string
+     */
+    var $database;
+    /**
+     *  Query to retrieve the records to be indexed from the database 
+     *  @var array
+     */
+    var $query;
+    /**
+     *  Array of database fields type in drop down box in the UI
+     *  @var array
+     */
+    var $databasefields_type_ddm = array(1=>'Int',
+                                         2=>'String',
+                                         3=>'Date',
+                                         4=>'Full Text Index');
+    
+    /**
+     * Creates a database archive iterator with the given parameters.
+     *
+     * @param string $iterate_timestamp timestamp of the arc archive bundle to 
+     *      iterate  over the pages of
+     * @param string $result_timestamp timestamp of the arc archive bundle
+     *      results are being stored in
+     */
+    function __construct($iterate_timestamp, $iterate_dir,
+        $result_timestamp, $result_dir)
+    {
+        $this->iterate_timestamp = $iterate_timestamp;
+        $this->iterate_dir = $iterate_dir;
+        $this->result_timestamp = $result_timestamp;
+        $this->result_dir = $result_dir;
+        $this->index_dir = CRAWL_DIR."/cache/IndexData".$this->result_timestamp;
+        $temp_array = explode("/", $this->index_dir);
+
+        for($i = 1;$i<count($temp_array);$i++){
+                if($temp_array[$i-1]=="htdocs"){
+                        while($temp_array[$i] != "cache"){
+                                 $this->path_for_html_files .= $temp_array[$i]
+."/";
+                                 $i++;
+                        }
+                        break;
+                }
+        }
+        if(file_exists("{$this->iterate_dir}/database_connection_details.txt"))
+        {
+                $database_connection_details_info = unserialize
+(file_get_contents("{$this->iterate_dir}/database_connection_details.txt"));
+                file_put_contents(
+"{$this->index_dir}/database_connection_details".$this->result_timestamp.".txt",
+serialize($database_connection_details_info));
+                @unlink("{$this->iterate_dir}/database_connection_details.txt");
+        }
+        if(!is_dir("{$this->index_dir}/htmlFiles")){
+                mkdir("{$this->index_dir}/htmlFiles");
+        }
+        $this->createRecords();
+
+        if(file_exists("{$this->result_dir}/iterate_status.txt")) {
+            $this->restoreCheckpoint();
+        }
+        else {
+            $this->reset();
+        }
+    }
+
+    /**
+     * Estimates the important of the site according to the weighting of
+     * the particular archive iterator
+     * @param $site an associative array containing info about a web page
+     * @return bool false we assume arc files were crawled according to 
+     *      OPIC and so we use the default doc_depth to estimate page importance
+     */
+    function weight(&$site) 
+    {
+        return false;
+    }
+
+    /**
+     * Resets the iterator to the start of the archive bundle
+     */
+    function reset()
+    {
+        $this->current_page_num = -1;
+        $this->end_of_iterator = false;
+        @unlink("{$this->result_dir}/iterate_status.txt");
+    }
+
+    /**
+     * Saves the current state so that a new instantiation can pick up just 
+     * after the last batch of pages extracted.
+     */
+    function saveCheckpoint($info = array())
+    {
+        $info['end_of_iterator'] = $this->end_of_iterator;
+        $info['current_page_num'] = $this->current_page_num;
+        $info['database_iterator'] = $this->database_iterator;
+        file_put_contents("{$this->result_dir}/iterate_status.txt",
+            serialize($info));
+    }
+
+    /**
+     * Restores state from a previous instantiation, after the last batch of 
+     * pages extracted.
+     */
+    function restoreCheckpoint()
+    {
+        $info = unserialize(file_get_contents(
+            "{$this->result_dir}/iterate_status.txt"));
+        $this->end_of_iterator = $info['end_of_iterator'];
+        $this->current_page_num = $info['current_page_num'];
+        $this->database_iterator = $info['database_iterator'];
+        return $info;
+    }
+
+    /**
+     * Creates Records array containing all the records to satisfying the query
+     */
+    function createRecords()
+    {
+        $this->databaseConnectionsArray = unserialize(file_get_contents(
+"{$this->index_dir}/database_connection_details".$this->result_timestamp
+.".txt"));
+        $this->host = $this->databaseConnectionsArray['HOSTNAME'];
+        $this->user_name = $this->databaseConnectionsArray['USERNAME'];
+        $this->password = $this->databaseConnectionsArray['PASSWORD'];
+        $this->database = $this->databaseConnectionsArray['DATABASENAME'];
+        $this->query = $this->databaseConnectionsArray['QUERY'];
+        $this->db_handle = mysql_connect($this->host, $this->user_name,
+ $this->password);
+        $this->db_found = mysql_select_db($this->database, $this->db_handle);
+
+        /*If database exists*/
+        if ($this->db_found) {
+                $result1 = mysql_query($this->query);
+                $num_fields = mysql_num_fields($result1);
+                for($i=0;$i<$num_fields;$i++){
+                        $this->field_names[$i] = mysql_field_name($result1, $i);
+                }
+                while ($row = mysql_fetch_row($result1)) {
+                        $this->records[] = $row;
+                }
+                $this->num_of_records = count($this->records);
+
+                mysql_free_result($result1);
+                mysql_close($db_handle);
+        }
+    }
+    /**
+     * Gets the next at most $num many records from the iterator. It might 
+     * return less than $num many documents if the end of the bundle is reached.
+     * @param int $num number of docs to get
+     * @return array associative arrays for $num pages
+     */
+    function nextPages($num)
+    {
+        $pages = array();
+        for($i = 0; $i < $num; $i++) {
+            $this->current_page_num++;
+            $page = $this->nextPage();
+            if($this->current_page_num >= $this->num_of_records) {
+                $this->end_of_iterator = true;
+                break;
+            }
+            else {
+                $pages[] = $page;
+            }
+        }
+     
+        $this->saveCheckpoint();
+        return $pages;
+    }
+
+    
+    /**
+     * Gets the next record from the iterator
+     * @return array associative array for record
+     */
+    function nextPage()
+    {
+        file_put_contents("{$this->iterate_dir}/test.txt","hello");
+        $site = array();
+        $field_nd = "";
+        $html_page = "";
+        $temp_record = array();
+        $temp_record = $this->records[$this->current_page_num];
+        $dom = new DOMDocument('1.0');
+        $root =$dom->createElement('html');
+        $root = $dom->appendChild($root);
+        $head = $dom->createElement('head');
+        $head = $root->appendChild($head);
+        $title = $dom->createElement('title');
+        $title = $head->appendChild($title);
+        $recordTitle = "Database Record".$this->current_page_num;
+        $text = $dom->createTextNode($recordTitle);
+        $text = $title->appendChild($text);
+        $body = $dom->createElement('body');
+        $body = $root->appendChild($body);
+        $field = $dom->createElement('p');
+        $field = $body->appendChild($field);
+        for($i=0;$i<count($this->field_names);$i++){
+                $field_nd .= $this->field_names[$i].":".$temp_record[$i]."<br>";
+        }
+        $text1 = $dom->createTextNode($field_nd);
+        $text1 = $field->appendChild($text1);
+        $desc = $dom->createElement('p');
+        $desc = $body->appendChild($desc);
+        $text3 = "This is database record ".$this->current_page_num;
+        $text2 = $dom->createTextNode($text3);
+        $text2 = $desc->appendChild($text2);
+        $site[self::PAGE] =$dom->saveHTML();
+        $html_page = "<html><head><title>DatabaseRecord".
+$this->current_page_num."</title></head><body><h1>The details of the 
+database record are: <br></h1><h3>".$field_nd."</h3></body></html>";
+        file_put_contents("{$this->index_dir}/htmlFiles/databaserecord".
+$this->current_page_num.".php",$html_page);
+        $site[self::URL] = "http://localhost/".$this->path_for_html_files.
+"cache/IndexData".$this->result_timestamp."/htmlFiles/databaserecord"
+.$this->current_page_num.".php";
+        $site[self::TYPE] ="text/html";
+        $site[self::HTTP_CODE] = 200;
+        $site[self::ENCODING] = "UTF-8";
+        $site[self::SERVER] = "unknown";
+        $site[self::SERVER_VERSION] = "unknown";
+        $site[self::OPERATING_SYSTEM] = "unknown";
+        $site[self::HASH] = FetchUrl::computePageHash($site[self::PAGE]);
+        $site[self::WEIGHT] = 1;
+
+        return $site;
+        
+    }
+}
+?>
diff --git a/lib/archive_bundle_iterators/log_archive_bundle_iterator.php b/lib/archive_bundle_iterators/log_archive_bundle_iterator.php
new file mode 100644
index 0000000..20e1ac6
--- /dev/null
+++ b/lib/archive_bundle_iterators/log_archive_bundle_iterator.php
@@ -0,0 +1,438 @@
+<?php
+/** 
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2012  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2012
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** 
+ *Loads base class for iterating
+ */
+require_once BASE_DIR.
+    '/lib/archive_bundle_iterators/archive_bundle_iterator.php';
+
+/**
+ * Used to iterate through the collection of log files stored in
+ * a WebArchiveBundle folder. Log is the file format which has the 
+ * activities the system or a server performs. Iteration would be 
+ * for the purpose making an index of these files.
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @see WebArchiveBundle
+ */
+
+class LogArchiveBundleIterator extends ArchiveBundleIterator 
+    implements CrawlConstants
+{
+    /**
+     * The path to the directory containing the archive partitions to be 
+     * iterated over.
+     * @var string
+     */
+    var $iterate_dir;
+
+    /**
+     * The path to the directory where the iteration status is stored.
+     * @var string
+     */
+    var $result_dir;
+
+    /**
+     * The path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $index_dir;
+
+    /**
+     * The part of the path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $path_for_html_files;
+
+    /**
+     * The number of log files in this log archive bundle
+     *  @var int
+     */
+    var $num_partitions;
+
+    /**
+     *  current record number in the master log file
+     *  @var int
+     */
+    var $current_page_num;
+
+    /**
+     *  number of records in the master log file
+     *  @var int
+     */
+    var $num_of_records;
+
+    /**
+     *  Array of log records in the master log file in the directory
+     *  @var array
+     */
+    var $records;
+
+    /**
+     *  Array of filenames of log files in this directory (glob order)
+     *  @var array
+     */
+    var $partitions;
+    
+    /**
+     *  Array of fields of log file specified by the user
+     *  @var array
+     */
+    var $fields;
+
+    /**
+     *  Array of fieldnames specified by the user
+     *  @var array
+     */
+    var $field_names;
+    
+    /**
+     *  Array of fieldtypes specified by the user
+     *  @var array
+     */
+    var $field_types;
+
+    /**
+     *  Array of fields in each record separately
+     *  @var array
+     */
+    var $page_info;
+
+    /**
+     *  Array of regular expressions for all the data types
+     *  @var array
+     */
+    var $regular_exprs = array( 
+'IP_Address'=>'/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/',
+'Timestamp'=>'/\[[^:]+:\d+:\d+:\d+ [^\]]+\]/',
+'Request'=> '/(GET|HEAD|POST|PUT|DELETE|TRACE|OPTIONS|CONNECT)+[^"]*/',
+'Status Code'=>'/\s[1-5]\d{2}\s/',
+'Int'=>'/\s[0-9]+\s/',
+'URL'=> '/(http|https|ftp):\/\/[A-Za-z0-9][A-Za-z0-9_-]*[\/]*(?:.[A-Za-z0-9][A-Za-z0-9_-]*[\/]*)+:?(d*)[\/]*/',
+'User Agent'=>'/"([a-zA-Z0-9][^"]+)"/');
+
+    /**
+     *  Array of log fields type in drop down box in the UI
+     *  @var array
+     */
+    var $logfields_type_ddm = array(1=>'IP_Address',
+                                    2=>'Timestamp',
+                                    3=>'URL',
+                                    4=>'Status Code',
+                                    5=>'User Agent',
+                                    6=>'Request',
+                                    7=>'Int');
+
+    /**
+     * Creates a log archive iterator with the given parameters.
+     *
+     * @param string $iterate_timestamp timestamp of the log archive bundle to 
+     *      iterate  over the pages of
+     * @param string $result_timestamp timestamp of the log archive bundle
+     *      results are being stored in
+     */
+    function __construct($iterate_timestamp, $iterate_dir,
+        $result_timestamp, $result_dir)
+    {
+        $this->path_for_html_files = "";
+        $this->iterate_timestamp = $iterate_timestamp;
+        $this->iterate_dir = $iterate_dir;
+        $this->result_timestamp = $result_timestamp;
+        $this->result_dir = $result_dir;
+        $this->index_dir = CRAWL_DIR."/cache/IndexData".$this->result_timestamp;
+        $temp_array = explode("/", $this->index_dir);
+
+        for($i = 1;$i<count($temp_array);$i++){
+                if($temp_array[$i-1]=="htdocs"){
+                        while($temp_array[$i] != "cache"){
+                                 $this->path_for_html_files 
+.= $temp_array[$i]."/";
+                                 $i++;
+                        }
+                        break;
+                }
+        }
+        if(file_exists("{$this->iterate_dir}/fields_data.txt"))
+        {
+                $fields_data = unserialize(file_get_contents(
+"{$this->iterate_dir}/fields_data.txt"));
+                file_put_contents(
+"{$this->index_dir}/fields_data".$this->result_timestamp.".txt",
+serialize($fields_data));
+                @unlink("{$this->iterate_dir}/fields_data.txt");
+        }
+        if(!is_dir("{$this->index_dir}/htmlFiles")){
+                mkdir("{$this->index_dir}/htmlFiles");
+        }
+        $this->partitions = array();
+        foreach(glob("{$this->iterate_dir}/*.log") as $filename) { 
+            if(strpos($filename,"master.log")!= true){
+                 $this->partitions[] = $filename;
+            }
+        }
+        $this->num_partitions = count($this->partitions);
+        $this->records = $this->createMasterAndRecords();
+        $this->num_of_records = count($this->records);
+        if(file_exists("{$this->result_dir}/iterate_status.txt")){
+            $this->restoreCheckpoint();
+        } 
+        else {
+            $this->reset();
+        }
+    }
+
+    /**
+     * Estimates the important of the site according to the weighting of
+     * the particular archive iterator
+     * @param $site an associative array containing info about a web page
+     * @return value 1 we assume all log files crawled have the same 
+     *  page importance
+     */
+    function weight(&$site) 
+    {
+        return 1;
+    }
+
+    /**
+     * Resets the iterator to the start of the archive bundle
+     */
+    function reset()
+    {
+        $this->current_page_num = -1;
+        $this->end_of_iterator = false;
+        @unlink("{$this->result_dir}/iterate_status.txt");
+    }
+
+    /**
+     * Saves the current state so that a new instantiation can pick up just 
+     * after the last batch of pages extracted.
+     */
+    function saveCheckpoint($info = array())
+    {
+        $info['end_of_iterator'] = $this->end_of_iterator;
+        $info['current_page_num'] = $this->current_page_num;
+        $info['log_iterator'] = $this->log_iterator;
+        file_put_contents("{$this->result_dir}/iterate_status.txt",
+            serialize($info));
+        if($this->end_of_iterator == true){
+            @unlink("{$this->iterate_dir}/master.log");
+        }
+    }
+
+    /**
+     * Restores state from a previous instantiation, after the last batch of 
+     * pages extracted.
+     */
+    function restoreCheckpoint()
+    {
+        $info = unserialize(file_get_contents(
+            "{$this->result_dir}/iterate_status.txt"));
+        $this->end_of_iterator = $info['end_of_iterator'];
+        $this->current_page_num = $info['current_page_num'];
+        $this->log_iterator = $info['log_iterator'];
+        return $info;
+    }
+
+    /**
+     * Pulls data from all the log files in the directory and place it in
+     * the master log file and splits the file into log records
+     * @return array  of log records from the master log file
+     */
+    function createMasterAndRecords(){
+        if(file_exists("{$this->iterate_dir}/master.log")){
+            @unlink("{$this->iterate_dir}/master.log");
+        }
+        for ($i=0;$i<$this->num_partitions;$i++){
+            $file_data = file_get_contents($this->partitions[$i]);
+            file_put_contents("{$this->iterate_dir}/master.log",
+ $file_data, FILE_APPEND);
+        }
+        $recordArray = explode("\n", 
+file_get_contents("{$this->iterate_dir}/master.log"));
+
+        return $recordArray;
+    }
+
+    /**
+     * Unserializes the array of log records stored when save
+     * options is clicked and stores them in the global arrays.
+     */
+     function getFieldDetails()
+     {
+        $fieldArray = unserialize(file_get_contents(
+"{$this->index_dir}/fields_data".$this->result_timestamp.".txt"));
+        foreach($fieldArray as $field=>$field_nt){
+                $matches = explode("::",$field_nt);
+                $this->fields[] = $field;
+                $this->field_names[] = $matches[0];
+                $this->field_types[] = $matches[1];
+        }
+     }
+
+    /**
+     * Takes the log record as input and parses the log record and returns
+     * the array containing the details of each record.
+     *
+     * @param string $record single log record in a file
+     * @return array $matches matched fields of a log record
+     */
+     function parseLogRecord($record)
+     {
+        $content = "";
+        for($j=0;$j<count($this->field_types);$j++){
+                $content = "";
+                $matches = array();
+                preg_match(
+$this->regular_exprs[$this->logfields_type_ddm[$this->field_types[$j]]],
+$record,$matches);
+                if (count($matches)>0){
+                        $spaces_removed = trim($matches[0]);
+                        $record = str_replace($spaces_removed,"",$record);
+                $return_page[$this->logfields_type_ddm[$this->field_types[$j]]] 
+= $matches[0];
+                }
+                else{
+                $return_page[$this->logfields_type_ddm[$this->field_types[$j]]] 
+= "-";
+                }
+        }
+        return $return_page;
+     }
+
+    /**
+     * Gets the next at most $num many docs from the iterator. It might return
+     * less than $num many documents if the partition changes or the end of the
+     * bundle is reached.
+     *
+     * @param int $num number of docs to get
+     * @return array associative arrays for $num pages
+     */
+    function nextPages($num)
+    {
+        $this->getFieldDetails();
+        $pages = array();
+        $page_count = 0;
+        for($i = 0; $i < $num; $i++) {
+            $this->current_page_num++;
+            $this->page_info 
+= $this->parseLogRecord($this->records[$this->current_page_num]);
+            $page = $this->nextPage();
+            if($this->current_page_num >= $this->num_of_records) {
+               $this->end_of_iterator = true;
+               break;
+            }
+            else {
+               $pages[] = $page;
+               $page_count++;
+           }
+        }
+        $this->saveCheckpoint();
+        return $pages;
+    }
+
+    
+    /**
+     * Gets the next doc from the iterator
+     * @return array associative array for doc
+     */
+    function nextPage()
+    {
+           
+        $site = array();
+        $field_nt = "";
+        $fields_count = count($this->page_info);
+        $dom = new DOMDocument('1.0');
+        $root =$dom->createElement('html');
+        $root = $dom->appendChild($root);
+        $head = $dom->createElement('head');
+        $head = $root->appendChild($head);
+        $title = $dom->createElement('title');
+        $title = $head->appendChild($title);
+        for($i=0;$i<$fields_count;$i++){
+                if($this->logfields_type_ddm[$this->field_types[$i]] =="Request"
+                        && $this->page_info['Request'] !="-"){
+                        $recordTitle 
+= "Line ".$this->current_page_num.":".
+$this->page_info[$this->logfields_type_ddm[$this->field_types[$i]]];
+                        break;
+                }
+        }
+        if($recordTitle == ""){
+                $recordTitle = "Line ".$this->current_page_num;
+        }
+        $text = $dom->createTextNode($recordTitle);
+        $text = $title->appendChild($text);
+        $body = $dom->createElement('body');
+        $body = $root->appendChild($body);
+        $field = $dom->createElement('p');
+        $field = $body->appendChild($field);
+        for($i=0;$i<$fields_count;$i++){
+               $field_nt .= $this->field_names[$i].":".
+$this->page_info[$this->logfields_type_ddm[$this->field_types[$i]]]."<br>";
+        }
+        $text1 = $dom->createTextNode($field_nt);
+        $text1 = $field->appendChild($text1);
+        $desc = $dom->createElement('p');
+        $desc = $body->appendChild($desc);
+        $text3 = "This is line ".$this->current_page_num;
+        $text2 = $dom->createTextNode($text3);
+        $text2 = $desc->appendChild($text2);
+        $site[self::PAGE] =$dom->saveHTML();
+        $html_page = "<html><head><title>LogRecord".$this->current_page_num.
+"</title></head><body><h1>The details of the log record are: <br></h1><h3>".
+$field_nt."</h3></body></html>";
+        file_put_contents("{$this->index_dir}/htmlFiles/logrecord".
+$this->current_page_num.".php",$html_page);
+        $site[self::URL] = "http://localhost/".$this->path_for_html_files.
+"cache/IndexData".$this->result_timestamp."/htmlFiles/logrecord".
+$this->current_page_num.".php";
+        $site[self::TYPE] ="text/html";
+        $site[self::HTTP_CODE] = 200;
+        $site[self::ENCODING] = "UTF-8";
+        $site[self::SERVER] = "unknown";
+        $site[self::SERVER_VERSION] = "unknown";
+        $site[self::OPERATING_SYSTEM] = "unknown";
+        $site[self::HASH] = FetchUrl::computePageHash($site[self::PAGE]);
+        $site[self::WEIGHT] = 1;
+
+        return $site;
+    }
+}
+?>
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
index 109b0b8..09b7a37 100644
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -207,6 +207,9 @@ interface CrawlConstants
     const IS_FEED = 'ch';
     const SOURCE_NAME = 'ci';
     const LINK_SEEN_URLS = 'cj';
+    const LOG_RECORDS = 'ck';
+    const DATABASE_RECORDS = 'cl';
+    const DATABASE_CONNECTION_DETAILS = 'cm';
 
     const NEEDS_OFFSET_FLAG = 0x7FFFFFFF;
 
diff --git a/views/elements/crawloptions_element.php b/views/elements/crawloptions_element.php
index 78c6b1a..287a679 100644
--- a/views/elements/crawloptions_element.php
+++ b/views/elements/crawloptions_element.php
@@ -162,6 +162,246 @@ class CrawloptionsElement extends Element
         </div>
         <?php } ?>
         </div>
+
+        <script>
+        obj = document.getElementById("crawl-indexes");
+        obj.onchange = function(){crawloptionsForm.submit();}
+        </script>
+
+<?php    $data['logfields_type'] = array(1=>'IP_Address',
+                                         2=>'Timestamp',
+                                         3=>'URL',
+                                         4=>'Status Code',
+                                         5=>'User Agent',
+                                         6=>'Request',
+                                         7=>'Int');
+        $flag = false;
+
+        /* If log files are selected as the option */ 
+        if($data['available_crawl_indexes'][$_POST['crawl_indexes']] ==
+                                                        'ARCFILE::Log Files') {
+
+                $LogFolderPath = CRAWL_DIR.'/cache/archives';
+                foreach(glob($LogFolderPath."/*") as $folder){
+                        if(is_dir($folder)){
+                                if(file_exists("$folder/arc_description.ini")){
+                                        $contents = 
+                               file_get_contents("$folder/arc_description.ini");
+                                        if(strpos($contents,"LogArchiveBundle")
+                                                                      == true){
+                                                $flag = true;
+                                                $LogFolderPath = $folder;
+                                        }
+                                }
+                        }
+                        if($flag == true) {break;}
+                }
+
+                /*Get all the file names into an array*/
+                $filenames = glob($LogFolderPath."/*.log");
+
+                /*Retrieve the first filename*/
+                $firstFile = $filenames[0];
+
+                /*Split the file content into an array*/
+                $l_delim = "\n";
+                $file_array = explode($l_delim, file_get_contents($firstFile));
+                echo "<br><b>".tl('crawloptions_element_first_line_text')."</b>
+                                                                          <br>";
+                echo "<br>".$file_array[0]."<br>";
+
+        ?>
+                <div id="Log_Records" class="topmargin"><b><?php 
+                e(tl('crawloptions_element_log_records_details'))?></b></div>
+
+                <table class="logrecordstable">
+                    <tr><th><?php e(tl('crawloptions_element_field'));?></th>
+                        <th><?php e(tl('crawloptions_element_field_name')); ?>
+                        </th>
+                        <th><?php e(tl('crawloptions_element_field_type')); ?>
+                        </th>
+                   </tr>
+                    <?php
+                    $i = 0;
+                    foreach($data['LOG_RECORDS'] as $field => $field_nt) {
+                        $matches = explode("::",$field_nt);
+                    ?>
+                       <tr><td class="input-field" >
+                                <input 
+title="<?php e(tl('crawloptions_element_field')); ?>" 
+name="LOG_RECORDS[<?php e($i); ?>][FIELD]" 
+value="<?php e($field); ?>" />
+                           </td>
+                           <td class="input-field-name"><input
+                     title="<?php e(tl('crawloptions_element_field_name')); ?>"
+                     name="LOG_RECORDS[<?php e($i); ?>]['FIELD_NAME']" 
+                     value="<?php e($matches[0]); ?>" 
+                                    />
+                           </td>
+                           <td class="input-field-type" >
+                                <?php $this->view->optionsHelper->render(
+'field-types', "LOG_RECORDS[$i]['FIELD_TYPE']", $data['logfields_type'],
+ $matches[1]);  ?>
+                           </td>
+                       </tr>
+                <?php
+                     $i++;
+                    }
+                ?>
+                <?php
+                   if($i==0){?>
+                        <tr>
+                            <td class="input-field"><input 
+type="text" 
+title="New Field" 
+name="LOG_RECORDS[<?php e($i); ?>][FIELD]" 
+value=""/></td>
+                            <td class="input-field-name"><input 
+type="text" 
+title="New Field Name" 
+name="LOG_RECORDS[<?php e($i); ?>]['FIELD_NAME']" 
+value=""/></td>
+                            <td class="input-field-type">
+                                <?php $this->view->optionsHelper->render(
+'field-types', "LOG_RECORDS[$i]['FIELD_TYPE']", $data['logfields_type'], 1); ?>
+                            </td>
+                       </tr>  
+                <?php 
+                      } ?>
+
+                <?php
+                         if(isset($_POST['add_fields']) && $i>0){
+                ?>
+                                <tr>
+                                    <td class="input-field"><input 
+type="text" 
+title="New Field" 
+name="LOG_RECORDS[<?php e($i); ?>]['FIELD']" 
+value=""/></td>
+                                    <td class="input-field-name"><input 
+type="text" 
+title="New Field Name" 
+name="LOG_RECORDS[<?php e($i); ?>]['FIELD_NAME']" 
+value=""/></td>
+                                    <td class="input-field-type">
+                                      <?php $this->view->optionsHelper->render(
+'field-types', "LOG_RECORDS[$i]['FIELD_TYPE']", $data['logfields_type'], 1); ?>
+                                    </td>
+                               </tr>   
+
+                <?php } ?>
+                </table>
+                <?php
+                if(isset($_POST['save_options']) 
+&& $data['available_crawl_indexes'][$_POST['crawl_indexes']] 
+== 'ARCFILE::Log Files'){
+                           file_put_contents($LogFolderPath."/fields_data.txt",
+ serialize($data['LOG_RECORDS']));
+                        }
+                ?>
+                <div class="logrecordnewfield">
+                <input type="submit" 
+id="add-fields" 
+name="add_fields" 
+value="<?php e(tl('crawloptions_element_add_new_field')); ?>"/></div>
+<?php }?>
+
+
+<?php    if($data['available_crawl_indexes'][$_POST['crawl_indexes']] 
+== 'ARCFILE::Database files') {
+
+                $flag1 = false;
+
+                $DatabaseFolderPath = CRAWL_DIR.'/cache/archives';
+                foreach(glob($DatabaseFolderPath."/*") as $folder){
+                        if(is_dir($folder)){
+                                if(file_exists("$folder/arc_description.ini")){
+                                      $contents 
+= file_get_contents("$folder/arc_description.ini");
+                                        if(strpos($contents,"Database files") 
+== true){
+                                                $flag1 = true;
+                                                $DatabaseFolderPath = $folder;
+                                        }
+                                }
+                        }
+                        if($flag1 == true) {break;}
+                }
+?>
+
+                <div id="Database_Connection_Details" class="topmargin"><b><?php
+                    e(tl('crawloptions_element_database_connection_details'))?>
+</b></div><br>
+                <table class="databaseconnectiondetailstable">
+                <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_hostname'))?></td>
+                <td class="input-data"><input 
+title="<?php e(tl('crawloptions_element_hostname')); ?>" 
+name="DATABASE_CONNECTION_DETAILS[HOSTNAME]" 
+value="" /></td>
+                </tr>
+                <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_username'))?></td>
+                <td class="input-data"><input 
+title="<?php e(tl('crawloptions_element_username')); ?>" 
+name="DATABASE_CONNECTION_DETAILS[USERNAME]" 
+value="" /></td>
+                </tr>
+                <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_password'))?></td>
+                <td class="input-data"><input 
+title="<?php e(tl('crawloptions_element_password')); ?>" 
+name="DATABASE_CONNECTION_DETAILS[PASSWORD]" 
+value="" /></td>
+                </tr>
+                <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_databasename'))?></td>
+                <td class="input-data"><input 
+title="<?php e(tl('crawloptions_element_databasename')); ?>" 
+name="DATABASE_CONNECTION_DETAILS[DATABASENAME]" 
+value="" /></td>
+                </tr>
+                <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_query'))?></td>
+                <td class="input-data"><input 
+title="<?php e(tl('crawloptions_element_query')); ?>" 
+name="DATABASE_CONNECTION_DETAILS[QUERY]" 
+value="" /></td>
+                </tr>
+                </table>
+                <div class="databaseconnectiondetailssubmit">
+                <input type="submit" 
+id="submit-details" 
+name="submit_details" 
+value="<?php e(tl('crawloptions_element_submit')); ?>"/></div>
+
+                <?php 
+                        if(isset($_POST['submit_details'])){
+                                 file_put_contents($DatabaseFolderPath.
+"/database_connection_details.txt",
+ serialize($data['DATABASE_CONNECTION_DETAILS']));
+                        }
+                ?>
+                <?php
+                        if(isset($_POST['save_options']) 
+&& $data['available_crawl_indexes'][$_POST['crawl_indexes']] 
+== 'ARCFILE::Database files'){
+                                 file_put_contents($DatabaseFolderPath.
+"/database_connection_details.txt", 
+serialize($data['DATABASE_CONNECTION_DETAILS']));
+                        }
+                ?>
+
+<?php   }?>
+        </div>
+        <?php } ?>
+        </div>
+
         <div class="topmargin"><b><?php 
             e(tl('crawloptions_element_meta_words'))?></b></div>
         <table class="metawordstable">
-- 
1.7.11.msysgit.1

