_binary From b9df5eee2a01884dd9bbebf8e9d25e739d987fda Mon Sep 17 00:00:00 2001
From: unknown <tanmayee.4170@gmail.com>
Date: Tue, 18 Dec 2012 14:02:22 -0800
Subject: [PATCH] Invisible_Web_Patch_Dec18

---
 configs/config.php                                 |   2 +-
 controllers/admin_controller.php                   |  53 +++
 css/search.css                                     |  90 +++++
 .../database_archive_bundle_iterator.php           | 375 +++++++++++++++++
 .../log_archive_bundle_iterator.php                | 448 +++++++++++++++++++++
 lib/crawl_constants.php                            |   3 +
 locale/en-US/configure.ini                         | 306 ++++++++------
 views/elements/crawloptions_element.php            | 283 ++++++++++++-
 8 files changed, 1435 insertions(+), 125 deletions(-)
 create mode 100644 lib/archive_bundle_iterators/database_archive_bundle_iterator.php
 create mode 100644 lib/archive_bundle_iterators/log_archive_bundle_iterator.php

diff --git a/configs/config.php b/configs/config.php
index 68f5e25..1fb0113 100644
--- a/configs/config.php
+++ b/configs/config.php
@@ -63,7 +63,7 @@ if(MAINTENANCE_MODE && $_SERVER["SERVER_ADDR"] != $_SERVER["REMOTE_ADDR"]) {
 if(!defined('WORK_DIRECTORY')) {
 /*+++ The next block of code is machine edited, change at 
 your own risk, please use configure web page instead +++*/
-define('WORK_DIRECTORY', '');
+define('WORK_DIRECTORY', 'c:/xampp/htdocs/yioop_data');
 /*++++++*/
 // end machine edited code
 }
diff --git a/controllers/admin_controller.php b/controllers/admin_controller.php
index 84dabf6..c972137 100755
--- a/controllers/admin_controller.php
+++ b/controllers/admin_controller.php
@@ -792,6 +792,15 @@ class AdminController extends Controller implements CrawlConstants
                     $crawl_params[self::META_WORDS] = 
                         isset($seed_info['meta_words']) ?
                         $seed_info['meta_words'] : array();
+
+                    $crawl_params[self::LOG_RECORDS] =
+                        isset($seed_info['log_records']) ?
+                        $seed_info['log_records'] : array();
+
+                    $crawl_params[self::DATABASE_CONNECTION_DETAILS] =
+                        isset($seed_info['database_connection_details']) ?
+                        $seed_info['database_connection_details'] : array();
+
                     if(isset($seed_info['indexing_plugins']['plugins'])) {
                         $crawl_params[self::INDEXING_PLUGINS] =
                             $seed_info['indexing_plugins']['plugins'];
@@ -1062,6 +1071,50 @@ class AdminController extends Controller implements CrawlConstants
                     } else if(isset($seed_info['meta_words'])){
                             $data['META_WORDS'] = $seed_info['meta_words'];
                     }
+
+                    $data['LOG_RECORDS'] = array();
+                    if(!$no_further_changes) {
+                        if(isset($_REQUEST["LOG_RECORDS"])){
+                            foreach($_REQUEST["LOG_RECORDS"] as $triplet) {
+                                list($field, $field_name,$field_type) =
+                                                        array_values($triplet);
+                                $field = $this->clean($field, "string");
+                                $field_name =
+                                        $this->clean($field_name, "string");
+                                $field_type =
+                                        $this->clean($field_type,"string");
+                                $field_nt = $field_name."::".$field_type;
+                                if(trim($field) != "" &&trim($field_nt) !=""){
+                                      $data['LOG_RECORDS'][$field] = $field_nt;
+                                }
+                            }
+                            $seed_info['log_records'] = $data['LOG_RECORDS'];
+                            $update_flag = true;
+                        } else if(isset($seed_info['log_records'])){
+                            $data['LOG_RECORDS'] = $seed_info['log_records'];
+                        }
+                    } else if(isset($seed_info['log_records'])){
+                            $data['LOG_RECORDS'] = $seed_info['log_records'];
+                    }
+
+                    $data['DATABASE_CONNECTION_DETAILS'] = array();
+                    if(!$no_further_changes) {
+                        if(isset($_REQUEST["DATABASE_CONNECTION_DETAILS"])){
+                            $data['DATABASE_CONNECTION_DETAILS']=
+                                       $_REQUEST["DATABASE_CONNECTION_DETAILS"];
+                            $seed_info['database_connection_details'] = 
+                                       $data['DATABASE_CONNECTION_DETAILS'];
+                            $update_flag = true;
+                        } else if(isset(
+                                $seed_info['database_connection_details'])){
+                            $data['DATABASE_CONNECTION_DETAILS'] = 
+                                     $seed_info['database_connection_details'];
+                        }
+                    } else if(isset($seed_info['database_connection_details'])){
+                            $data['DATABASE_CONNECTION_DETAILS'] = 
+                                     $seed_info['database_connection_details'];
+                    }
+
                     $data['INDEXING_PLUGINS'] = array();
                     $included_plugins = array();
                     if(!$no_further_changes && isset($_REQUEST["posted"])) {
diff --git a/css/search.css b/css/search.css
index 514bbe7..1210aa1 100755
--- a/css/search.css
+++ b/css/search.css
@@ -1724,6 +1724,96 @@ ul.in-list li
     width: 97%;
 }
 
+.logrecordstable
+{
+   width:100%;
+}
+
+.logrecordstable,
+.logrecordstable th,
+.logrecordstable td
+{
+    border: 1px ridge black;
+}
+
+.logrecordstable th
+{
+    text-align: center;
+    padding: .03in;
+}
+.logrecordstable td.input-field
+{
+    width: 1.3in;
+}
+.logrecordstable td.input-field input
+{
+    width: 1.5in;
+    margin:.05in;
+}
+.logrecordstable td.input-field-name
+{
+    margin:.03in;
+    width: 100%;
+}
+.logrecordstable td.input-field-name input
+{
+    margin:.05in;
+    width: 97%;
+}
+.logrecordstable td.input-field-type
+{
+    margin:.03in;
+    width: 100%;
+}
+.logrecordstable td.input-field-type input
+{
+    margin:.05in;
+    width: 97%;
+}
+.html-rtl .logrecordnewfield
+{
+    position: relative;
+}
+.html-ltr .logrecordnewfield
+{
+    position: relative;
+}
+
+.databaseconnectiondetailstable
+{
+   width:100%;
+}
+
+.databaseconnectiondetailstable,
+.databaseconnectiondetailstable td
+{
+    border: 1px ridge black;
+}
+
+.databaseconnectiondetailstable td.input-name
+{
+    width: 1.3in;
+}
+.databaseconnectiondetailstable td.input-data
+{
+    margin:.03in;
+    width: 100%;
+}
+.databaseconnectiondetailstable td.input-data input
+{
+    margin:.05in;
+    width: 98%;
+}
+.html-rtl .database-connection-details-submit
+{
+    position: relative;
+}
+.html-ltr .database-connection-details-submit
+{
+    position: relative;
+}
+
+
 .indexingplugintable
 {
    width:100%;
diff --git a/lib/archive_bundle_iterators/database_archive_bundle_iterator.php b/lib/archive_bundle_iterators/database_archive_bundle_iterator.php
new file mode 100644
index 0000000..abf3ea7
--- /dev/null
+++ b/lib/archive_bundle_iterators/database_archive_bundle_iterator.php
@@ -0,0 +1,375 @@
+<?php
+/** 
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2012  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2012
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** 
+ *Loads base class for iterating
+ */
+require_once BASE_DIR.
+    '/lib/archive_bundle_iterators/archive_bundle_iterator.php';
+
+/**
+ * Used to iterate through the records of a database stored in a 
+ * DatabaseArchiveBundle folder. Database is a collection of tables with 
+ * various rows in each table. Iteration would be for the purpose of making 
+ * an index of each of these records.
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @see WebArchiveBundle
+ */
+class DatabaseArchiveBundleIterator extends ArchiveBundleIterator 
+    implements CrawlConstants
+{
+    /**
+     * The path to the directory containing the archive partitions to be 
+     * iterated over.
+     * @var string
+     */
+    var $iterate_dir;
+    /**
+     * The path to the directory where the iteration status is stored.
+     * @var string
+     */
+    var $result_dir;
+    /**
+     * The path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $index_dir;
+
+    /**
+     * The part of the path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $path_for_html_files;
+    /**
+     *  current number of record in the current database 
+     *  @var int
+     */
+    var $current_page_num;
+    /**
+     * The number of database records in this database archive bundle
+     *  @var int
+     */
+    var $num_of_records;
+    /**
+     *  Array of database records according to the query specified by the user
+     *  @var array
+     */
+    var $records;
+    /**
+     *  Array of fields of database record specified by the user
+     *  @var array
+     */
+    var $fields;
+    /**
+     *  Array of field names in database record specified by the user
+     *  @var array
+     */
+    var $field_names;
+    /**
+     *  Array of database fieldtypes specified by the user
+     *  @var array
+     */
+    var $field_types;
+    /**
+     *  Database handle for a database
+     *  @var resource
+     */
+    var $db_handle;
+    /**
+     *  Whether database exists or not
+     *  @var resource
+     */
+    var $db_found;
+    /**
+     *  Array of database connection details to connect to database
+     *  @var array
+     */
+    var $databaseConnectionsArray; 
+    /**
+     *  Host name for the database
+     *  @var string
+     */
+    var $host; 
+    /**
+     *  User Name for the localhost
+     *  @var string
+     */
+    var $user_name;
+    /**
+     *  Password for the connection
+     *  @var string
+     */
+    var $password;
+    /**
+     *  Name of the database for the records
+     *  @var string
+     */
+    var $database;
+    /**
+     *  Query to retrieve the records to be indexed from the database 
+     *  @var array
+     */
+    var $query;
+    /**
+     *  Constants required for log archive_bundle_iterator
+     *  @var const
+     */
+    const DATABASE_CONNECTION_DETAILS_FILE = 'database_connection_details.txt';
+
+    /**
+     * Creates a database archive iterator with the given parameters.
+     * @param string $iterate_timestamp timestamp of the arc archive bundle to 
+     *      iterate  over the pages of
+     * @param string $result_timestamp timestamp of the arc archive bundle
+     *      results are being stored in
+     */
+    function __construct($iterate_timestamp, $iterate_dir,
+        $result_timestamp, $result_dir)
+    {
+        $this->iterate_timestamp = $iterate_timestamp;
+        $this->iterate_dir = $iterate_dir;
+        $this->result_timestamp = $result_timestamp;
+        $this->result_dir = $result_dir;
+        $this->index_dir = CRAWL_DIR."/cache/IndexData".$this->result_timestamp;
+        $temp_array = explode("/", $this->index_dir);
+        $temp_array_len = count($temp_array);
+
+        for($i = 1;$i<$temp_array_len;$i++){
+            if($temp_array[$i-1]=="htdocs"){
+                while($temp_array[$i] != "cache"){
+                    $this->path_for_html_files .= $temp_array[$i]."/";
+                    $i++;
+                }
+                break;
+            }
+        }
+        if(file_exists("{$this->iterate_dir}/".
+            self::DATABASE_CONNECTION_DETAILS_FILE))
+        {
+            $database_connection_details_info = unserialize
+               (file_get_contents(
+               "{$this->iterate_dir}/".self::DATABASE_CONNECTION_DETAILS_FILE));
+            file_put_contents("{$this->index_dir}/database_connection_details".
+                $this->result_timestamp.".txt", 
+                serialize($database_connection_details_info));
+            @unlink("{$this->iterate_dir}/".
+                self::DATABASE_CONNECTION_DETAILS_FILE);
+        }
+        if(!is_dir("{$this->index_dir}/HTML_FILES")){
+            mkdir("{$this->index_dir}/HTML_FILES");
+        }
+        $this->createRecords();
+
+        if(file_exists("{$this->result_dir}/iterate_status.txt")) {
+            $this->restoreCheckpoint();
+        }
+        else {
+            $this->reset();
+        }
+    }
+
+    /**
+     * Estimates the important of the site according to the weighting of
+     * the particular archive iterator
+     * @param $site an associative array containing info about a web page
+     * @return bool false we assume arc files were crawled according to 
+     *      OPIC and so we use the default doc_depth to estimate page importance
+     */
+    function weight(&$site) 
+    {
+        return false;
+    }
+
+    /**
+     * Resets the iterator to the start of the archive bundle
+     */
+    function reset()
+    {
+        $this->current_page_num = -1;
+        $this->end_of_iterator = false;
+        @unlink("{$this->result_dir}/iterate_status.txt");
+    }
+
+    /**
+     * Saves the current state so that a new instantiation can pick up just 
+     * after the last batch of pages extracted.
+     */
+    function saveCheckpoint($info = array())
+    {
+        $info['end_of_iterator'] = $this->end_of_iterator;
+        $info['current_page_num'] = $this->current_page_num;
+        $info['database_iterator'] = $this->database_iterator;
+        file_put_contents("{$this->result_dir}/iterate_status.txt",
+            serialize($info));
+    }
+
+    /**
+     * Restores state from a previous instantiation, after the last batch of 
+     * pages extracted.
+     */
+    function restoreCheckpoint()
+    {
+        $info = unserialize(file_get_contents(
+            "{$this->result_dir}/iterate_status.txt"));
+        $this->end_of_iterator = $info['end_of_iterator'];
+        $this->current_page_num = $info['current_page_num'];
+        $this->database_iterator = $info['database_iterator'];
+        return $info;
+    }
+
+    /**
+     * Creates Records array containing all the records to satisfying the query
+     */
+    function createRecords()
+    {
+        $this->databaseConnectionsArray = unserialize(file_get_contents(
+            "{$this->index_dir}/database_connection_details".
+            $this->result_timestamp.".txt"));
+        $this->host = $this->databaseConnectionsArray['HOSTNAME'];
+        $this->user_name = $this->databaseConnectionsArray['USERNAME'];
+        $this->password = $this->databaseConnectionsArray['PASSWORD'];
+        $this->database = $this->databaseConnectionsArray['DATABASENAME'];
+        $this->query = $this->databaseConnectionsArray['QUERY'];
+        $this->db_handle = mysql_connect($this->host, $this->user_name,
+            $this->password);
+        $this->db_found = mysql_select_db($this->database, $this->db_handle);
+
+        /*If database exists*/
+        if ($this->db_found) {
+            $result1 = mysql_query($this->query);
+            $num_fields = mysql_num_fields($result1);
+            for($i = 0; $i < $num_fields; $i++) {
+                $this->field_names[$i] = mysql_field_name($result1, $i);
+            }
+            while ($row = mysql_fetch_row($result1)) {
+                $this->records[] = $row;
+            }
+            $this->num_of_records = count($this->records);
+            mysql_free_result($result1);
+            mysql_close($db_handle);
+        }
+    }
+    /**
+     * Gets the next at most $num many records from the iterator. It might 
+     * return less than $num many documents if the end of the bundle is reached.
+     * @param int $num number of docs to get
+     * @return array associative arrays for $num pages
+     */
+    function nextPages($num)
+    {
+        $pages = array();
+        for($i = 0; $i < $num; $i++) {
+            $this->current_page_num++;
+            $page = $this->nextPage();
+            if($this->current_page_num >= $this->num_of_records) {
+                $this->end_of_iterator = true;
+                break;
+            }
+            else {
+                $pages[] = $page;
+            }
+        }
+     
+        $this->saveCheckpoint();
+        return $pages;
+    }
+
+    
+    /**
+     * Gets the next record from the iterator
+     * @return array associative array for record
+     */
+    function nextPage()
+    {
+        $site = array();
+        $field_nd = "";
+        $html_page = "";
+        $temp_record = array();
+        $temp_record = $this->records[$this->current_page_num];
+        $dom = new DOMDocument('1.0');
+        $root =$dom->createElement('html');
+        $root = $dom->appendChild($root);
+        $head = $dom->createElement('head');
+        $head = $root->appendChild($head);
+        $title = $dom->createElement('title');
+        $title = $head->appendChild($title);
+        $recordTitle = "Database Record".$this->current_page_num;
+        $text = $dom->createTextNode($recordTitle);
+        $text = $title->appendChild($text);
+        $body = $dom->createElement('body');
+        $body = $root->appendChild($body);
+        $field = $dom->createElement('p');
+        $field = $body->appendChild($field);
+        $fieldnames_len = count($this->field_names);
+        for($i = 0; $i < $fieldnames_len; $i++) {
+                $field_nd .= $this->field_names[$i]." : ".$temp_record[$i]
+                ."<br>";
+        }
+        $text1 = $dom->createTextNode($field_nd);
+        $text1 = $field->appendChild($text1);
+        $desc = $dom->createElement('p');
+        $desc = $body->appendChild($desc);
+        $text3 = "This is database record ".$this->current_page_num;
+        $text2 = $dom->createTextNode($text3);
+        $text2 = $desc->appendChild($text2);
+        $site[self::PAGE] =$dom->saveHTML();
+        $html_page = "<html><head><title>DatabaseRecord".
+            $this->current_page_num.
+            "</title></head><body><h1>The details of the database record are:".
+            "<br></h1><h3>".$field_nd."</h3></body></html>";
+        file_put_contents("{$this->index_dir}/HTML_FILES/databaserecord".
+            $this->current_page_num.".php",$html_page);
+        $site[self::URL] = "http://localhost/".$this->path_for_html_files.
+            "cache/IndexData".$this->result_timestamp.
+            "/HTML_FILES/databaserecord".$this->current_page_num.".php";
+        $site[self::TYPE] ="text/html";
+        $site[self::HTTP_CODE] = 200;
+        $site[self::ENCODING] = "UTF-8";
+        $site[self::SERVER] = "unknown";
+        $site[self::SERVER_VERSION] = "unknown";
+        $site[self::OPERATING_SYSTEM] = "unknown";
+        $site[self::HASH] = FetchUrl::computePageHash($site[self::PAGE]);
+        $site[self::WEIGHT] = 1;
+
+        return $site;
+        
+    }
+}
+?>
diff --git a/lib/archive_bundle_iterators/log_archive_bundle_iterator.php b/lib/archive_bundle_iterators/log_archive_bundle_iterator.php
new file mode 100644
index 0000000..6cd2ccc
--- /dev/null
+++ b/lib/archive_bundle_iterators/log_archive_bundle_iterator.php
@@ -0,0 +1,448 @@
+<?php
+/** 
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2012  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2012
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** 
+ *Loads base class for iterating
+ */
+require_once BASE_DIR.
+    '/lib/archive_bundle_iterators/archive_bundle_iterator.php';
+
+/**
+ * Used to iterate through the collection of log files stored in
+ * a WebArchiveBundle folder. Log is the file format which has the 
+ * activities the system or a server performs. Iteration would be 
+ * for the purpose making an index of these files.
+ *
+ * @author Tanmayee Potluri
+ * @package seek_quarry
+ * @subpackage iterator
+ * @see WebArchiveBundle
+ */
+
+class LogArchiveBundleIterator extends ArchiveBundleIterator 
+    implements CrawlConstants
+{
+    /**
+     * The path to the directory containing the archive partitions to be 
+     * iterated over.
+     * @var string
+     */
+    var $iterate_dir;
+
+    /**
+     * The path to the directory where the iteration status is stored.
+     * @var string
+     */
+    var $result_dir;
+
+    /**
+     * The path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $index_dir;
+
+    /**
+     * The part of the path to the directory where all html files are stored 
+     * and used to point to.
+     * @var string
+     */
+    var $path_for_html_files;
+
+    /**
+     * The number of log files in this log archive bundle
+     *  @var int
+     */
+    var $num_partitions;
+
+    /**
+     *  current record number in the master log file
+     *  @var int
+     */
+    var $current_page_num;
+
+    /**
+     *  number of records in the master log file
+     *  @var int
+     */
+    var $num_of_records;
+
+    /**
+     *  Array of log records in the master log file in the directory
+     *  @var array
+     */
+    var $records;
+
+    /**
+     *  Array of filenames of log files in this directory (glob order)
+     *  @var array
+     */
+    var $partitions;
+    
+    /**
+     *  Array of fields of log file specified by the user
+     *  @var array
+     */
+    var $fields;
+
+    /**
+     *  Array of fieldnames specified by the user
+     *  @var array
+     */
+    var $field_names;
+    
+    /**
+     *  Array of fieldtypes specified by the user
+     *  @var array
+     */
+    var $field_types;
+
+    /**
+     *  Array of fields in each record separately
+     *  @var array
+     */
+    var $page_info;
+
+    /**
+     *  Array of regular expressions for all the data types
+     *  @var array
+     */
+    var $regular_exprs = array( 
+        'IP_Address' => '/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/',
+        'Timestamp'  => '/\[[^:]+:\d+:\d+:\d+ [^\]]+\]/',
+        'Request'    => '/(GET|HEAD|POST|PUT|DELETE|TRACE|OPTIONS|CONNECT)+[^"]*/',
+        'Status Code'=> '/\s[1-5]\d{2}\s/',
+        'Int'        => '/\s[0-9]+\s/',
+        'URL'        => '/(http|https|ftp):\/\/[A-Za-z0-9][A-Za-z0-9_-]*[\/]*(?:.[A-Za-z0-9][A-Za-z0-9_-]*[\/]*)+:?(d*)[\/]*/',
+        'User Agent' => '/"([a-zA-Z0-9][^"]+)"/');
+
+    /**
+     *  Array of log fields type in drop down box in the UI
+     *  @var array
+     */
+    var $logfields_type_ddm = array(
+        1=>'IP_Address',
+        2=>'Timestamp',
+        3=>'URL',
+        4=>'Status Code',
+        5=>'User Agent',
+        6=>'Request',
+        7=>'Int');
+
+    /**
+     *  Constants required for log archive_bundle_iterator
+     *  @var const
+     */
+    const FIELDS_DATA_FILE = 'fields_data.txt';
+    const MASTER_LOG_FILE = 'master.log';
+
+    /**
+     * Creates a log archive iterator with the given parameters.
+     *
+     * @param string $iterate_timestamp timestamp of the log archive bundle to 
+     *      iterate  over the pages of
+     * @param string $result_timestamp timestamp of the log archive bundle
+     *      results are being stored in
+     */
+    function __construct($iterate_timestamp, $iterate_dir,
+        $result_timestamp, $result_dir)
+    {
+        $this->path_for_html_files = "";
+        $this->iterate_timestamp = $iterate_timestamp;
+        $this->iterate_dir = $iterate_dir;
+        $this->result_timestamp = $result_timestamp;
+        $this->result_dir = $result_dir;
+        $this->index_dir = CRAWL_DIR."/cache/IndexData".$this->result_timestamp;
+        $temp_array = explode("/", $this->index_dir);
+        $temp_array_len = count($temp_array);
+
+        for($i = 1; $i < $temp_array_len; $i++) {
+            if($temp_array[$i-1] == "htdocs") {
+                while($temp_array[$i] != "cache") {
+                    $this->path_for_html_files .= $temp_array[$i]."/";
+                    $i++;
+                }
+                break;
+            }
+        }
+        if(file_exists("{$this->iterate_dir}/".self::FIELDS_DATA_FILE)) {
+            $fields_data = unserialize(file_get_contents(
+                               "{$this->iterate_dir}/".self::FIELDS_DATA_FILE));
+            file_put_contents(
+                "{$this->index_dir}/fields_data".$this->result_timestamp.".txt",
+                serialize($fields_data));
+            @unlink("{$this->iterate_dir}/".self::FIELDS_DATA_FILE);
+        }
+        if(!is_dir("{$this->index_dir}/HTML_FILES")) {
+            mkdir("{$this->index_dir}/HTML_FILES");
+        }
+        $this->partitions = array();
+        foreach(glob("{$this->iterate_dir}/*.log") as $filename) { 
+            if(strpos($filename,self::MASTER_LOG_FILE)!= true) {
+                $this->partitions[] = $filename;
+            }
+        }
+        $this->num_partitions = count($this->partitions);
+        $this->records = $this->createMasterAndRecords();
+        $this->num_of_records = count($this->records);
+        if(file_exists("{$this->result_dir}/iterate_status.txt")){
+            $this->restoreCheckpoint();
+        } 
+        else {
+            $this->reset();
+        }
+    }
+
+    /**
+     * Estimates the important of the site according to the weighting of
+     * the particular archive iterator
+     * @param $site an associative array containing info about a web page
+     * @return value 1 we assume all log files crawled have the same 
+     *  page importance
+     */
+    function weight(&$site) 
+    {
+        return 1;
+    }
+
+    /**
+     * Resets the iterator to the start of the archive bundle
+     */
+    function reset()
+    {
+        $this->current_page_num = -1;
+        $this->end_of_iterator = false;
+        @unlink("{$this->result_dir}/iterate_status.txt");
+    }
+
+    /**
+     * Saves the current state so that a new instantiation can pick up just 
+     * after the last batch of pages extracted.
+     */
+    function saveCheckpoint($info = array())
+    {
+        $info['end_of_iterator'] = $this->end_of_iterator;
+        $info['current_page_num'] = $this->current_page_num;
+        $info['log_iterator'] = $this->log_iterator;
+        file_put_contents("{$this->result_dir}/iterate_status.txt",
+            serialize($info));
+        if($this->end_of_iterator == true){
+            @unlink("{$this->iterate_dir}/".self::MASTER_LOG_FILE);
+        }
+    }
+
+    /**
+     * Restores state from a previous instantiation, after the last batch of 
+     * pages extracted.
+     */
+    function restoreCheckpoint()
+    {
+        $info = unserialize(file_get_contents(
+            "{$this->result_dir}/iterate_status.txt"));
+        $this->end_of_iterator = $info['end_of_iterator'];
+        $this->current_page_num = $info['current_page_num'];
+        $this->log_iterator = $info['log_iterator'];
+        return $info;
+    }
+
+    /**
+     * Pulls data from all the log files in the directory and place it in
+     * the master log file and splits the file into log records
+     * @return array  of log records from the master log file
+     */
+    function createMasterAndRecords(){
+        if(file_exists("{$this->iterate_dir}/".self::MASTER_LOG_FILE)) {
+            @unlink("{$this->iterate_dir}/".self::MASTER_LOG_FILE);
+        }
+        for ($i=0; $i < $this->num_partitions; $i++){
+            $file_data = file_get_contents($this->partitions[$i]);
+            file_put_contents("{$this->iterate_dir}/".self::MASTER_LOG_FILE,
+                 $file_data, FILE_APPEND);
+        }
+        $recordArray = explode("\n", 
+                file_get_contents("{$this->iterate_dir}/".
+                        self::MASTER_LOG_FILE));
+
+        return $recordArray;
+    }
+
+    /**
+     * Unserializes the array of log records stored when save
+     * options is clicked and stores them in the global arrays.
+     */
+     function getFieldDetails()
+     {
+        $fieldArray = unserialize(file_get_contents(
+            "{$this->index_dir}/fields_data".$this->result_timestamp.".txt"));
+        foreach($fieldArray as $field=>$field_nt){
+            $matches = explode("::",$field_nt);
+            $this->fields[] = $field;
+            $this->field_names[] = $matches[0];
+            $this->field_types[] = $matches[1];
+        }
+     }
+
+    /**
+     * Takes the log record as input and parses the log record and returns
+     * the array containing the details of each record.
+     *
+     * @param string $record single log record in a file
+     * @return array $matches matched fields of a log record
+     */
+     function parseLogRecord($record)
+     {
+        $content = "";
+        $field_types_len = count($this->field_types);
+        for($j = 0; $j < $field_types_len; $j++) {
+            $content = "";
+            $matches = array();
+            preg_match(
+        $this->regular_exprs[$this->logfields_type_ddm[$this->field_types[$j]]],
+        $record,$matches);
+            if (count($matches)>0) {
+                $spaces_removed = trim($matches[0]);
+                $record = str_replace($spaces_removed,"",$record);
+                $return_page[$this->logfields_type_ddm[$this->field_types[$j]]] 
+                        = $matches[0];
+                }
+                else {
+                $return_page[$this->logfields_type_ddm[$this->field_types[$j]]] 
+                        = "-";
+                }
+        }
+        return $return_page;
+     }
+
+    /**
+     * Gets the next at most $num many docs from the iterator. It might return
+     * less than $num many documents if the partition changes or the end of the
+     * bundle is reached.
+     *
+     * @param int $num number of docs to get
+     * @return array associative arrays for $num pages
+     */
+    function nextPages($num)
+    {
+        $this->getFieldDetails();
+        $pages = array();
+        $page_count = 0;
+        for($i = 0; $i < $num; $i++) {
+            $this->current_page_num++;
+            $this->page_info 
+               = $this->parseLogRecord($this->records[$this->current_page_num]);
+            $page = $this->nextPage();
+            if($this->current_page_num >= $this->num_of_records) {
+               $this->end_of_iterator = true;
+               break;
+            }
+            else {
+               $pages[] = $page;
+               $page_count++;
+           }
+        }
+        $this->saveCheckpoint();
+        return $pages;
+    }
+
+    
+    /**
+     * Gets the next doc from the iterator
+     * @return array associative array for doc
+     */
+    function nextPage()
+    {
+        $site = array();
+        $field_nt = "";
+        $fields_count = count($this->page_info);
+        $dom = new DOMDocument('1.0');
+        $root =$dom->createElement('html');
+        $root = $dom->appendChild($root);
+        $head = $dom->createElement('head');
+        $head = $root->appendChild($head);
+        $title = $dom->createElement('title');
+        $title = $head->appendChild($title);
+        for($i=0;$i<$fields_count;$i++){
+            if($this->logfields_type_ddm[$this->field_types[$i]] =="Request"
+                && $this->page_info['Request'] !="-") {
+                $recordTitle = "Line ".$this->current_page_num.":".
+        $this->page_info[$this->logfields_type_ddm[$this->field_types[$i]]];
+                break;
+                }
+        }
+        if($recordTitle == ""){
+                $recordTitle = "Line ".$this->current_page_num;
+        }
+        $text = $dom->createTextNode($recordTitle);
+        $text = $title->appendChild($text);
+        $body = $dom->createElement('body');
+        $body = $root->appendChild($body);
+        $field = $dom->createElement('p');
+        $field = $body->appendChild($field);
+        for($i=0;$i<$fields_count;$i++){
+            $field_nt .= $this->field_names[$i].":".
+        $this->page_info[$this->logfields_type_ddm[$this->field_types[$i]]]
+        ."<br/>";
+        }
+        $text1 = $dom->createTextNode($field_nt);
+        $text1 = $field->appendChild($text1);
+        $desc = $dom->createElement('p');
+        $desc = $body->appendChild($desc);
+        $text3 = "This is line ".$this->current_page_num;
+        $text2 = $dom->createTextNode($text3);
+        $text2 = $desc->appendChild($text2);
+        $site[self::PAGE] =$dom->saveHTML();
+        $html_page = "<html><head><title>LogRecord".$this->current_page_num.
+                "</title></head><body><h1>".
+                "The details of the log record are: <br/></h1><h3>".
+                $field_nt."</h3></body></html>";
+        file_put_contents("{$this->index_dir}/HTML_FILES/logrecord".
+                $this->current_page_num.".php",$html_page);
+        $site[self::URL] = "http://localhost/".$this->path_for_html_files.
+                "cache/IndexData".$this->result_timestamp.
+                "/HTML_FILES/logrecord"
+                .$this->current_page_num.".php";
+        $site[self::TYPE] ="text/html";
+        $site[self::HTTP_CODE] = 200;
+        $site[self::ENCODING] = "UTF-8";
+        $site[self::SERVER] = "unknown";
+        $site[self::SERVER_VERSION] = "unknown";
+        $site[self::OPERATING_SYSTEM] = "unknown";
+        $site[self::HASH] = FetchUrl::computePageHash($site[self::PAGE]);
+        $site[self::WEIGHT] = 1;
+
+        return $site;
+    }
+}
+?>
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
index 6a833c0..09c6bcc 100644
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -210,6 +210,9 @@ interface CrawlConstants
     const LINK_SEEN_URLS = 'cj';
     const POST_MAX_SIZE = 'ck';
     const LOGGING = 'cl';
+    const LOG_RECORDS = 'cm';
+    const DATABASE_RECORDS = 'cn';
+    const DATABASE_CONNECTION_DETAILS = 'co';
 
     const NEEDS_OFFSET_FLAG = 0x7FFFFFFF;
 
diff --git a/locale/en-US/configure.ini b/locale/en-US/configure.ini
index 796ba79..0b5abc8 100755
--- a/locale/en-US/configure.ini
+++ b/locale/en-US/configure.ini
@@ -26,7 +26,7 @@
 ;
 [strings]
 ;
-; /Applications/XAMPP/xamppfiles/htdocs/git/yioop//controllers
+; C:/xampp/htdocs/yioop//controllers
 ;
 ; admin_controller.php line: 132
 admin_controller_login_successful = "Login Successful!!"
@@ -127,277 +127,277 @@ admin_controller_activity_deleted = "Activity Deleted"
 ; admin_controller.php line: 750
 admin_controller_starting_new_crawl = "Starting New Crawl!"
 ;
-; admin_controller.php line: 806
+; admin_controller.php line: 815
 admin_controller_no_description = "No Description for Crawl"
 ;
-; admin_controller.php line: 832
+; admin_controller.php line: 841
 admin_controller_stop_crawl = "Stopping crawl. . .This will take a moment to refresh."
 ;
-; admin_controller.php line: 846
+; admin_controller.php line: 855
 admin_controller_resume_crawl = "Resuming crawl. . .This will take a moment to refresh."
 ;
-; admin_controller.php line: 868
+; admin_controller.php line: 877
 admin_controller_delete_crawl_success = "Deleting Crawl. . .This will take a moment to refresh."
 ;
-; admin_controller.php line: 873
+; admin_controller.php line: 882
 admin_controller_delete_crawl_fail = "Delete Crawl Failed!!"
 ;
-; admin_controller.php line: 880
+; admin_controller.php line: 889
 admin_controller_set_index = "Setting Crawl To Use as Index"
 ;
-; admin_controller.php line: 907
+; admin_controller.php line: 916
 admin_controller_use_below = "Use options below"
 ;
-; admin_controller.php line: 908
+; admin_controller.php line: 917
 admin_controller_use_defaults = "Use Yioop! defaults"
 ;
-; admin_controller.php line: 910
+; admin_controller.php line: 919
 admin_controller_use_below = "Use options below"
 ;
-; admin_controller.php line: 913
+; admin_controller.php line: 922
 admin_controller_previous_crawl = "Previous Crawl:"
 ;
-; admin_controller.php line: 1004
+; admin_controller.php line: 1013
 admin_controller_breadth_first = "Breadth First"
 ;
-; admin_controller.php line: 1006
+; admin_controller.php line: 1015
 admin_controller_page_importance = "Page Importance"
 ;
-; admin_controller.php line: 1110
+; admin_controller.php line: 1163
 admin_controller_urls_injected = "Urls Injected!"
 ;
-; admin_controller.php line: 1121
+; admin_controller.php line: 1174
 admin_controller_update_seed_info = "Updating Seed Site Info!"
 ;
-; admin_controller.php line: 1206
+; admin_controller.php line: 1259
 admin_controller_select_crawl = "Select Crawl"
 ;
-; admin_controller.php line: 1207
+; admin_controller.php line: 1260
 admin_controller_default_crawl = "Default Crawl"
 ;
-; admin_controller.php line: 1209
+; admin_controller.php line: 1262
 admin_controller_select_crawl = "Select Crawl"
 ;
-; admin_controller.php line: 1211
+; admin_controller.php line: 1264
 admin_controller_default_crawl = "Default Crawl"
 ;
-; admin_controller.php line: 1238
+; admin_controller.php line: 1291
 admin_controller_unnamed = "Unnamed Crawl"
 ;
-; admin_controller.php line: 1243
+; admin_controller.php line: 1296
 admin_controller_mix_created = "Crawl Mix Created!"
 ;
-; admin_controller.php line: 1252
+; admin_controller.php line: 1305
 admin_controller_set_index = "Setting Crawl To Use as Index"
 ;
-; admin_controller.php line: 1262
+; admin_controller.php line: 1315
 admin_controller_mix_doesnt_exists = "Mix to Delete Does not Exist!"
 ;
-; admin_controller.php line: 1270
+; admin_controller.php line: 1323
 admin_controller_mix_deleted = "Crawl Mix Deleted!"
 ;
-; admin_controller.php line: 1306
+; admin_controller.php line: 1359
 editmix_element_add_crawls = "Add Crawls"
 ;
-; admin_controller.php line: 1308
+; admin_controller.php line: 1361
 editmix_element_num_results = "Number of Results"
 ;
-; admin_controller.php line: 1309
+; admin_controller.php line: 1362
 editmix_element_del_grp = "Delete Group"
 ;
-; admin_controller.php line: 1310
+; admin_controller.php line: 1363
 editmix_element_weight = "Weight"
 ;
-; admin_controller.php line: 1311
+; admin_controller.php line: 1364
 editmix_element_name = "Name"
 ;
-; admin_controller.php line: 1312
+; admin_controller.php line: 1365
 editmix_add_keywords = "Keywords"
 ;
-; admin_controller.php line: 1313
+; admin_controller.php line: 1366
 editmix_element_actions = "Actions"
 ;
-; admin_controller.php line: 1314
+; admin_controller.php line: 1367
 editmix_add_query = "Add Query"
 ;
-; admin_controller.php line: 1315
+; admin_controller.php line: 1368
 editmix_element_delete = "Delete"
 ;
-; admin_controller.php line: 1367
+; admin_controller.php line: 1420
 admin_controller_mix_saved = "Crawl Mix Changes Saved!"
 ;
-; admin_controller.php line: 1439
+; admin_controller.php line: 1492
 admin_controller_recrawl_never = "Never"
 ;
-; admin_controller.php line: 1440
+; admin_controller.php line: 1493
 admin_controller_recrawl_1day = "1 days"
 ;
-; admin_controller.php line: 1441
+; admin_controller.php line: 1494
 admin_controller_recrawl_2day = "2 days"
 ;
-; admin_controller.php line: 1442
+; admin_controller.php line: 1495
 admin_controller_recrawl_3day = "3 days"
 ;
-; admin_controller.php line: 1443
+; admin_controller.php line: 1496
 admin_controller_recrawl_7day = "7 days"
 ;
-; admin_controller.php line: 1444
+; admin_controller.php line: 1497
 admin_controller_recrawl_14day = "14 days"
 ;
-; admin_controller.php line: 1499
+; admin_controller.php line: 1552
 admin_controller_page_options_updated = "Page Options Updated!"
 ;
-; admin_controller.php line: 1531
+; admin_controller.php line: 1584
 admin_controller_results_editor_update = "Filter Pages Updated!"
 ;
-; admin_controller.php line: 1545
+; admin_controller.php line: 1598
 admin_controller_edited_pages = "Select a Previously Edited URL"
 ;
-; admin_controller.php line: 1558
+; admin_controller.php line: 1611
 admin_controller_results_editor_need_url = "Result Page Update needs to Specify the URL!"
 ;
-; admin_controller.php line: 1564
+; admin_controller.php line: 1617
 admin_controller_results_editor_page_updated = "Result Page Updated!"
 ;
-; admin_controller.php line: 1577
+; admin_controller.php line: 1630
 admin_controller_results_editor_page_loaded = "Page Loaded!"
 ;
-; admin_controller.php line: 1623
+; admin_controller.php line: 1676
 admin_controller_select_machine = "Select Machine"
 ;
-; admin_controller.php line: 1695
+; admin_controller.php line: 1748
 admin_controller_machine_added = "Machine Added!"
 ;
-; admin_controller.php line: 1702
+; admin_controller.php line: 1755
 admin_controller_machine_exists = "Machine Name Already Exists; Please Delete First!"
 ;
-; admin_controller.php line: 1706
+; admin_controller.php line: 1759
 admin_controller_machine_incomplete = "Missing Fields From Machine Form!"
 ;
-; admin_controller.php line: 1715
+; admin_controller.php line: 1768
 admin_controller_machine_doesnt_exists = "Machine Name does not Exists!"
 ;
-; admin_controller.php line: 1732
+; admin_controller.php line: 1785
 admin_controller_stop_service_first = "Machine in use. Please stop the service running on it!"
 ;
-; admin_controller.php line: 1745
+; admin_controller.php line: 1798
 admin_controller_machine_deleted = "Machine Deleted!"
 ;
-; admin_controller.php line: 1793
+; admin_controller.php line: 1846
 admin_controller_no_machine_log = "No Log File Found."
 ;
-; admin_controller.php line: 1822
+; admin_controller.php line: 1875
 admin_controller_machine_servers_updated = "Machine&#039;s Servers Updated!"
 ;
-; admin_controller.php line: 1826
+; admin_controller.php line: 1879
 admin_controller_machine_no_action = "Unable to Perform Action!"
 ;
-; admin_controller.php line: 1857
+; admin_controller.php line: 1910
 admin_controller_select_localename = "Select Locale"
 ;
-; admin_controller.php line: 1901
+; admin_controller.php line: 1954
 admin_controller_locale_added = "Locale Added!"
 ;
-; admin_controller.php line: 1908
+; admin_controller.php line: 1961
 admin_controller_localename_doesnt_exists = "Locale Does Not Exist!"
 ;
-; admin_controller.php line: 1917
+; admin_controller.php line: 1970
 admin_controller_localename_deleted = "Locale Deleted"
 ;
-; admin_controller.php line: 1926
+; admin_controller.php line: 1979
 admin_controller_select_staticpages = "Select a page"
 ;
-; admin_controller.php line: 1945
+; admin_controller.php line: 1998
 admin_controller_staticpage_updated = "Static Page Updated!"
 ;
-; admin_controller.php line: 1972
+; admin_controller.php line: 2025
 admin_controller_localestrings_updated = "Locale Strings Updated!"
 ;
-; admin_controller.php line: 2029
+; admin_controller.php line: 2082
 admin_controller_php_version = "PHP Version 5.3 or Newer"
 ;
-; admin_controller.php line: 2037
+; admin_controller.php line: 2090
 admin_controller_no_write_config_php = "configs/config.php not web server writable."
 ;
-; admin_controller.php line: 2042
+; admin_controller.php line: 2095
 admin_controller_no_write_work_dir = "Work directory needs to be writable by web server. "
 ;
-; admin_controller.php line: 2047
+; admin_controller.php line: 2100
 admin_controller_post_size_small = "php.ini file variable post_max_size should be at least 2M"
 ;
-; admin_controller.php line: 2053
+; admin_controller.php line: 2106
 admin_controller_missing_required = "The following required items were missing: %s"
 ;
-; admin_controller.php line: 2076
+; admin_controller.php line: 2129
 admin_controller_missing_optional = "The following optional items were missing: %s"
 ;
-; admin_controller.php line: 2081
+; admin_controller.php line: 2134
 admin_controller_check_passed = "Check Passed."
 ;
-; admin_controller.php line: 2086
+; admin_controller.php line: 2139
 admin_controller_using_local_config = "Using configs/local_config.php so changing work directory above may not work."
 ;
-; admin_controller.php line: 2111
+; admin_controller.php line: 2164
 admin_controller_media_kind = "Media Kind"
 ;
-; admin_controller.php line: 2112
+; admin_controller.php line: 2165
 admin_controller_video = "Video"
 ;
-; admin_controller.php line: 2113
+; admin_controller.php line: 2166
 admin_controller_rss_feed = "RSS"
 ;
-; admin_controller.php line: 2127
+; admin_controller.php line: 2180
 admin_controller_sources_indexes = "Index/Mix to Use"
 ;
-; admin_controller.php line: 2180
+; admin_controller.php line: 2233
 admin_controller_media_source_added = "Media Source Added!"
 ;
-; admin_controller.php line: 2188
+; admin_controller.php line: 2241
 admin_controller_media_source_deleted = "Media Source Deleted!"
 ;
-; admin_controller.php line: 2204
+; admin_controller.php line: 2257
 admin_controller_subsearch_added = "Subsearch Added!"
 ;
-; admin_controller.php line: 2212
+; admin_controller.php line: 2265
 admin_controller_subsearch_deleted = "Subsearch Deleted!"
 ;
-; admin_controller.php line: 2281
+; admin_controller.php line: 2334
 admin_controller_configure_use_absolute_path = "Must use an Absolute path for Work Directory"
 ;
-; admin_controller.php line: 2293
+; admin_controller.php line: 2346
 admin_controller_configure_diff_base_dir = "Work Directory cannot be contained in Yioop folder!"
 ;
-; admin_controller.php line: 2327
+; admin_controller.php line: 2380
 admin_controller_configure_work_dir_set = "Work Directory Set! You may need to re-login!"
 ;
-; admin_controller.php line: 2341
+; admin_controller.php line: 2394
 admin_controller_name_your_bot = "Please Name Your robot"
 ;
-; admin_controller.php line: 2349
+; admin_controller.php line: 2402
 admin_controller_configure_work_profile_made = "Working Directory and Profile Created!"
 ;
-; admin_controller.php line: 2358
+; admin_controller.php line: 2411
 admin_controller_configure_no_set_config = "Unable to Update config.php File!"
 ;
-; admin_controller.php line: 2370
+; admin_controller.php line: 2423
 admin_controller_configure_no_create_profile = "Unable to Create Profile!"
 ;
-; admin_controller.php line: 2381
+; admin_controller.php line: 2434
 admin_controller_configure_work_dir_invalid = "Work Directory is Invalid! Cannot Create Profile!"
 ;
-; admin_controller.php line: 2393
+; admin_controller.php line: 2446
 admin_controller_configure_work_dir_invalid = "Work Directory is Invalid! Cannot Create Profile!"
 ;
-; admin_controller.php line: 2472
+; admin_controller.php line: 2525
 admin_controller_configure_no_change_db = "Problem Updating Database!"
 ;
-; admin_controller.php line: 2487
+; admin_controller.php line: 2540
 admin_controller_configure_profile_change = "Profile Updated!"
 ;
-; admin_controller.php line: 2502
+; admin_controller.php line: 2555
 admin_controller_configure_no_change_profile = "There was a Problem Updating Profile!"
 ;
-; admin_controller.php line: 2540
+; admin_controller.php line: 2593
 admin_controller_describe_robot = "Please Describe Your Robot"
 ;
 ; search_controller.php line: 210
@@ -409,58 +409,58 @@ search_controller_mix_info = "Using Crawl Mix: %s"
 ; search_controller.php line: 352
 search_controller_crawl_info = "Index: %s -- Size: %s pages/%s urls"
 ;
-; search_controller.php line: 397
+; search_controller.php line: 400
 search_controller_search = "Search"
 ;
-; search_controller.php line: 513
+; search_controller.php line: 516
 search_controller_no_index_set = "No Search Index Set For Use!"
 ;
-; search_controller.php line: 516
+; search_controller.php line: 519
 search_controller_no_index_set = "No Search Index Set For Use!"
 ;
-; search_controller.php line: 1179
+; search_controller.php line: 1182
 search_controller_extracted_title = "Extracted Title"
 ;
-; search_controller.php line: 1181
+; search_controller.php line: 1184
 search_controller_extracted_description = "Extracted Description"
 ;
-; search_controller.php line: 1183
+; search_controller.php line: 1186
 search_controller_extracted_links = "Extracted Links"
 ;
-; search_controller.php line: 1188
+; search_controller.php line: 1191
 search_controller_extracted_allow_paths = "Extracted Allowed To Crawl Paths"
 ;
-; search_controller.php line: 1194
+; search_controller.php line: 1197
 search_controller_extracted_disallow_paths = "Extracted Disallowed To Crawl Paths"
 ;
-; search_controller.php line: 1200
+; search_controller.php line: 1203
 search_controller_crawl_delay = "YioopBot Crawl Delay"
 ;
-; search_controller.php line: 1239
+; search_controller.php line: 1242
 search_controller_no_archive_page = "The website in question has requested this page not be archived."
 ;
-; search_controller.php line: 1269
+; search_controller.php line: 1272
 search_controller_original_page = "This image appeared on the page:"
 ;
-; search_controller.php line: 1300
+; search_controller.php line: 1303
 search_controller_cache_comment = "Yioop Cache Page... This page has been modified to add a robots directive,  make links absolute, add extracted summaries, and to highlight query terms."
 ;
-; search_controller.php line: 1355
+; search_controller.php line: 1358
 search_controller_cached_version = "This cached version of %s was obtained by the Yioop crawler on %s."
 ;
-; search_controller.php line: 1367
+; search_controller.php line: 1370
 search_controller_header_summaries = "Toggle Extracted Headers and Summaries"
 ;
-; search_controller.php line: 1460
+; search_controller.php line: 1463
 search_controller_history = "Toggle History"
 ;
-; search_controller.php line: 1590
+; search_controller.php line: 1593
 search_controller_all_cached = "All Cached Versions - Change Year and/or Months to see Links"
 ;
-; search_controller.php line: 1621
+; search_controller.php line: 1624
 search_controller_year = "Year:"
 ;
-; search_controller.php line: 1622
+; search_controller.php line: 1625
 search_controller_month = "Month:"
 ;
 ; settings_controller.php line: 134
@@ -469,7 +469,7 @@ settings_controller_crawl_mix = "crawl mix"
 ; settings_controller.php line: 151
 settings_controller_settings_saved = "Settings Saved!"
 ;
-; /Applications/XAMPP/xamppfiles/htdocs/git/yioop//views
+; C:/xampp/htdocs/yioop//views
 ;
 ; admin_view.php line: 88
 admin_view_admin = "Admin"
@@ -600,7 +600,7 @@ crawlstatus_view_delete = "Delete"
 ; crawlstatus_view.php line: 243
 crawlstatus_view_no_previous_crawl = "No Previous Crawls"
 ;
-; /Applications/XAMPP/xamppfiles/htdocs/git/yioop//views/elements
+; C:/xampp/htdocs/yioop//views/elements
 ;
 ; activity_element.php line: 57
 activity_element_activities = "Activities"
@@ -761,31 +761,91 @@ crawloptions_element_reindex_crawl = "Crawl or Arc Folder to Re-index:"
 ; crawloptions_element.php line: 159
 crawloptions_element_need_api_for_mix = "Yioop API access required for mix archive crawls  "
 ;
-; crawloptions_element.php line: 165
+; crawloptions_element.php line: 205
+crawloptions_element_first_line_text = "The first line in the file is (format of every record in each log file):"
+;
+; crawloptions_element.php line: 211
+crawloptions_element_log_records_details = "Log Records Details"
+;
+; crawloptions_element.php line: 214
+crawloptions_element_field = "Field"
+;
+; crawloptions_element.php line: 215
+crawloptions_element_field_name = "Field Name"
+;
+; crawloptions_element.php line: 216
+crawloptions_element_field_type = "Field Type"
+;
+; crawloptions_element.php line: 224
+crawloptions_element_field = "Field"
+;
+; crawloptions_element.php line: 231
+crawloptions_element_field_name = "Field Name"
+;
+; crawloptions_element.php line: 323
+crawloptions_element_add_new_field = "Add new field"
+;
+; crawloptions_element.php line: 350
+crawloptions_element_database_connection_details = "Database Connection Details"
+;
+; crawloptions_element.php line: 355
+crawloptions_element_hostname = "Hostname"
+;
+; crawloptions_element.php line: 359
+crawloptions_element_hostname = "Hostname"
+;
+; crawloptions_element.php line: 367
+crawloptions_element_username = "Username"
+;
+; crawloptions_element.php line: 371
+crawloptions_element_username = "Username"
+;
+; crawloptions_element.php line: 379
+crawloptions_element_password = "Password"
+;
+; crawloptions_element.php line: 383
+crawloptions_element_password = "Password"
+;
+; crawloptions_element.php line: 391
+crawloptions_element_databasename = "Database Name"
+;
+; crawloptions_element.php line: 395
+crawloptions_element_databasename = "Database Name"
+;
+; crawloptions_element.php line: 403
+crawloptions_element_query = "Query"
+;
+; crawloptions_element.php line: 407
+crawloptions_element_query = "Query"
+;
+; crawloptions_element.php line: 418
+crawloptions_element_submit = "Submit"
+;
+; crawloptions_element.php line: 445
 crawloptions_element_meta_words = "Meta Words"
 ;
-; crawloptions_element.php line: 167
+; crawloptions_element.php line: 447
 crawloptions_element_word = "Word"
 ;
-; crawloptions_element.php line: 169
+; crawloptions_element.php line: 449
 crawloptions_element_url_pattern = "URL Pattern"
 ;
-; crawloptions_element.php line: 175
+; crawloptions_element.php line: 455
 crawloptions_element_word = "Word"
 ;
-; crawloptions_element.php line: 181
+; crawloptions_element.php line: 461
 crawloptions_element_url_pattern = "URL Pattern"
 ;
-; crawloptions_element.php line: 203
+; crawloptions_element.php line: 483
 crawloptions_element_indexing_plugins = "Indexing Plugins"
 ;
-; crawloptions_element.php line: 205
+; crawloptions_element.php line: 485
 crawloptions_element_plugin = "Plugin"
 ;
-; crawloptions_element.php line: 208
+; crawloptions_element.php line: 488
 crawloptions_element_plugin_include = "Use in Crawl"
 ;
-; crawloptions_element.php line: 232
+; crawloptions_element.php line: 512
 crawloptions_element_save_options = "Save Options"
 ;
 ; editlocales_element.php line: 62
@@ -1250,7 +1310,7 @@ subsearch_element_more = "More"
 ; subsearch_element.php line: 87
 subsearch_element_more = "More"
 ;
-; /Applications/XAMPP/xamppfiles/htdocs/git/yioop//views/helpers
+; C:/xampp/htdocs/yioop//views/helpers
 ;
 ; feeds_helper.php line: 70
 feeds_helper_view_feed_results = "News Results for %s"
@@ -1288,7 +1348,7 @@ toggle_helper_on = "On"
 ; toggle_helper.php line: 81
 toggle_helper_off = "Off"
 ;
-; /Applications/XAMPP/xamppfiles/htdocs/git/yioop//views/layouts
+; C:/xampp/htdocs/yioop//views/layouts
 ;
 ; rss_layout.php line: 64
 rss_layout_title = "PHP Search Engine - Yioop! : %s"
diff --git a/views/elements/crawloptions_element.php b/views/elements/crawloptions_element.php
index 66a8d93..118e9e2 100644
--- a/views/elements/crawloptions_element.php
+++ b/views/elements/crawloptions_element.php
@@ -159,6 +159,286 @@ class CrawloptionsElement extends Element
             <div class="center red"><?php 
             e(tl('crawloptions_element_need_api_for_mix')); ?></div>
         <?php } ?>
+
+        <script>
+        obj = document.getElementById("crawl-indexes");
+        obj.onchange = function(){crawloptionsForm.submit();}
+        </script>
+
+        <?php $data['logfields_type'] = array(
+        1=>'IP_Address',
+        2=>'Timestamp',
+        3=>'URL',
+        4=>'Status Code',
+        5=>'User Agent',
+        6=>'Request',
+        7=>'Int');
+
+        $flag = false;
+
+        /* If log files are selected as the option */ 
+        if(isset($_POST['crawl_indexes'])
+            && $data['available_crawl_indexes'][$_POST['crawl_indexes']] 
+            == 'ARCFILE::Log Files') {
+                $LogFolderPath = CRAWL_DIR.'/cache/archives';
+                foreach(glob($LogFolderPath."/*") as $folder){
+                  if(is_dir($folder)){
+                    if(file_exists("$folder/arc_description.ini")){
+                      $contents = 
+                        file_get_contents("$folder/arc_description.ini");
+                      if(strpos($contents,"LogArchiveBundle")
+                        == true){
+                        $flag = true;
+                        $LogFolderPath = $folder;
+                      }
+                    }
+                  }
+                  if($flag == true) {break;}
+                }
+
+        /*Get all the file names into an array*/
+        $filenames = glob($LogFolderPath."/*.log");
+        /*Retrieve the first filename*/
+        $firstFile = $filenames[0];
+        /*Split the file content into an array*/
+        $l_delim = "\n";
+        $file_array = explode($l_delim, file_get_contents($firstFile));
+        echo "<br/><b>".tl('crawloptions_element_first_line_text')."</b><br/>";
+        echo "<br/>".$file_array[0]."<br/>";
+
+        ?>
+
+        <div id="Log_Records" class="topmargin"><b><?php 
+            e(tl('crawloptions_element_log_records_details'))?></b></div>
+
+        <table class="logrecordstable">
+            <tr><th><?php e(tl('crawloptions_element_field'));?></th>
+            <th><?php e(tl('crawloptions_element_field_name')); ?></th>
+            <th><?php e(tl('crawloptions_element_field_type')); ?></th></tr>
+        <?php
+            $i = 0;
+            foreach($data['LOG_RECORDS'] as $field => $field_nt) {
+                $matches = explode("::",$field_nt);
+        ?>
+            <tr><td class="input-field" >
+                <input
+                       title="<?php e(tl('crawloptions_element_field')); ?>" 
+                       name="LOG_RECORDS[<?php e($i); ?>][FIELD]" 
+                       value="<?php e($field); ?>" 
+                />
+                </td>
+                <td class="input-field-name">
+                <input 
+                     title="<?php e(tl('crawloptions_element_field_name')); ?>"
+                     name="LOG_RECORDS[<?php e($i); ?>]['FIELD_NAME']" 
+                     value="<?php e($matches[0]); ?>" 
+                />
+                </td>
+                <td class="input-field-type" >
+                <?php $this->view->optionsHelper->render(
+                        'field-types',
+                        "LOG_RECORDS[$i]['FIELD_TYPE']",
+                        $data['logfields_type'],
+                        $matches[1]);
+                ?>
+                </td>
+            </tr>
+            <?php
+                $i++;
+                }
+                if($i==0){
+            ?>
+            <tr><td class="input-field">
+                <input
+                       type="text" 
+                       title="New Field" 
+                       name="LOG_RECORDS[<?php e($i); ?>][FIELD]" 
+                       value=""
+                />
+                </td>
+                <td class="input-field-name">
+                <input
+                       type="text" 
+                       title="New Field Name" 
+                       name="LOG_RECORDS[<?php e($i); ?>]['FIELD_NAME']" 
+                       value=""
+                />
+                </td>
+                <td class="input-field-type">
+                <?php $this->view->optionsHelper->render(
+                        'field-types',
+                        "LOG_RECORDS[$i]['FIELD_TYPE']",
+                        $data['logfields_type'],
+                        1);
+                ?>
+                </td>
+            </tr>
+
+                <?php } ?>
+
+                <?php
+                    if(isset($_POST['add_fields']) && $i>0){
+                ?>
+            <tr>
+                <td class="input-field">
+                <input
+                       type="text" 
+                       title="New Field" 
+                       name="LOG_RECORDS[<?php e($i); ?>]['FIELD']" 
+                       value=""
+                />
+                </td>
+                <td class="input-field-name">
+                <input
+                       type="text" 
+                       title="New Field Name" 
+                       name="LOG_RECORDS[<?php e($i); ?>]['FIELD_NAME']" 
+                       value=""
+                />
+                </td>
+                <td class="input-field-type">
+                <?php $this->view->optionsHelper->render(
+                        'field-types',
+                        "LOG_RECORDS[$i]['FIELD_TYPE']",
+                        $data['logfields_type'],
+                        1);
+                ?>
+                </td>
+            </tr>
+
+                <?php } ?>
+        </table>
+        <?php
+        if(isset($_POST['save_options']) 
+        && $data['available_crawl_indexes'][$_POST['crawl_indexes']] 
+        == 'ARCFILE::Log Files'){
+            file_put_contents($LogFolderPath."/fields_data.txt",
+                serialize($data['LOG_RECORDS']));
+        }
+        ?>
+        <div class="log-record-new-field">
+        <input
+               type="submit" 
+               id="add-fields" 
+               name="add_fields" 
+               value="<?php e(tl('crawloptions_element_add_new_field')); ?>"
+        />
+        </div>
+        <?php } ?>
+
+        <?php
+        if(isset($_POST['crawl_indexes']) && 
+            $data['available_crawl_indexes'][$_POST['crawl_indexes']] 
+            == 'ARCFILE::Database files') {
+            $flag1 = false;
+            $DatabaseFolderPath = CRAWL_DIR.'/cache/archives';
+            foreach(glob($DatabaseFolderPath."/*") as $folder){
+                if(is_dir($folder)){
+                    if(file_exists("$folder/arc_description.ini")){
+                        $contents = 
+                            file_get_contents("$folder/arc_description.ini");
+                        if(strpos($contents,"Database files") == true){
+                            $flag1 = true;
+                            $DatabaseFolderPath = $folder;
+                        }
+                    }
+                }
+                if($flag1 == true) {break;}
+            }
+        ?>
+
+        <div id="Database_Connection_Details" class="topmargin"><b><?php
+            e(tl('crawloptions_element_database_connection_details'))?></b>
+        </div><br/>
+
+        <table class="databaseconnectiondetailstable">
+           <tr><td class="input-name"><?php 
+                    e(tl('crawloptions_element_hostname'))?>
+                </td>
+                <td class="input-data">
+                <input
+                       title="<?php e(tl('crawloptions_element_hostname')); ?>" 
+                       name="DATABASE_CONNECTION_DETAILS[HOSTNAME]" 
+                       value="" 
+                />
+                </td>
+            </tr>
+            <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_username'))?>
+                </td>
+                <td class="input-data">
+                <input
+                       title="<?php e(tl('crawloptions_element_username')); ?>" 
+                       name="DATABASE_CONNECTION_DETAILS[USERNAME]" 
+                       value="" 
+                />
+                </td>
+            </tr>
+            <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_password'))?>
+                </td>
+                <td class="input-data">
+                <input
+                       title="<?php e(tl('crawloptions_element_password')); ?>" 
+                       name="DATABASE_CONNECTION_DETAILS[PASSWORD]" 
+                       value="" 
+                />
+                </td>
+            </tr>
+            <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_databasename'))?>
+                </td>
+                <td class="input-data">
+                <input
+                   title= "<?php e(tl('crawloptions_element_databasename')); ?>"
+                   name="DATABASE_CONNECTION_DETAILS[DATABASENAME]" 
+                   value="" 
+                />
+                </td>
+            </tr>
+            <tr>
+                <td class="input-name"><?php 
+                    e(tl('crawloptions_element_query'))?>
+                </td>
+                <td class="input-data">
+                <input 
+                       title="<?php e(tl('crawloptions_element_query')); ?>" 
+                       name="DATABASE_CONNECTION_DETAILS[QUERY]" 
+                       value="" 
+                />
+                </td>
+            </tr>
+        </table>
+        <div class="database-connection-details-submit">
+            <input type="submit" 
+                   id="submit-details" 
+                   name="submit_details" 
+                   value="<?php e(tl('crawloptions_element_submit')); ?>"
+            />
+        </div>
+
+        <?php 
+        if(isset($_POST['submit_details'])){
+            file_put_contents($DatabaseFolderPath.
+                "/database_connection_details.txt",
+                serialize($data['DATABASE_CONNECTION_DETAILS']));
+        }
+        ?>
+        <?php
+        if(isset($_POST['save_options']) 
+            && $data['available_crawl_indexes'][$_POST['crawl_indexes']] 
+            == 'ARCFILE::Database files'){
+            file_put_contents($DatabaseFolderPath.
+                "/database_connection_details.txt", 
+                serialize($data['DATABASE_CONNECTION_DETAILS']));
+        }
+        ?>
+
+        <?php } ?>
+
         </div>
         <?php } ?>
         </div>
@@ -230,7 +510,8 @@ class CrawloptionsElement extends Element
             
             
         <div class="center slightpad"><button class="buttonbox" 
-            type="submit"><?php e(tl('crawloptions_element_save_options')); 
+            type="submit" name="save_options">
+            <?php e(tl('crawloptions_element_save_options')); 
             ?></button></div>
         </form>
         </div>
-- 
1.7.11.msysgit.1

