_binary From 36a09f1f8347ea4909ea2e403c60895be4a94a53 Mon Sep 17 00:00:00 2001
From: pomishra13 <pooja192009@gmail.com>
Date: Mon, 21 Sep 2015 20:16:16 -0700
Subject: [PATCH] media_job changes

---
 src/controllers/crawl_controller.php    | 492 +++++++++++++++++++++++
 src/controllers/resource_controller.php | 413 +++++++++++++++++++
 src/executables/media_updater.php       | 688 ++++++++++++++++++++++++++++++++
 src/library/jobs/media_job.php          |  35 ++
 src/library/jobs/news_updater_job.php   | 110 +++++
 src/library/jobs/video_updater_job.php  | 435 ++++++++++++++++++++
 6 files changed, 2173 insertions(+)
 create mode 100644 src/controllers/crawl_controller.php
 create mode 100644 src/controllers/resource_controller.php
 create mode 100644 src/executables/media_updater.php
 create mode 100644 src/library/jobs/media_job.php
 create mode 100644 src/library/jobs/news_updater_job.php
 create mode 100644 src/library/jobs/video_updater_job.php

diff --git a/src/controllers/crawl_controller.php b/src/controllers/crawl_controller.php
new file mode 100644
index 0000000..20f0fdf
--- /dev/null
+++ b/src/controllers/crawl_controller.php
@@ -0,0 +1,492 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+if (!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+/** Load base controller class if needed */
+require_once BASE_DIR."/controllers/controller.php";
+/** Loads common constants for media operations*/
+require_once BASE_DIR."/lib/media_constants.php";
+/** Loads common constants for web crawling*/
+require_once BASE_DIR."/lib/crawl_constants.php";
+/** Need getHost to partition urls to different queue_servers*/
+require_once BASE_DIR."/lib/url_parser.php";
+/**
+ * Controller used to manage networked installations of Yioop where
+ * there might be mutliple queue_servers and a name_server. Command
+ * sent to the nameserver web page are mapped out to queue_servers
+ * using this controller. Each method of the controller essentially
+ * mimics one method of CrawlModel, PhraseModel, or in general anything
+ * that extends ParallelModel and is used to proxy that information
+ * through a result web page back to the name_server.
+ *
+ * @author Chris Pollett
+ * @package seek_quarry\controller
+ */
+class CrawlController extends Controller implements CrawlConstants,
+    MediaConstants
+{
+    /**
+     * These are the activities supported by this controller
+     * @var array
+     */
+    var $activities = array("countWords","clearQuerySavePoint",
+        "crawlStalled", "crawlStatus", "deleteCrawl", "injectUrlsCurrentCrawl",
+        "combinedCrawlInfo", "getInfoTimestamp", "getCrawlItems",
+        "getCrawlList", "getCrawlSeedInfo", "getMailingListFile",
+         "getSplitFileForConversion", "getUpdateProperties",
+        "sendStartCrawlMessage", "sendStopCrawlMessage", "setCrawlSeedInfo",
+        );
+        
+    var $jobs=array("news_updater_job","video_updater_job");
+    var $methods = array("getNewsSources","getSplitFileForConversion");
+    /**
+     * Checks that the request seems to be coming from a legitimate fetcher then
+     * determines which activity the fetcher is requesting and calls that
+     * activity for processing.
+     *
+     */
+    function processRequest()
+    {
+        $data = array();
+        /* do a quick test to see if this is a request seems like
+           from a legitimate machine
+         */
+        if (!$this->checkRequest()) {return; }
+        
+        $activity = $_REQUEST['a'];
+        if (in_array($activity, $this->activities)) {
+            $this->call($activity);
+        }
+        if(isset($_REQUEST['job_name'])){
+            $job=$_REQUEST['job_name'];
+            if (in_array($job, $this->jobs)) {               
+                require_once BASE_DIR."/lib/jobs/".$job."php";
+                $job_name= $job_name_split=str_replace("_", " ", $job);
+                $class_name=ucwords($job_name);            
+                $job_object = new $class_name;
+                if(isset($_REQUEST['method_name'])){
+                    $method=$_REQUEST['method_name'];
+                    if (in_array($method, $this->methods)) {
+                        $this->job_object->$method();
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Handles a request for whether or not the crawl is stalled on the
+     * given local server (which means no fetcher has spoken to it in a while)
+     * outputs this info back as body of the http response (url encoded,
+     * serialized php data)
+     */
+    function crawlStalled()
+    {
+        echo webencode(serialize($this->model("crawl")->crawlStalled()));
+    }
+    /**
+     * Handles a request for the crawl status (memory use, recent fetchers
+     * crawl rate, etc) data from a remote name server
+     * and retrieves that the statistic about this that are held by the
+     * local queue server
+     * outputs this info back as body of the http response (url encoded,
+     * serialized php data)
+     */
+    function crawlStatus()
+    {
+        echo webencode(serialize($this->model("crawl")->crawlStatus()));
+    }
+    /**
+     * Handles a request for the starting parameters of a crawl of a given
+     * timestamp and retrieves that information from the bundle held by the
+     * local queue server
+     * outputs this info back as body of the http response (url encoded,
+     * serialized php data)
+     */
+    function getCrawlSeedInfo()
+    {
+        $timestamp = 0;
+        if (isset($_REQUEST["arg"]) ) {
+            $timestamp = unserialize(webdecode($_REQUEST["arg"]));
+            $timestamp = substr($this->clean($timestamp, "int"), 0,
+                TIMESTAMP_LEN);
+        }
+        echo webencode(serialize($this->model("crawl")->getCrawlSeedInfo(
+            $timestamp)));
+    }
+    /**
+     * Handles a request to change the parameters of a crawl of a given
+     * timestamp on the local machine (does nothing if crawl doesn't exist)
+     */
+    function setCrawlSeedInfo()
+    {
+        if (isset($_REQUEST["arg"]) ) {
+            list($timestamp, $info) = unserialize(webdecode($_REQUEST["arg"]));
+            $timestamp = substr($this->clean($timestamp, "int"), 0,
+                TIMESTAMP_LEN);
+            if ($timestamp && $info) {
+                $this->model("crawl")->setCrawlSeedInfo($timestamp, $info);
+            }
+        }
+    }
+    /**
+     * Handles a request for information about a crawl with a given timestamp
+     * from a remote name server and retrieves statistics about this crawl
+     * that are held by the local queue server (number of pages, name, etc)
+     * outputs this info back as body of the http response (url encoded,
+     * serialized php data)
+     */
+    function getInfoTimestamp()
+    {
+        $timestamp = 0;
+        if (isset($_REQUEST["arg"]) ) {
+            $timestamp = unserialize(webdecode($_REQUEST["arg"]));
+            $timestamp = substr($this->clean($timestamp, "int"), 0,
+                TIMESTAMP_LEN);
+        }
+        echo webencode(serialize($this->model("crawl")->getInfoTimestamp(
+            $timestamp)));
+    }
+    /**
+     * Handles a request for the crawl list (what crawl are stored on the
+     * machine) data from a remote name server and retrieves the
+     * statistic about this that are held by the local queue server
+     * outputs this info back as body of the http response (url encoded,
+     * serialized php data)
+     */
+    function getCrawlList()
+    {
+        $return_arc_bundles = false;
+        $return_recrawls = false;
+        if (isset($_REQUEST["arg"]) ) {
+            $arg = trim(webdecode($_REQUEST["arg"]));
+            $arg = $this->clean($arg, "int");
+            if ($arg == 3 || $arg == 1) {$return_arc_bundles = true; }
+            if ($arg == 3 || $arg == 2) {$return_recrawls = true; }
+        }
+        echo webencode(serialize($this->model("crawl")->getCrawlList(
+            $return_arc_bundles, $return_recrawls)));
+    }
+    /**
+     * Handles a request for the combined crawl list, stalled, and status
+     * data from a remote name server and retrieves that the statistic about
+     * this that are held by the local queue server
+     * outputs this info back as body of the http response (url encoded,
+     * serialized php data)
+     */
+    function combinedCrawlInfo()
+    {
+        $combined =  $this->model("crawl")->combinedCrawlInfo();
+        echo webencode(serialize($combined));
+    }
+    /**
+     * Receives a request to delete a crawl from a remote name server
+     * and then deletes crawl on the local queue server
+     */
+    function deleteCrawl()
+    {
+        if (!isset($_REQUEST["arg"]) ) {
+            return;
+        }
+        $timestamp = substr($this->clean($timestamp, "int"), 0,
+            TIMESTAMP_LEN);
+        $timestamp = unserialize(webdecode($_REQUEST["arg"]));
+        $this->model("crawl")->deleteCrawl($timestamp);
+    }
+
+    /**
+     * Receives a request to inject new urls into the active
+     * crawl from a remote name server and then does this for
+     * the local queue server
+     */
+    function injectUrlsCurrentCrawl()
+    {
+        if (!isset($_REQUEST["arg"]) || !isset($_REQUEST["num"])
+            || !isset($_REQUEST["i"])) {
+            return;
+        }
+        $num = $this->clean($_REQUEST["num"], "int");
+        $i = $this->clean($_REQUEST["i"], "int");
+        list($timestamp, $inject_urls) =
+            unserialize(webdecode($_REQUEST["arg"]));
+        $timestamp = substr($this->clean($timestamp, "int"), 0,
+            TIMESTAMP_LEN);
+        $inject_urls = partitionByHash($inject_urls,
+            null, $num, $i, "UrlParser::getHost");
+        $this->model("crawl")->injectUrlsCurrentCrawl($timestamp,
+            $inject_urls, null);
+    }
+    /**
+     * Receives a request to get crawl summary data for an array of urls
+     * from a remote name server and then looks these up on the local
+     * queue server
+     */
+     function getCrawlItems()
+     {
+        $crawl_model = $this->model("crawl");
+        $start_time = microtime();
+        if (!isset($_REQUEST["arg"]) || !isset($_REQUEST["num"])
+            || !isset($_REQUEST["i"])) {
+            return;
+        }
+        $num = $this->clean($_REQUEST["num"], "int");
+        $i = $this->clean($_REQUEST["i"], "int");
+        $crawl_model->current_machine = $i;
+        $lookups = unserialize(webdecode($_REQUEST["arg"]));
+        $our_lookups = array();
+        foreach ($lookups as $lookup => $lookup_info) {
+            if (count($lookup_info) == 2 && ($lookup_info[0][0] === 'h'
+                || $lookup_info[0][0] === 'r'
+                || $lookup_info[0][0] === 'g')) {
+                $our_lookups[$lookup] = $lookup_info;
+            } else {
+                $our_lookups[$lookup] = array();
+                foreach ($lookup_info as $lookup_item) {
+                    if (count($lookup_item) == 2) {
+                        $our_lookups[$lookup][] = $lookup_item;
+                    } else {
+                        list($index, , , , ) = $lookup_item;
+                        if ($index == $i) {
+                            $our_lookups[$lookup][] = $lookup_item;
+                        }
+                    }
+                }
+            }
+        }
+        $items = $crawl_model->getCrawlItems($our_lookups);
+        $items["ELAPSED_TIME"] = changeInMicrotime($start_time);
+        header("Content-Type: text/plain");
+        $items = webencode(serialize($items));
+        header("Content-Length: ".strlen($items));
+        echo $items;
+        flush();
+     }
+    /**
+     * Receives a request to get counts of the number of occurrences of an
+     * array of words a remote name server and then
+     * determines and outputs these counts for the local queue server
+     */
+     function countWords()
+     {
+        if (!isset($_REQUEST["arg"]) ) {
+            return;
+        }
+        $crawl_model = $this->model("crawl");
+        list($words, $index_name) = unserialize(webdecode($_REQUEST["arg"]));
+        $crawl_model->index_name = $index_name;
+        echo webencode(serialize(
+            $crawl_model->countWords($words)));
+     }
+    /**
+     * Receives a request to stop a crawl from a remote name server
+     * and then stop the current crawl on the local queue server
+     */
+    function sendStopCrawlMessage()
+    {
+        $this->model("crawl")->sendStopCrawlMessage();
+    }
+    /**
+     * Receives a request to start a crawl from a remote name server
+     * and then starts the crawl process on the local queue server
+     */
+    function sendStartCrawlMessage()
+    {
+        if (!isset($_REQUEST["arg"]) || !isset($_REQUEST["num"])
+            || !isset($_REQUEST["i"])) {
+            return;
+        }
+        $num = $this->clean($_REQUEST["num"], "int");
+        $i = $this->clean($_REQUEST["i"], "int");
+        list($crawl_params,
+            $seed_info) = unserialize(webdecode($_REQUEST["arg"]));
+        $seed_info['seed_sites']['url'] =
+            partitionByHash($seed_info['seed_sites']['url'],
+            null, $num, $i, "UrlParser::getHost");
+        $this->model("crawl")->sendStartCrawlMessage($crawl_params, $seed_info,
+            null);
+    }
+    /**
+     * A save point is used to store to disk a sequence generation-doc-offset
+     * pairs of a particular mix query when doing an archive crawl of a crawl
+     * mix. This is used so that the mix can remember where it was the next
+     * time it is invoked by the web app on the machine in question.
+     * This function deletes such a save point associated with a timestamp
+     */
+    function clearQuerySavePoint()
+    {
+        if (!isset($_REQUEST["arg"])) {
+            return;
+        }
+        $save_timestamp = substr($this->clean($_REQUEST["arg"], "int"), 0,
+            TIMESTAMP_LEN);
+        $this->model("crawl")->clearQuerySavePoint($save_timestamp);
+    }
+    /**
+     * Used to get the update properties of a media updater. Outputs
+     * either name_server or distributed depending on whether there is
+     * only supposed to be a media updater on the name server or on all
+     * machines in the Yioop instance
+     */
+    function getUpdateProperties()
+    {
+        $profile_model = $this->model("profile");
+        $profile =  $profile_model->getProfile(WORK_DIRECTORY);
+        $response = array();
+        $response['MEDIA_MODE'] = (isset($profile['MEDIA_MODE'])) ?
+            $profile['MEDIA_MODE'] : "name_server";
+        echo webencode(serialize($response));
+    }
+    /**
+     * Handles the request to get the  array of news feed sources which hash to
+     * a particular value i.e. match with the index of requesting machine's
+     * hashed url/name from array of available machines hash
+     
+    function getNewsSources()
+    {
+        if (!isset($_REQUEST["arg"])) {
+            return;
+        }
+        $source_model = $this->model("source");
+        $current_machine = $this->clean(webdecode($_REQUEST["arg"]), "string");
+        $machine_hashes = $source_model->getMachineHashUrls();
+        $machine_index_match = array_search($current_machine, $machine_hashes);
+        if ($machine_index_match === false) {
+            echo webencode(serialize(array()));
+            return;
+        }
+        $num_machines = count($machine_hashes);
+        $pre_feeds = $source_model->getMediaSources("rss");
+        $pre_feeds = array_merge($pre_feeds,
+            $source_model->getMediaSources("html"));
+        if (!$pre_feeds) { return false; }
+        $feeds = array();
+        foreach ($pre_feeds as $pre_feed) {
+            if (!isset($pre_feed['NAME'])) {continue; }
+            $hash_int = unpack("N", crawlHash($pre_feed['NAME']));
+            if (!isset($hash_int[1])) {continue; }
+            $hash_index = ($hash_int[1]) % $num_machines;
+            if ($machine_index_match != $hash_index) {continue; }
+            $aux_parts = explode("###",
+                html_entity_decode($pre_feed['AUX_INFO']));
+            $offset = 0;
+            if ($pre_feed['TYPE'] == 'html') {
+                list($pre_feed['CHANNEL_PATH'], $pre_feed['ITEM_PATH'],
+                    $pre_feed['TITLE_PATH'], $pre_feed['DESCRIPTION_PATH'],
+                    $pre_feed['LINK_PATH'], $pre_feed['LINK_PATH']) =
+                    $aux_parts;
+                $offset = 5;
+            }
+            if (isset($aux_parts[$offset])) {
+                $pre_feed['IMAGE_XPATH'] = $aux_parts[$offset];
+            } else {
+                $pre_feed['IMAGE_XPATH'] = "";
+            }
+            $feeds[] = $pre_feed;
+        }
+        echo webencode(serialize($feeds));
+    }*/
+    
+    /**
+     * Handles the request to get the video file from the video directory for
+     * conversion. This selection is based upon if the file was taken
+     * previously or not. If it was then its timestamp is checked.
+     * Otherwise new file is sent for conversion along with its folder name.
+     
+    function getSplitFileForConversion()
+    {
+        if (!isset($_REQUEST["arg"])) { return; }
+        $current_machine = $this->clean(webdecode($_REQUEST["arg"]), "string");
+        $convert_folder = WORK_DIRECTORY.self::CONVERT_FOLDER;
+        $current_time = time();
+        foreach (glob($convert_folder . "/*") as $folder) {
+            foreach (glob($folder."/*.{mov,avi}", GLOB_BRACE) as $file) {
+                $folder_name = str_replace("$convert_folder/", "", $folder);
+                $file_name = str_replace($folder."/", "", $file);
+                $time_file_name = $folder . "/" . $file_name . ".time.txt";
+                if(file_exists($time_file_name)) {
+                    $file_time = file_get_contents($time_file_name);
+                    if ($current_time - $file_time > MAX_FILE_TIMESTAMP_LIMIT) {
+                        file_put_contents($time_file_name, $current_time);
+                        $folder_file = $folder_name . ":" . $file_name;
+                    }
+                } else {
+                    file_put_contents($time_file_name, $current_time);
+                    $folder_file = $folder_name . ":" . $file_name;
+                }
+                if (isset($folder_file)) {
+                    break 2;
+                }
+            }
+        }
+        if(isset($folder_file)) {
+            echo webencode($folder_file);
+        }
+    }*/
+    /**
+     * Handles the request to get the mailer list file for
+     * sending emails. This selection is based upon if the file was taken
+     * previously or not. If it was then it is skipped.
+     * Otherwise new file is sent for sending emails and new text file
+     * with taken prepended to the file name is generated.
+     */
+    function getMailingListFile()
+    {
+        $mail_directory = WORK_DIRECTORY.self::MAIL_FOLDER;
+        if (!file_exists($mail_directory)){ return; }
+        $files = glob($mail_directory."/*.txt");
+        foreach($files as $file){
+            $file_name = str_replace($mail_directory."/","",$file);
+            if (strpos($file_name, 'taken') !== false) {
+                continue;
+            } else {
+                $is_taken_file = $mail_directory."/taken-".$file_name;
+                if (in_array($is_taken_file, $files)){
+                    continue;
+                } else {
+                    $fp = fopen($mail_directory."/".$file_name, "a+");
+                    if (flock($fp, LOCK_EX | LOCK_NB)) {
+                        $taken_file_name = "taken-".$file_name;
+                        file_put_contents($mail_directory."/".$taken_file_name,
+                             "working on");
+                        $mailer_file = $file_name;
+                    } else {
+                        continue;
+                    }
+                }
+            }
+            if (!empty($mailer_file)){
+                break;
+            } else {
+                continue;
+            }
+        }
+        echo webencode($mailer_file);
+    }
+}
+
diff --git a/src/controllers/resource_controller.php b/src/controllers/resource_controller.php
new file mode 100644
index 0000000..191b809
--- /dev/null
+++ b/src/controllers/resource_controller.php
@@ -0,0 +1,413 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+if (!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+/** Load base controller class if needed */
+require_once BASE_DIR."/controllers/controller.php";
+/** Loads common constants for media operations*/
+require_once BASE_DIR."/lib/media_constants.php";
+/** Loads common constants for web crawling*/
+require_once BASE_DIR."/lib/crawl_constants.php";
+/** Loads url_parser to clean resource name*/
+require_once BASE_DIR."/lib/url_parser.php";
+/**
+ * Used to serve resources, css, or scripts such as images from APP_DIR
+ *
+ * @author Chris Pollett
+ * @package seek_quarry\controller
+ */
+class ResourceController extends Controller implements CrawlConstants,
+    MediaConstants
+{
+    /**
+     * These are the activities supported by this controller
+     * @var array
+     */
+    var $activities = array("get", "removeMailingListFile", 
+        "syncList", "syncNotify", "suggest");
+        
+    var $jobs=array("video_updater_job");
+    var $methods=array("uploadConvertedVideoFile");
+    /**
+     * Checks that the request seems to be coming from a legitimate fetcher
+     * or mirror server then determines which activity  is being requested
+     * and calls the method for that activity.
+     *
+     */
+    function processRequest()
+    {
+        if (!isset($_REQUEST['a']) ||
+            (!in_array($_REQUEST['a'], array("get", "suggest"))
+            && !$this->checkRequest())) {
+            return;
+        }
+        $activity = $_REQUEST['a'];
+
+        if (in_array($activity, $this->activities)) {
+            $this->call($activity);
+        }
+        
+        if(isset($_REQUEST['job_name'])){
+            $job=$_REQUEST['job_name'];
+            if (in_array($job, $this->jobs)) {
+                require_once BASE_DIR."/lib/jobs/".$job."php";
+                $job_name= $job_name_split=str_replace("_", " ", $job);
+                $class_name=ucwords($job_name);            
+                $job_object = new $class_name;
+                if(isset($_REQUEST['method_name'])){
+                    $method=$_REQUEST['method_name'];
+                    if (in_array($method, $this->methods)) {
+                        $this->job_object->$method."()";
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Gets the resource $_REQUEST['n'] from APP_DIR/$_REQUEST['f'] or
+     * CRAWL_DIR/$_REQUEST['f']  after cleaning
+     */
+    function get()
+    {
+        if (!isset($_REQUEST['n']) || !isset($_REQUEST['f'])) { return; }
+        $name = $this->clean($_REQUEST['n'], "string");
+        if (in_array($_REQUEST['f'], array("css", "scripts", "resources"))) {
+            /* notice in this case we don't check if request come from a
+               legitimate source but we do try to restrict it to being
+               a file (not a folder) in the above array
+            */
+            $base_dir = $this->getBaseFolder();
+            if (!$base_dir) {
+                header('HTTP/1.1 401 Unauthorized');
+                echo "<html><head><title>401 Unauthorized</title></head>".
+                    "<body><p>401 Unauthorized</p></body></html>";
+                return;
+            }
+            $type = UrlParser::getDocumentType($name);
+            $name = UrlParser::getDocumentFilename($name);
+            $name = ($type != "") ? "$name.$type" :$name;
+            if (isset($_REQUEST['t'])) {
+                $name .= ".jpg";
+            }
+        } else if (in_array($_REQUEST['f'], array("cache"))) {
+            /*  perform check since these request should come from a known
+                machine
+            */
+            if (!$this->checkRequest()) {
+                return;
+            }
+            $folder = $_REQUEST['f'];
+            $base_dir = CRAWL_DIR."/$folder";
+        } else if(in_array($_REQUEST['f'], array("schedules"))) {
+
+            if(!$this->checkRequest()) {
+                return;
+            }
+            $folder = $this->clean($_REQUEST['sf'], "string");
+            $folder = str_replace("..", "", $folder);
+            if (strpos($folder, 'mail') !== false){
+                $base_dir = CRAWL_DIR . self::MAIL_FOLDER;
+            } else {
+                $base_dir = CRAWL_DIR . self::CONVERT_FOLDER . "/$folder";
+            }
+        } else {
+            return;
+        }
+        if (isset($_REQUEST['o']) && isset($_REQUEST['l'])) {
+            $offset = $this->clean($_REQUEST['o'], "int");
+            $limit = $this->clean($_REQUEST['l'], "int");
+        }
+        $path = "$base_dir/$name";
+        if (file_exists($path)) {
+            $mime_type = mimeType($path);
+            $size = filesize($path);
+            $start = 0;
+            $end = $size - 1;
+            header("Content-type: $mime_type");
+            header("Accept-Ranges: bytes");
+            if (isset($_SERVER['HTTP_RANGE'])) {
+                $this->serveRangeRequest($path, $size, $start, $end);
+                return;
+            }
+            header("Content-Length: ".$size);
+            header("Content-Range: bytes $start-$end/$size");
+            if (isset($offset) && isset($limit)) {
+                echo file_get_contents($path, false, null, $offset, $limit);
+            } else {
+                readfile($path);
+            }
+        } else {
+            header("Location:./error.php");
+        }
+    }
+    /**
+     * Computes based on the request the folder that should be used to
+     * find a file during a resource get request. It also checks if user
+     * has access to the requested folder.
+     *
+     * @return mixed either a string with the folder name in it or false if
+     *      the user does not have access or that folder does not exist.
+     */
+    function getBaseFolder()
+    {
+        $folder = $this->clean($_REQUEST['f'], 'string');
+        $base_dir = APP_DIR."/$folder";
+        $add_to_path = false;
+        if (isset($_REQUEST['s']) && $folder == "resources") {
+            // handle sub-folders of resource (must be numeric)
+            $subfolder = $this->clean($_REQUEST['s'], "hash");
+            $prefix_folder = substr($subfolder, 0, 3);
+            $add_to_path = true;
+        } else if (isset($_REQUEST['g'])) {
+            $user_id = isset($_SESSION['USER_ID']) ? $_SESSION['USER_ID'] :
+                PUBLIC_USER_ID;
+            if (isset($_REQUEST['p'])) {
+                $page_id = $this->clean($_REQUEST['p'], 'string');
+            }
+            $group_id = $this->clean($_REQUEST['g'], "int");
+            $group_model = $this->model('group');
+            $group = $group_model->getGroupById($group_id, $user_id);
+            if (!$group) { return false; }
+            $hash_word = (isset($_REQUEST['t'])) ? 'thumb' : 'group';
+            $subfolder = crawlHash(
+                $hash_word . $group_id. $page_id . AUTH_KEY);
+            $prefix_folder = substr($subfolder, 0, 3);
+            $add_to_path = true;
+        }
+        if ($add_to_path) {
+            $base_dir .= "/$prefix_folder/$subfolder";
+        }
+        return $base_dir;
+    }
+    /**
+     * Code to handle HTTP range requests of resources. This allows
+     * HTTP pseudo-streaming of video. This code was inspired by:
+     * http://www.tuxxin.com/php-mp4-streaming/
+     *
+     * @param string $file Name of file to serve range request for
+     * @param int $size size of the file in bytes
+     * @param int $start starting byte location want to serve
+     * @param int $end ending byte location want ot serve
+     */
+    function serveRangeRequest($file, $size, $start, $end)
+    {
+        $current_start = $start;
+        $current_end = $end;
+        list(, $range) = explode('=', $_SERVER['HTTP_RANGE'], 2);
+        if (strpos($range, ',') !== false) {
+            header('HTTP/1.1 416 Requested Range Not Satisfiable');
+            header("Content-Range: bytes $start-$end/$size");
+            return;
+        }
+        if ($range == '-') {
+            $current_start = $size - 1;
+        } else {
+            $range = explode('-', $range);
+            $current_start = trim($range[0]);
+            $current_end = (isset($range[1]) && is_numeric(trim($range[1])))
+                ? trim($range[1]) : $size;
+            if ($current_start === "") {
+                $current_start = max(0, $size - $range[1] - 1);
+            }
+        }
+        $current_end = ($current_end > $end) ? $end : $current_end;
+        if ($current_start > $current_end || $current_start > $size - 1 ||
+            $current_end >= $size) {
+            header('HTTP/1.1 416 Requested Range Not Satisfiable');
+            header("Content-Range: bytes $start-$end/$size");
+            return;
+        }
+        $start = $current_start;
+        $end = $current_end;
+        $length = $end - $start + 1;
+        $fp = @fopen($file, 'rb');
+        fseek($fp, $start);
+        header('HTTP/1.1 206 Partial Content');
+        header("Content-Range: bytes $start-$end/$size");
+        header("Content-Length: ".$length);
+        $buffer = 8192;
+        $position = 0;
+        while(!feof($fp) && $position <= $end) {
+            $position = ftell($fp);
+            if ($position + $buffer > $end) {
+                $buffer = $end - $position + 1;
+            }
+            echo fread($fp, $buffer);
+            flush();
+        }
+        fclose($fp);
+    }
+    /**
+     * Used to get a keyword suggest trie. This sends additional
+     * header so will be decompressed on the fly
+     */
+    function suggest()
+    {
+        if (!isset($_REQUEST["locale"])){return;}
+        $locale = $_REQUEST["locale"];
+        $count = preg_match("/^[a-zA-z]{2}(-[a-zA-z]{2})?$/", $locale);
+        if ($count != 1) {return;}
+        $path = LOCALE_DIR."/$locale/resources/suggest_trie.txt.gz";
+        if (file_exists($path)) {
+            header("Content-Type: application/json");
+            header("Content-Encoding: gzip");
+            header("Content-Length: ".filesize($path));
+            readfile($path);
+        }
+    }
+    /**
+     * Used to notify a machine that another machine acting as a mirror
+     * is still alive. Data is stored in a txt file self::mirror_table_name
+     */
+    function syncNotify()
+    {
+        if (isset($_REQUEST['last_sync']) && $_REQUEST['last_sync'] > 0 ) {
+            $mirror_table_name = CRAWL_DIR."/".self::mirror_table_name;
+            $mirror_table = array();
+            $time = time();
+            if (file_exists($mirror_table_name) ) {
+                $mirror_table = unserialize(
+                    file_get_contents($mirror_table_name));
+                if (isset($mirror_table['time']) &&
+                    $mirror_table['time'] - $time > MIRROR_SYNC_FREQUENCY) {
+                    $mirror_table = array();
+                    // truncate table periodically to get rid of stale entries
+                }
+            }
+            if (isset($_REQUEST['robot_instance'])) {
+                $mirror_table['time'] = $time;
+                $mirror_table['machines'][
+                    $this->clean($_REQUEST['robot_instance'], "string")] =
+                    array($_SERVER['REMOTE_ADDR'], $_REQUEST['machine_uri'],
+                    $time,
+                    $this->clean($_REQUEST['last_sync'], "int"));
+                file_put_contents($mirror_table_name, serialize($mirror_table));
+            }
+        }
+    }
+    /**
+     * Returns a list of syncable files and the modification times
+     */
+    function syncList()
+    {
+        $this->syncNotify();
+        $info = array();
+        if (isset($_REQUEST["last_sync"])) {
+            $last_sync = $this->clean($_REQUEST["last_sync"], "int");
+        } else {
+            $last_sync = 0;
+        }
+        // substrings to exclude from our list
+        $excludes = array(".DS", "__MACOSX", "queries", "QueueBundle", "tmp",
+            "thumb");
+        $sync_files = $this->model("crawl")->getDeltaFileInfo(
+            CRAWL_DIR."/cache", $last_sync, $excludes);
+        if (count($sync_files) > 0 ) {
+            $info[self::STATUS] = self::CONTINUE_STATE;
+            $info[self::DATA] = $sync_files;
+        } else {
+            $info[self::STATUS] = self::NO_DATA_STATE;
+        }
+        echo base64_encode(gzcompress(serialize($info)));
+    }
+    /**
+     * Handles request to upload the posted data (video file data)
+     * in correct location as per the request attributes such as
+     * folder name and file name.
+     * @return string message concerning success or non-success of upload
+    
+    function uploadConvertedVideoFile()
+    {
+        if (!isset($_REQUEST['data']) || !isset($_REQUEST['folder_name'])||
+            !isset($_REQUEST['file_name'])) {
+            echo "Missing parameters in upload message";
+            return;
+        }
+        $convert_folder = WORK_DIRECTORY . self::CONVERT_FOLDER;
+        $converted_folder = WORK_DIRECTORY . self::CONVERTED_FOLDER;
+        $data = $_REQUEST['data'];
+        $folder_name = webdecode($_REQUEST['folder_name']);
+        $file_name = webdecode($_REQUEST['file_name']);
+        $uploaded_file = $converted_folder . "/" . $folder_name . "/" .
+            $file_name;
+        $original_split_pre_file = $convert_folder . "/" . $folder_name."/".
+            substr($file_name, 0, -4);
+        if (!$data) {
+            echo "No data received by web server.";
+            return;
+        }
+        if (file_exists($converted_folder . "/" .
+            $folder_name . self::CONCATENATED_FILE)){
+            return;
+        }
+        $upload_flag = false;
+        if (file_exists($converted_folder."/".$folder_name)){
+            if (file_exists($uploaded_file)){
+                echo "Video file had already been uploaded!";
+                return;
+            } else {
+                file_put_contents($uploaded_file, webdecode($data));
+                $upload_flag = true;
+            }
+        } else {
+            if (!file_exists($converted_folder)) {
+                mkdir($converted_folder);
+            }
+            mkdir($converted_folder . "/" . $folder_name);
+            file_put_contents($uploaded_file, webdecode($data));
+            $upload_flag = true;
+        }
+        $out = "Deleting pre-convert-segment:\n$original_split_pre_file\n";
+        if ($upload_flag) {
+            $originals =
+                glob($original_split_pre_file.".{mov,avi}", GLOB_BRACE);
+            foreach($originals as $original) {
+                unlink($original);
+            }
+        }
+        echo $out . "Upload success!";
+        return;
+    } */
+    /**
+     * Handles request to unlock the mailing list file
+     * and delete it.
+     */
+    function removeMailingListFile()
+    {
+        if (!isset($_REQUEST['n'])) { return; }
+        $file_name = $this->clean($_REQUEST['n'], "string");
+        if(!preg_match("/\d+\.txt/", $file_name)) { return; }
+        $mail_directory = WORK_DIRECTORY.self::MAIL_FOLDER;
+        if (!file_exists($mail_directory. "/" . $file_name)) { return; }
+        unlink($mail_directory. "/" . $file_name);
+        unlink($mail_directory. "/" . "taken-".$file_name);
+    }
+}
+
diff --git a/src/executables/media_updater.php b/src/executables/media_updater.php
new file mode 100644
index 0000000..4613f0e
--- /dev/null
+++ b/src/executables/media_updater.php
@@ -0,0 +1,688 @@
+<?php
+/**
+ * SeekQuarry/Yioop --
+ * Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ * Copyright (C) 2009 - 2015  Chris Pollett chris@pollett.org
+ *
+ * LICENSE:
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2015
+ * @filesource
+ */
+if (php_sapi_name() != 'cli') {echo "BAD REQUEST"; exit();}
+/**
+ * Calculate base directory of script
+ * @ignore
+ */
+define("BASE_DIR", substr(
+    dirname(realpath($_SERVER['PHP_SELF'])), 0,
+    -strlen("/bin")));
+ini_set("memory_limit", "1300M");
+/** Load in global configuration settings */
+require_once BASE_DIR.'/configs/config.php';
+if (!PROFILE) {
+    echo "Please configure the search engine instance by visiting" .
+        "its web interface on localhost.\n";
+    exit();
+}
+/** CRAWLING means don't try to use memcache
+ * @ignore
+ */
+define("NO_CACHE", true);
+/** We do want logging, but crawl model and other will try to turn off
+ * if we don't set this
+ */
+define("NO_LOGGING", false);
+/**
+ * Shortest time through one iteration of news updater's loop
+ */
+define("MINIMUM_UPDATE_LOOP_TIME", 10);
+/** for crawlDaemon function */
+require_once BASE_DIR."/lib/crawl_daemon.php";
+/** To guess language based on page encoding */
+require_once BASE_DIR."/lib/locale_functions.php";
+/** Loads common constants for media convert files*/
+require_once BASE_DIR."/lib/media_constants.php";
+/** Used to manage the process of sending emails to users */
+require_once BASE_DIR."/lib/mail_server.php";
+/** Loads common constants for web crawling*/
+require_once BASE_DIR."/lib/crawl_constants.php";
+/** Used to reparse wiki pages after a media conversion */
+require_once BASE_DIR."/lib/wiki_parser.php";
+/**Load base model class used by source model */
+require_once BASE_DIR."/models/model.php";
+/** UrlParser is used to get file extensions*/
+require_once BASE_DIR."/lib/url_parser.php";
+/** Source model is used to manage news feed sites*/
+if (file_exists(APP_DIR."/models/source_model.php")) {
+    require_once APP_DIR."/models/source_model.php";
+}  else {
+    require_once BASE_DIR."/models/source_model.php";
+}
+/** Group model is used to manage media conversion*/
+if (file_exists(APP_DIR."/models/group_model.php")) {
+    require_once APP_DIR."/models/group_model.php";
+}  else {
+    require_once BASE_DIR."/models/group_model.php";
+}
+/*
+ * We'll set up multi-byte string handling to use UTF-8
+ */
+mb_internal_encoding("UTF-8");
+mb_regex_encoding("UTF-8");
+
+if (function_exists('lcfirst') === false) {
+    /**
+     * Lower cases the first letter in a string
+     *
+     * This function is only defined if the PHP version is before 5.3
+     * @param string $str  string to be lower cased
+     * @return string the lower cased string
+     */
+    function lcfirst( $str )
+    {
+        return (string)(strtolower(substr($str, 0, 1)).substr($str, 1));
+    }
+}
+  /*$job_files = glob(BASE_DIR ."/lib/jobs/*.php");
+        //echo "jobs are -";  
+        print_r($job_files);
+          foreach($job_files as $job_file){
+            echo "before";
+           require_once $job_file;
+           echo "after";
+           }*/
+
+/**
+ * Separate process/command-line script which can be used to update
+ * news sources for Yioop and also handle other kinds of activities such as
+ * video conversion. This is as an alternative to using the web app
+ * for updating. Makes use of the web-apps code.
+ *
+ * @author Chris Pollett
+ * @package seek_quarry\bin
+ */
+class MediaUpdater implements CrawlConstants,MediaConstants
+{
+    /**
+     * The last time feeds were checked for updates
+     * @var int
+     */
+    var $update_time;
+    /**
+     * Controls whether media updating should be viewed as only occurring
+     * on the name server or should it be viewed as a distributed process
+     * amongst all machines in this Yioop instance
+     * @var string
+     */
+    var $media_mode;
+    /**
+     * Supported file types of videos that we can convert to mp4.
+     * @var array
+     */
+    var $video_convert_types = array("mov", "avi");
+    /**
+     * Mail Server object used to send mails from media updater
+     * @var object
+     */
+    var $mail_server;
+    /**
+     * Sets up the field variables so that media updating can begin
+     */
+    var $jobs;
+    function __construct()
+    {
+        $this->delete_time = 0;
+        $this->retry_time = 0;
+        $this->update_time = 0;
+        $this->media_mode = "name_server";
+        $this->mail_server = new MailServer(MAIL_SENDER, MAIL_SERVER,
+            MAIL_SERVERPORT, MAIL_USERNAME, MAIL_PASSWORD,
+            MAIL_SECURITY);
+        //echo BASE_DIR;
+      $job_files = glob(BASE_DIR ."/lib/jobs/*.php");
+        //echo "jobs are -";  
+       // print_r($job_files);
+        
+        $i=0;
+        require_once BASE_DIR."/lib/jobs/media_job.php";
+        foreach($job_files as $job_file){
+           $job_file_name = str_replace(BASE_DIR ."/lib/jobs/", "", $job_file);
+           $job_name= explode(".",$job_file_name);
+           echo  $job_name[0];
+           $job_name_split=str_replace("_", " ", $job_name[0]);
+           echo  $job_name_split;
+           //$this->jobs[$i]= str_replace(BASE_DIR ."/lib/jobs/", "", $job_file);
+           $job_capitalized=ucwords($job_name_split);
+           echo $job_capitalized;
+           $job_name_split_capital=explode(" ",$job_capitalized);
+           $this->jobs[$i]= implode("",$job_name_split_capital);           
+           require_once $job_file;         
+           $i++;           
+        }  
+        //print_r($jobs);
+    }
+    /**
+     * This is the function that should be called to get the media_updater to
+     * start to start updating. Calls init to handle the command-line
+     * arguments then enters news_updaters main loop
+     */
+    function start()
+    {
+        global $argv;
+        CrawlDaemon::init($argv, "media_updater");
+        crawlLog("\n\nInitialize logger..", "media_updater", true);
+        $this->sourceModel = new SourceModel();
+        $this->groupModel = new GroupModel();
+        $this->loop();
+    }
+
+
+    /**
+     * Main loop for the news updater.
+     */
+    function loop()
+    {
+        crawlLog("In Media Update Loop");
+        print_r($this->jobs);
+        $info[self::STATUS] = self::CONTINUE_STATE;
+        $local_archives = array("");
+        while (CrawlDaemon::processHandler()) {
+            $start_time = microtime();
+            $this->getUpdateProperties();
+            
+            foreach($this->jobs as $job){
+                if($job !='MediaJob'){
+                
+                    echo "calling".$job;
+                    $Ob = new  $job;
+                    
+                    $Ob->processJob();
+                }
+            }           
+            $this->sendGroupNotificationEmailsInBatches();
+            $sleep_time = max(0, ceil(
+                MINIMUM_UPDATE_LOOP_TIME - changeInMicrotime($start_time)));
+            if ($sleep_time > 0) {
+                crawlLog("Ensure minimum loop time by sleeping...".$sleep_time);
+                sleep($sleep_time);
+            }
+        } //end while
+        crawlLog("Media Updater shutting down!!");
+    }
+    /**
+     * Makes a request to the name server to find out if we are running
+     * as a media updater just on the name server or on both the name server
+     * as well as all other machines in the Yioop instance
+     */
+    function getUpdateProperties()
+    {
+        crawlLog("Checking Name Server for Media Updater properties...");
+        $current_machine = $this->sourceModel->getCurrentMachine();
+        $pre_properties = $this->sourceModel->execMachines(
+            "getUpdateProperties", array(NAME_SERVER), $current_machine);
+        $properties = array();
+        if (isset($pre_properties[0][self::PAGE])) {
+            $properties =
+                unserialize(webdecode($pre_properties[0][self::PAGE]));
+            if(isset($properties['MEDIA_MODE'])) {
+                $this->media_mode = $properties['MEDIA_MODE'];
+                crawlLog("...Setting media mode to: " .
+                    $properties['MEDIA_MODE']);
+            }
+        }
+        crawlLog("Done checking Name Server for Media Updater properties");
+    }
+    /**
+     * If news_update time has passed, then updates news feeds associated with
+     * this Yioop instance
+     *
+     * @param array $data used by view to render itself. In this case, if there
+     *     is a problem updating the news then we will flash a message
+     * @param bool $no_news_process if true than assume media_updater.php is
+     *     not running. If false, assume being run from media_updater.php so
+     *     update news_process cron time.
+     */
+  /*  function newsUpdate()
+    {
+        crawlLog("Checking for News Updates...");
+        $time = time();
+        $something_updated = false;
+        $delta = $time - $this->update_time;
+        // every hour get items from feeds
+        if ($delta > ONE_HOUR) {
+            $this->update_time = $time;
+            crawlLog("Performing news feeds update");
+            $this->sourceModel->updateFeedItems(ONE_WEEK, $this->media_mode);
+            $something_updated = true;
+        }
+        /*
+         //   if anything changed rebuild shard
+         
+        if ($something_updated) {
+            crawlLog("Deleting feed items and rebuild shard...");
+            $this->sourceModel->rebuildFeedShard(ONE_WEEK);
+            crawlLog("... delete complete, shard rebuilt");
+        } else {
+            crawlLog("No updates needed.");
+        }
+    } */
+    /**
+     * This function calls the video updater functionality.
+     * It calls the all the required functions based on single or multiple.
+     * machine setting.
+     
+    function videoUpdate()
+    {
+        crawlLog("Checking for video files to process...");
+        $current_machine = $this->sourceModel->getCurrentMachine();
+        if ($current_machine == crawlHash(NAME_SERVER)) {
+           $this->splitVideos();
+           $this->moveVideoFoldersToConvertedDirectory();
+           $this->generateAssembleVideoFile();
+           $this->concatenateVideos();
+        } else {
+           $this->checkGetConvertVideos();
+        }
+        crawlLog("Video updates done!...");
+    }*/
+    /**
+     * Checks if it has been more than two minutes since the last time
+     * a file was modified
+     *
+     * @param string $file_name file to check
+     * @return bool true if it has been more than two minutes
+     */
+    function isNoLongerModified($file_name)
+    {
+        if (file_exists($file_name)) {
+            clearstatcache($file_name);
+            if (time() < filemtime($file_name) + 2 * ONE_MINUTE) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Generates a thumbnail from a video file assuming FFMPEG
+     *
+     * @param string $video_name full name and path of video file to make
+     *      thumbnail from
+     * @param string $thumb_name full name and path for thumbnail file
+     
+    function thumbFileFromVideo($video_name, $thumb_name)
+    {
+        $make_thumb_string =
+            FFMPEG." -i \"$video_name\" -vframes 1 -map 0:v:0".
+            " -vf \"scale=".THUMB_DIM.":".THUMB_DIM."\" ".
+            "\"$thumb_name\" 2>&1";
+        crawlLog("Making thumb with $make_thumb_string");
+        exec($make_thumb_string);
+        clearstatcache($thumb_name);
+    }
+    /**
+     * Splits a video into small chunks of 5 minutes
+     *
+     * @param string.$file_path full path of video file to be split
+     * @param string file_name.name of video file along with extension
+     * @param.string.$destination_directory.destination directory.name
+     *      where split files would be produced
+     
+    function splitVideo($file_path, $file_name, $destination_directory)
+    {
+        crawlLog("Splitting $file_path/$file_name...");
+        $extension = "." . UrlParser::getDocumentType($file_name, "");
+        $new_name = substr($file_name, 0, -strlen($extension));
+        $ffmpeg = FFMPEG." -i \"$file_path/$file_name\" ".
+            " -acodec copy -f segment -segment_time 300 ".
+            "-vcodec copy -reset_timestamps 1 -map 0 ".
+            "\"$destination_directory/%d$new_name$extension\"";
+        crawlLog($ffmpeg);
+        exec($ffmpeg);
+    }
+    /**
+     * Function to look through all the video directories present in media.
+     * convert folder generated by group model.and split the eligible.files.
+     
+    function splitVideos()
+    {
+        $convert_folder = WORK_DIRECTORY.self::CONVERT_FOLDER;
+        if(!defined('FFMPEG') || !file_exists($convert_folder)) { return; }
+        crawlLog("   Looking for video files to split...");
+        $type_string = "{" . implode(",", $this->video_convert_types) . "}";
+        $video_paths = glob($convert_folder."/*");
+        foreach ($video_paths as $video_path) {
+            if (is_dir($video_path)){
+                if (!file_exists($video_path.self::SPLIT_FILE)) {return; }
+                if (file_exists($video_path.self::SPLIT_FILE)) {
+                    crawlLog("Splitting the video $video_path");
+                    $lines = file($video_path.self::FILE_INFO);
+                    $folder_name = rtrim($lines[1]);
+                    $file_name = rtrim($lines[3]);
+                    crawlLog("$folder_name : $file_name");
+                    if ($folder_name && $file_name){
+                        $this->splitVideo($folder_name, $file_name,
+                            $video_path);
+                        unlink($video_path . self::SPLIT_FILE);
+                        file_put_contents($video_path . self::COUNT_FILE,
+                            count(glob($video_path . "/*.$type_string",
+                                GLOB_BRACE)));
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Function to look through all the video directories present in media.
+     * convert folder and move them to converted folders if all the split files.
+     * are converted and are present in video.directory.under.converted.
+    
+    function moveVideoFoldersToConvertedDirectory()
+    {
+        crawlLog("Moving video folders from media_convert to converted...");
+        $convert_folder = WORK_DIRECTORY.self::CONVERT_FOLDER;
+        $converted_folder = WORK_DIRECTORY.self::CONVERTED_FOLDER;
+        if(!file_exists($converted_folder)) {
+            mkdir($converted_folder);
+        }
+        $video_paths = glob($convert_folder."/*");
+        foreach ($video_paths as $video_path) {
+            crawlLog("  Video Path : $video_path");
+            $actual_count = file_get_contents($video_path.self::COUNT_FILE);
+            crawlLog("  Actual_count : $actual_count");
+            $timestamp_files = glob($video_path."/*.time.txt");
+            $checked_out = count($timestamp_files);
+            crawlLog("  Checked out count : $checked_out");
+            $video_folder = str_replace($convert_folder."/", "", $video_path);
+            $converted_video_path = $converted_folder . "/" . $video_folder;
+            $converted_count = count(glob($converted_video_path .
+                "/*.{mp4}", GLOB_BRACE));
+            crawlLog("  Converted count : $converted_count");
+            if ($converted_count == $actual_count) {
+                crawlLog("  Conversion of segments complete!");
+                rename($video_path . self::COUNT_FILE,
+                    $converted_video_path . self::COUNT_FILE);
+                rename($video_path . self::FILE_INFO,
+                    $converted_video_path . self::FILE_INFO);
+                $this->sourceModel->db->unlinkRecursive($video_path);
+            }
+        }
+    }
+    /**
+     * Function to look through all the converted.video directories present in
+     * media and generate the assemble video files needed for concatenating the
+     * converted splitfiles.
+    
+    function generateAssembleVideoFile()
+    {
+        crawlLog("Inside generateAssembleVideoFile function...");
+        $converted_folder = WORK_DIRECTORY.self::CONVERTED_FOLDER;
+        if(!file_exists($converted_folder)) {
+            mkdir($converted_folder);
+        }
+        foreach (glob($converted_folder."/*") as $video_path) {
+            if (file_exists($video_path . self::CONCATENATED_FILE)){ continue; }
+            if (file_exists($video_path . self::ASSEMBLE_FILE)) {
+                continue;
+            }
+            if (!file_exists($video_path.self::COUNT_FILE)) { continue; }
+            $actual_count = file_get_contents($video_path.self::COUNT_FILE);
+            $video_segments = glob($video_path . "/*.mp4");
+            $converted_count = count($video_segments);
+            if ($actual_count == $converted_count) {
+                foreach($video_segments as $video_segment){
+                    file_put_contents($video_path . self::ASSEMBLE_FILE,
+                        "file "."'".(str_replace($video_path."/", "",
+                        $video_segment))."'", FILE_APPEND);
+                    file_put_contents($video_path.self::ASSEMBLE_FILE,
+                        PHP_EOL, FILE_APPEND);
+                }
+            }
+        }
+    }
+    /**
+     * Concatenates split video files to generate one video file
+     *
+     * @param string.$text_file_name file path containing.the relative file.
+     *      paths of the files to be concatenated
+     * @param string file_name name of video file to be given to output file.
+     * @param string $destination_directory.destination directory.name
+     *      where concatenated file would be produced
+    
+    function mergeVideo($text_file_name , $file_name, $destination_directory)
+    {
+        $extension = "." . UrlParser::getDocumentType($file_name, "");
+        $new_name = substr($file_name, 0, -strlen($extension));
+        if (!file_exists($text_file_name)) {return; }
+        $generate_output = $destination_directory."/$new_name.mp4";
+        $ffmpeg = FFMPEG." -f concat -i \"$text_file_name\" -c copy ".
+            "\"$generate_output\"";
+        crawlLog($ffmpeg);
+        exec($ffmpeg);
+        if(file_exists($generate_output)) {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Function to look.through each video directory and call the function to
+     * concatenate split files.
+     
+    function concatenateVideos()
+    {
+        crawlLog("Concatenating videos...");
+        $converted_folder = WORK_DIRECTORY.self::CONVERTED_FOLDER;
+        if(!file_exists($converted_folder)) {
+            mkdir($converted_folder);
+        }
+        foreach (glob($converted_folder."/*") as $video_path) {
+            crawlLog("  Video Path " . $video_path);
+            if (is_dir($video_path)){
+                if(!file_exists($video_path . self::ASSEMBLE_FILE)) {
+                    continue;
+                }
+                $assemble_file = $video_path . self::ASSEMBLE_FILE;
+                $lines = file($video_path . self::FILE_INFO);
+                $folder = trim($lines[1]);
+                $thumb_folder = trim($lines[2]);
+                $file_name = trim($lines[3]);
+                if($this->mergeVideo($assemble_file, $file_name, $folder)){
+                    $this->sourceModel->db->unlinkRecursive($video_path);
+                    $video_name = $folder. "/" . $file_name;
+                    $extension_len = strlen(
+                        UrlParser::getDocumentType($video_name));
+                    $file_prefix = substr($file_name, 0, -$extension_len - 1);
+                    $thumb_file_name = $file_prefix . ".mp4.jpg";
+                    $thumb_name = $thumb_folder . "/" . $thumb_file_name;
+                    $this->thumbFileFromVideo($video_name, $thumb_name);
+                }
+            }
+        }
+    }
+    /**
+     * Function to convert avi or mov file to mp4 format.
+     *
+     * @param string $file_name full path of the file.
+     
+    function convertVideo($file_name)
+    {
+        $extension = "." . UrlParser::getDocumentType($file_name, "");
+        $new_name = substr($file_name, 0, -strlen($extension));
+        switch($extension)
+        {
+            case '.mov':
+                $ffmpeg = FFMPEG." -i \"$file_name\" ".
+                    " -vcodec h264 -acodec aac -preset veryfast -crf 28 ".
+                    "-strict -2 \"$new_name.mp4\"";
+            break;
+            case '.avi':
+                $ffmpeg = FFMPEG." -i \"$file_name\" ".
+                    " -vcodec libx264  -preset slow -acodec aac -crf 28 ".
+                    "-strict experimental -b:a 192k -ac 2 \"$new_name.mp4\"";
+            break;
+        }
+        crawlLog($ffmpeg);
+        exec($ffmpeg);
+    }
+    /**
+     * Checks name server for a video segment to convert. If there are
+     * converts the mov or avi segment file to an mp4 file
+     * This function would only be called by slave media updaters.
+     
+    function checkGetConvertVideos()
+    {
+        crawlLog("Checking Name Server for video segments to convert..");
+        $convert_folder = WORK_DIRECTORY . self::CONVERT_FOLDER;
+        if (!file_exists($convert_folder)) {
+            @mkdir($convert_folder);
+            if (!file_exists($convert_folder)) {
+                crawlLog("Unable to create $convert_folder. Bailing!");
+                return;
+            }
+        }
+        $folders = glob($convert_folder."/*", GLOB_ONLYDIR);
+        if (count($folders) > 0) {
+            foreach($folders as $folder){
+                $this->sourceModel->db->unlinkRecursive($folder);
+            }
+        }
+        $time = time();
+        $session = md5($time . AUTH_KEY);
+        $server = NAME_SERVER;
+        $folder_file = $this->sourceModel->requestFileForConversion();
+        $db = $this->sourceModel->db;
+        if ($folder_file) {
+            $info = explode(":", $folder_file);
+            $folder_name = trim($info[0]);
+            $file_name = trim($info[1]);
+            /* Download the file from the server 
+            $request = "$server?c=resource&a=get&time=$time&session=$session".
+                "&f=schedules&n=". urlencode($file_name)."&sf=$folder_name";
+            $data = FetchUrl::getPage($request, NULL, true);
+            $convert_path = $convert_folder."/".$folder_name;
+            if(file_exists( $convert_path)) {
+                $db->unlinkRecursive( $convert_path);
+            }
+            mkdir($convert_path);
+            $downloaded_file =  $convert_path . "/" . $file_name;
+            file_put_contents($downloaded_file, $data);
+            $this->convertVideo($downloaded_file);
+            $files = glob($convert_path . "/*.{mp4}", GLOB_BRACE);
+            if (!$files[0]) {
+                crawlLog("Will try to convert the file again later");
+            } else {
+                $converted_file_name = substr($files[0],
+                    strlen($convert_path) + 1);
+                /* Upload the file to the server 
+                $file_data = file_get_contents($files[0]);
+                $post_data['c'] = "resource";
+                $post_data['a'] = "uploadConvertedVideoFile";
+                $post_data['time'] = $time;
+                $post_data['session'] = $session;
+                $post_data['data'] =  webencode($file_data);
+                $post_data['file_name'] = webencode($converted_file_name);
+                $post_data['folder_name'] = webencode($folder_name);
+                crawlLog("Attempting to upload converted video file...");
+                $response = FetchUrl::getPage($server, $post_data , true);
+                crawlLog("...server response:\n $response");
+            }
+        } else {
+            crawlLog("No files on server to convert!");
+        }
+    }*/
+    /**
+     *
+     * Function to send emails to mailer batches created by
+     * mail_server. This function would periodically be invoked and
+     * send emails reading data from the text files.
+     */
+    function sendGroupNotificationEmailsInBatches()
+    {
+        crawlLog("Checking for mailer files to be sent out...");
+        $current_machine = $this->sourceModel->getCurrentMachine();
+        if ($current_machine == crawlHash(NAME_SERVER) &&
+            MEDIA_MODE != 'distributed') {
+            $mail_directory = WORK_DIRECTORY . self::MAIL_FOLDER;
+            if(!file_exists($mail_directory)) { return; }
+            $files = glob($mail_directory."/*.txt");
+            if(!isset($files[0])) {
+                return;
+            }
+            $sendable_file = false;
+            foreach($files as $email_file) {
+                if(time() - filemtime($email_file) > MAX_MAIL_TIMESTAMP_LIMIT) {
+                    $sendable_file = $email_file;
+                    break;
+                }
+            }
+            if(!$sendable_file) {
+                return;
+            }
+            $emails_string = file_get_contents($sendable_file);
+            unlink($email_file);
+            $emails = explode(self::MESSAGE_SEPARATOR, $emails_string);
+            foreach ($emails as $serialized_email) {
+                $email = unserialize($serialized_email);
+                if(count($email) == 4) {
+                    crawlLog("Sending email to {$email[2]} about {$email[0]}");
+                    $this->mail_server->sendImmediate(
+                        $email[0], $email[1], $email[2], $email[3]);
+                }
+            }
+        } else {
+            $file_name = $this->sourceModel->requestFileForMailingList();
+            if(empty($file_name)){
+                crawlLog("...Could not get any response from name server!");
+                return;
+            }
+            $time = time();
+            $session = md5($time . AUTH_KEY);
+            $server = NAME_SERVER;
+            /* Download the file from the server */
+            $request = "$server?c=resource&a=get&time=$time&session=$session".
+                "&f=schedules&n=" . urlencode($file_name) . "&sf=mail";
+            $emails_string = FetchUrl::getPage($request, NULL, true);
+            if(!$emails_string) {
+                crawlLog("  No mail data returning");
+                return;
+            }
+            $emails = explode(self::MESSAGE_SEPARATOR, $emails_string);
+            foreach ($emails as $serialized_email) {
+                $email = unserialize($serialized_email);
+                if(count($email) == 4) {
+                    crawlLog("Sending email to {$email[2]} about {$email[0]}");
+                    $this->mail_server->sendImmediate(
+                        $email[0], $email[1], $email[2], $email[3]);
+                }
+            }
+            $request = "$server?c=resource&a=removeMailingListFile".
+                "&time=$time&session=$session&n=".urlencode($file_name);
+            FetchUrl::getPage($request, NULL, true);
+        }
+    }
+}
+/*
+ * Instantiate and run the MediaUpdater program
+ */
+$media_updater =  new MediaUpdater();
+$media_updater->start();
+
diff --git a/src/library/jobs/media_job.php b/src/library/jobs/media_job.php
new file mode 100644
index 0000000..1f03cb3
--- /dev/null
+++ b/src/library/jobs/media_job.php
@@ -0,0 +1,35 @@
+<?php
+require_once BASE_DIR."/lib/crawl_daemon.php";
+/** To guess language based on page encoding */
+require_once BASE_DIR."/lib/locale_functions.php";
+/** Loads common constants for media convert files*/
+require_once BASE_DIR."/lib/media_constants.php";
+/** Used to manage the process of sending emails to users */
+require_once BASE_DIR."/lib/mail_server.php";
+/** Loads common constants for web crawling*/
+require_once BASE_DIR."/lib/crawl_constants.php";
+/** Used to reparse wiki pages after a media conversion */
+require_once BASE_DIR."/lib/wiki_parser.php";
+/**Load base model class used by source model */
+require_once BASE_DIR."/models/model.php";
+/** UrlParser is used to get file extensions*/
+require_once BASE_DIR."/lib/url_parser.php";
+/** Source model is used to manage news feed sites*/
+if (file_exists(APP_DIR."/models/source_model.php")) {
+    require_once APP_DIR."/models/source_model.php";
+}  else {
+    require_once BASE_DIR."/models/source_model.php";
+}
+/** Group model is used to manage media conversion*/
+if (file_exists(APP_DIR."/models/group_model.php")) {
+    require_once APP_DIR."/models/group_model.php";
+}  else {
+    require_once BASE_DIR."/models/group_model.php";
+}
+
+error_reporting (15);
+echo "hi";
+abstract class MediaJob implements CrawlConstants,MediaConstants
+{
+    public abstract function processJob();
+}
\ No newline at end of file
diff --git a/src/library/jobs/news_updater_job.php b/src/library/jobs/news_updater_job.php
new file mode 100644
index 0000000..78348d6
--- /dev/null
+++ b/src/library/jobs/news_updater_job.php
@@ -0,0 +1,110 @@
+<?php
+require_once BASE_DIR."/lib/crawl_daemon.php";
+/** To guess language based on page encoding */
+require_once BASE_DIR."/lib/locale_functions.php";
+/** Loads common constants for media convert files*/
+require_once BASE_DIR."/lib/media_constants.php";
+/** Used to manage the process of sending emails to users */
+require_once BASE_DIR."/lib/mail_server.php";
+/** Loads common constants for web crawling*/
+require_once BASE_DIR."/lib/crawl_constants.php";
+/** Used to reparse wiki pages after a media conversion */
+require_once BASE_DIR."/lib/wiki_parser.php";
+/**Load base model class used by source model */
+require_once BASE_DIR."/models/model.php";
+/** UrlParser is used to get file extensions*/
+require_once BASE_DIR."/lib/url_parser.php";
+/** Source model is used to manage news feed sites*/
+if (file_exists(APP_DIR."/models/source_model.php")) {
+    require_once APP_DIR."/models/source_model.php";
+}  else {
+    require_once BASE_DIR."/models/source_model.php";
+}
+/** Group model is used to manage media conversion*/
+if (file_exists(APP_DIR."/models/group_model.php")) {
+    require_once APP_DIR."/models/group_model.php";
+}  else {
+    require_once BASE_DIR."/models/group_model.php";
+}
+
+class NewsUpdaterJob extends MediaJob
+{
+    function processJob(){
+        $this->newsUpdate();
+    }
+    
+    
+     function newsUpdate()
+    {
+        crawlLog("Checking for News Updates in job...");
+        $time = time();
+        $this->sourceModel = new SourceModel();
+        $something_updated = false;
+        $delta = $time - $this->update_time;
+        
+        // every hour get items from feeds
+        if ($delta > ONE_HOUR) {
+            $this->update_time = $time;
+            crawlLog("Performing news feeds update.....");
+            $this->sourceModel->updateFeedItems(ONE_WEEK, $this->media_mode);
+            echo "crawlLog done!";
+            $something_updated = true;
+        }
+        /*
+            if anything changed rebuild shard
+         */
+        if ($something_updated) {
+            crawlLog("Deleting feed items and rebuild shard...");
+            $this->sourceModel->rebuildFeedShard(ONE_WEEK);
+            crawlLog("... delete complete, shard rebuilt");
+        } else {
+            crawlLog("No updates needed.");
+        }
+    }
+    
+    
+     function getNewsSources()
+    {
+        if (!isset($_REQUEST["arg"])) {
+            return;
+        }
+        $source_model = $this->model("source");
+        $current_machine = $this->clean(webdecode($_REQUEST["arg"]), "string");
+        $machine_hashes = $source_model->getMachineHashUrls();
+        $machine_index_match = array_search($current_machine, $machine_hashes);
+        if ($machine_index_match === false) {
+            echo webencode(serialize(array()));
+            return;
+        }
+        $num_machines = count($machine_hashes);
+        $pre_feeds = $source_model->getMediaSources("rss");
+        $pre_feeds = array_merge($pre_feeds,
+            $source_model->getMediaSources("html"));
+        if (!$pre_feeds) { return false; }
+        $feeds = array();
+        foreach ($pre_feeds as $pre_feed) {
+            if (!isset($pre_feed['NAME'])) {continue; }
+            $hash_int = unpack("N", crawlHash($pre_feed['NAME']));
+            if (!isset($hash_int[1])) {continue; }
+            $hash_index = ($hash_int[1]) % $num_machines;
+            if ($machine_index_match != $hash_index) {continue; }
+            $aux_parts = explode("###",
+                html_entity_decode($pre_feed['AUX_INFO']));
+            $offset = 0;
+            if ($pre_feed['TYPE'] == 'html') {
+                list($pre_feed['CHANNEL_PATH'], $pre_feed['ITEM_PATH'],
+                    $pre_feed['TITLE_PATH'], $pre_feed['DESCRIPTION_PATH'],
+                    $pre_feed['LINK_PATH'], $pre_feed['LINK_PATH']) =
+                    $aux_parts;
+                $offset = 5;
+            }
+            if (isset($aux_parts[$offset])) {
+                $pre_feed['IMAGE_XPATH'] = $aux_parts[$offset];
+            } else {
+                $pre_feed['IMAGE_XPATH'] = "";
+            }
+            $feeds[] = $pre_feed;
+        }
+        echo webencode(serialize($feeds));
+    }
+}
diff --git a/src/library/jobs/video_updater_job.php b/src/library/jobs/video_updater_job.php
new file mode 100644
index 0000000..251af02
--- /dev/null
+++ b/src/library/jobs/video_updater_job.php
@@ -0,0 +1,435 @@
+<?php
+require_once BASE_DIR."/lib/crawl_daemon.php";
+/** To guess language based on page encoding */
+require_once BASE_DIR."/lib/locale_functions.php";
+/** Loads common constants for media convert files*/
+require_once BASE_DIR."/lib/media_constants.php";
+/** Used to manage the process of sending emails to users */
+require_once BASE_DIR."/lib/mail_server.php";
+/** Loads common constants for web crawling*/
+require_once BASE_DIR."/lib/crawl_constants.php";
+/** Used to reparse wiki pages after a media conversion */
+require_once BASE_DIR."/lib/wiki_parser.php";
+/**Load base model class used by source model */
+require_once BASE_DIR."/models/model.php";
+/** UrlParser is used to get file extensions*/
+require_once BASE_DIR."/lib/url_parser.php";
+/** Source model is used to manage news feed sites*/
+if (file_exists(APP_DIR."/models/source_model.php")) {
+    require_once APP_DIR."/models/source_model.php";
+}  else {
+    require_once BASE_DIR."/models/source_model.php";
+}
+/** Group model is used to manage media conversion*/
+if (file_exists(APP_DIR."/models/group_model.php")) {
+    require_once APP_DIR."/models/group_model.php";
+}  else {
+    require_once BASE_DIR."/models/group_model.php";
+}
+
+class VideoUpdaterJob extends MediaJob
+{
+    function processJob(){
+        $this->videoUpdate();        
+    }
+    
+    function videoUpdate(){
+        crawlLog("Checking for video files to process...");
+        $time = time();
+        $this->sourceModel = new SourceModel();
+        $current_machine = $this->sourceModel->getCurrentMachine();
+        if ($current_machine == crawlHash(NAME_SERVER)) {
+           $this->splitVideos();
+           $this->moveVideoFoldersToConvertedDirectory();
+           $this->generateAssembleVideoFile();
+           $this->concatenateVideos();
+        } else {
+           $this->checkGetConvertVideos();
+        }
+        crawlLog("Video updates done!...");
+        
+    }
+    
+    /**
+     * Generates a thumbnail from a video file assuming FFMPEG
+     *
+     * @param string $video_name full name and path of video file to make
+     *      thumbnail from
+     * @param string $thumb_name full name and path for thumbnail file
+     */
+    function thumbFileFromVideo($video_name, $thumb_name)
+    {
+        $make_thumb_string =
+            FFMPEG." -i \"$video_name\" -vframes 1 -map 0:v:0".
+            " -vf \"scale=".THUMB_DIM.":".THUMB_DIM."\" ".
+            "\"$thumb_name\" 2>&1";
+        crawlLog("Making thumb with $make_thumb_string");
+        exec($make_thumb_string);
+        clearstatcache($thumb_name);
+    }
+    /**
+     * Splits a video into small chunks of 5 minutes
+     *
+     * @param string.$file_path full path of video file to be split
+     * @param string file_name.name of video file along with extension
+     * @param.string.$destination_directory.destination directory.name
+     *      where split files would be produced
+     */
+    function splitVideo($file_path, $file_name, $destination_directory)
+    {
+        crawlLog("Splitting $file_path/$file_name...");
+        $extension = "." . UrlParser::getDocumentType($file_name, "");
+        $new_name = substr($file_name, 0, -strlen($extension));
+        $ffmpeg = FFMPEG." -i \"$file_path/$file_name\" ".
+            " -acodec copy -f segment -segment_time 300 ".
+            "-vcodec copy -reset_timestamps 1 -map 0 ".
+            "\"$destination_directory/%d$new_name$extension\"";
+        crawlLog($ffmpeg);
+        exec($ffmpeg);
+    }
+    /**
+     * Function to look through all the video directories present in media.
+     * convert folder generated by group model.and split the eligible.files.
+     */
+    function splitVideos()
+    {
+        $convert_folder = WORK_DIRECTORY.self::CONVERT_FOLDER;
+        if(!defined('FFMPEG') || !file_exists($convert_folder)) { return; }
+        crawlLog("   Looking for video files to split...");
+        $type_string = "{" . implode(",", $this->video_convert_types) . "}";
+        $video_paths = glob($convert_folder."/*");
+        foreach ($video_paths as $video_path) {
+            if (is_dir($video_path)){
+                if (!file_exists($video_path.self::SPLIT_FILE)) {return; }
+                if (file_exists($video_path.self::SPLIT_FILE)) {
+                    crawlLog("Splitting the video $video_path");
+                    $lines = file($video_path.self::FILE_INFO);
+                    $folder_name = rtrim($lines[1]);
+                    $file_name = rtrim($lines[3]);
+                    crawlLog("$folder_name : $file_name");
+                    if ($folder_name && $file_name){
+                        $this->splitVideo($folder_name, $file_name,
+                            $video_path);
+                        unlink($video_path . self::SPLIT_FILE);
+                        file_put_contents($video_path . self::COUNT_FILE,
+                            count(glob($video_path . "/*.$type_string",
+                                GLOB_BRACE)));
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Function to look through all the video directories present in media.
+     * convert folder and move them to converted folders if all the split files.
+     * are converted and are present in video.directory.under.converted.
+     */
+    function moveVideoFoldersToConvertedDirectory()
+    {
+        crawlLog("Moving video folders from media_convert to converted...");
+        $convert_folder = WORK_DIRECTORY.self::CONVERT_FOLDER;
+        $converted_folder = WORK_DIRECTORY.self::CONVERTED_FOLDER;
+        if(!file_exists($converted_folder)) {
+            mkdir($converted_folder);
+        }
+        $video_paths = glob($convert_folder."/*");
+        foreach ($video_paths as $video_path) {
+            crawlLog("  Video Path : $video_path");
+            $actual_count = file_get_contents($video_path.self::COUNT_FILE);
+            crawlLog("  Actual_count : $actual_count");
+            $timestamp_files = glob($video_path."/*.time.txt");
+            $checked_out = count($timestamp_files);
+            crawlLog("  Checked out count : $checked_out");
+            $video_folder = str_replace($convert_folder."/", "", $video_path);
+            $converted_video_path = $converted_folder . "/" . $video_folder;
+            $converted_count = count(glob($converted_video_path .
+                "/*.{mp4}", GLOB_BRACE));
+            crawlLog("  Converted count : $converted_count");
+            if ($converted_count == $actual_count) {
+                crawlLog("  Conversion of segments complete!");
+                rename($video_path . self::COUNT_FILE,
+                    $converted_video_path . self::COUNT_FILE);
+                rename($video_path . self::FILE_INFO,
+                    $converted_video_path . self::FILE_INFO);
+                $this->sourceModel->db->unlinkRecursive($video_path);
+            }
+        }
+    }
+    /**
+     * Function to look through all the converted.video directories present in
+     * media and generate the assemble video files needed for concatenating the
+     * converted splitfiles.
+     */
+    function generateAssembleVideoFile()
+    {
+        crawlLog("Inside generateAssembleVideoFile function...");
+        $converted_folder = WORK_DIRECTORY.self::CONVERTED_FOLDER;
+        if(!file_exists($converted_folder)) {
+            mkdir($converted_folder);
+        }
+        foreach (glob($converted_folder."/*") as $video_path) {
+            if (file_exists($video_path . self::CONCATENATED_FILE)){ continue; }
+            if (file_exists($video_path . self::ASSEMBLE_FILE)) {
+                continue;
+            }
+            if (!file_exists($video_path.self::COUNT_FILE)) { continue; }
+            $actual_count = file_get_contents($video_path.self::COUNT_FILE);
+            $video_segments = glob($video_path . "/*.mp4");
+            $converted_count = count($video_segments);
+            if ($actual_count == $converted_count) {
+                foreach($video_segments as $video_segment){
+                    file_put_contents($video_path . self::ASSEMBLE_FILE,
+                        "file "."'".(str_replace($video_path."/", "",
+                        $video_segment))."'", FILE_APPEND);
+                    file_put_contents($video_path.self::ASSEMBLE_FILE,
+                        PHP_EOL, FILE_APPEND);
+                }
+            }
+        }
+    }
+    /**
+     * Concatenates split video files to generate one video file
+     *
+     * @param string.$text_file_name file path containing.the relative file.
+     *      paths of the files to be concatenated
+     * @param string file_name name of video file to be given to output file.
+     * @param string $destination_directory.destination directory.name
+     *      where concatenated file would be produced
+     */
+    function mergeVideo($text_file_name , $file_name, $destination_directory)
+    {
+        $extension = "." . UrlParser::getDocumentType($file_name, "");
+        $new_name = substr($file_name, 0, -strlen($extension));
+        if (!file_exists($text_file_name)) {return; }
+        $generate_output = $destination_directory."/$new_name.mp4";
+        $ffmpeg = FFMPEG." -f concat -i \"$text_file_name\" -c copy ".
+            "\"$generate_output\"";
+        crawlLog($ffmpeg);
+        exec($ffmpeg);
+        if(file_exists($generate_output)) {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Function to look.through each video directory and call the function to
+     * concatenate split files.
+     */
+    function concatenateVideos()
+    {
+        crawlLog("Concatenating videos...");
+        $converted_folder = WORK_DIRECTORY.self::CONVERTED_FOLDER;
+        if(!file_exists($converted_folder)) {
+            mkdir($converted_folder);
+        }
+        foreach (glob($converted_folder."/*") as $video_path) {
+            crawlLog("  Video Path " . $video_path);
+            if (is_dir($video_path)){
+                if(!file_exists($video_path . self::ASSEMBLE_FILE)) {
+                    continue;
+                }
+                $assemble_file = $video_path . self::ASSEMBLE_FILE;
+                $lines = file($video_path . self::FILE_INFO);
+                $folder = trim($lines[1]);
+                $thumb_folder = trim($lines[2]);
+                $file_name = trim($lines[3]);
+                if($this->mergeVideo($assemble_file, $file_name, $folder)){
+                    $this->sourceModel->db->unlinkRecursive($video_path);
+                    $video_name = $folder. "/" . $file_name;
+                    $extension_len = strlen(
+                        UrlParser::getDocumentType($video_name));
+                    $file_prefix = substr($file_name, 0, -$extension_len - 1);
+                    $thumb_file_name = $file_prefix . ".mp4.jpg";
+                    $thumb_name = $thumb_folder . "/" . $thumb_file_name;
+                    $this->thumbFileFromVideo($video_name, $thumb_name);
+                }
+            }
+        }
+    }
+    /**
+     * Function to convert avi or mov file to mp4 format.
+     *
+     * @param string $file_name full path of the file.
+     */
+    function convertVideo($file_name)
+    {
+        $extension = "." . UrlParser::getDocumentType($file_name, "");
+        $new_name = substr($file_name, 0, -strlen($extension));
+        switch($extension)
+        {
+            case '.mov':
+                $ffmpeg = FFMPEG." -i \"$file_name\" ".
+                    " -vcodec h264 -acodec aac -preset veryfast -crf 28 ".
+                    "-strict -2 \"$new_name.mp4\"";
+            break;
+            case '.avi':
+                $ffmpeg = FFMPEG." -i \"$file_name\" ".
+                    " -vcodec libx264  -preset slow -acodec aac -crf 28 ".
+                    "-strict experimental -b:a 192k -ac 2 \"$new_name.mp4\"";
+            break;
+        }
+        crawlLog($ffmpeg);
+        exec($ffmpeg);
+    }
+    /**
+     * Checks name server for a video segment to convert. If there are
+     * converts the mov or avi segment file to an mp4 file
+     * This function would only be called by slave media updaters.
+     */
+    function checkGetConvertVideos()
+    {
+        crawlLog("Checking Name Server for video segments to convert..");
+        $convert_folder = WORK_DIRECTORY . self::CONVERT_FOLDER;
+        if (!file_exists($convert_folder)) {
+            @mkdir($convert_folder);
+            if (!file_exists($convert_folder)) {
+                crawlLog("Unable to create $convert_folder. Bailing!");
+                return;
+            }
+        }
+        $folders = glob($convert_folder."/*", GLOB_ONLYDIR);
+        if (count($folders) > 0) {
+            foreach($folders as $folder){
+                $this->sourceModel->db->unlinkRecursive($folder);
+            }
+        }
+        $time = time();
+        $session = md5($time . AUTH_KEY);
+        $server = NAME_SERVER;
+        $folder_file = $this->sourceModel->requestFileForConversion();
+        $db = $this->sourceModel->db;
+        if ($folder_file) {
+            $info = explode(":", $folder_file);
+            $folder_name = trim($info[0]);
+            $file_name = trim($info[1]);
+            /* Download the file from the server */
+            $request = "$server?c=resource&a=get&time=$time&session=$session".
+                "&f=schedules&n=". urlencode($file_name)."&sf=$folder_name";
+            $data = FetchUrl::getPage($request, NULL, true);
+            $convert_path = $convert_folder."/".$folder_name;
+            if(file_exists( $convert_path)) {
+                $db->unlinkRecursive( $convert_path);
+            }
+            mkdir($convert_path);
+            $downloaded_file =  $convert_path . "/" . $file_name;
+            file_put_contents($downloaded_file, $data);
+            $this->convertVideo($downloaded_file);
+            $files = glob($convert_path . "/*.{mp4}", GLOB_BRACE);
+            if (!$files[0]) {
+                crawlLog("Will try to convert the file again later");
+            } else {
+                $converted_file_name = substr($files[0],
+                    strlen($convert_path) + 1);
+                /* Upload the file to the server */
+                $file_data = file_get_contents($files[0]);
+                $post_data['c'] = "resource";
+                $post_data['a'] = "uploadConvertedVideoFile";
+                $post_data['time'] = $time;
+                $post_data['session'] = $session;
+                $post_data['data'] =  webencode($file_data);
+                $post_data['file_name'] = webencode($converted_file_name);
+                $post_data['folder_name'] = webencode($folder_name);
+                crawlLog("Attempting to upload converted video file...");
+                $response = FetchUrl::getPage($server, $post_data , true);
+                crawlLog("...server response:\n $response");
+            }
+        } else {
+            crawlLog("No files on server to convert!");
+        }
+    }
+    
+    /**
+     * Handles request to upload the posted data (video file data)
+     * in correct location as per the request attributes such as
+     * folder name and file name.
+     * @return string message concerning success or non-success of upload
+     */
+    function uploadConvertedVideoFile()
+    {
+        if (!isset($_REQUEST['data']) || !isset($_REQUEST['folder_name'])||
+            !isset($_REQUEST['file_name'])) {
+            echo "Missing parameters in upload message";
+            return;
+        }
+        $convert_folder = WORK_DIRECTORY . self::CONVERT_FOLDER;
+        $converted_folder = WORK_DIRECTORY . self::CONVERTED_FOLDER;
+        $data = $_REQUEST['data'];
+        $folder_name = webdecode($_REQUEST['folder_name']);
+        $file_name = webdecode($_REQUEST['file_name']);
+        $uploaded_file = $converted_folder . "/" . $folder_name . "/" .
+            $file_name;
+        $original_split_pre_file = $convert_folder . "/" . $folder_name."/".
+            substr($file_name, 0, -4);
+        if (!$data) {
+            echo "No data received by web server.";
+            return;
+        }
+        if (file_exists($converted_folder . "/" .
+            $folder_name . self::CONCATENATED_FILE)){
+            return;
+        }
+        $upload_flag = false;
+        if (file_exists($converted_folder."/".$folder_name)){
+            if (file_exists($uploaded_file)){
+                echo "Video file had already been uploaded!";
+                return;
+            } else {
+                file_put_contents($uploaded_file, webdecode($data));
+                $upload_flag = true;
+            }
+        } else {
+            if (!file_exists($converted_folder)) {
+                mkdir($converted_folder);
+            }
+            mkdir($converted_folder . "/" . $folder_name);
+            file_put_contents($uploaded_file, webdecode($data));
+            $upload_flag = true;
+        }
+        $out = "Deleting pre-convert-segment:\n$original_split_pre_file\n";
+        if ($upload_flag) {
+            $originals =
+                glob($original_split_pre_file.".{mov,avi}", GLOB_BRACE);
+            foreach($originals as $original) {
+                unlink($original);
+            }
+        }
+        echo $out . "Upload success!";
+        return;
+    }
+    /**
+     * Handles the request to get the video file from the video directory for
+     * conversion. This selection is based upon if the file was taken
+     * previously or not. If it was then its timestamp is checked.
+     * Otherwise new file is sent for conversion along with its folder name.
+     */
+    function getSplitFileForConversion()
+    {
+        if (!isset($_REQUEST["arg"])) { return; }
+        $current_machine = $this->clean(webdecode($_REQUEST["arg"]), "string");
+        $convert_folder = WORK_DIRECTORY.self::CONVERT_FOLDER;
+        $current_time = time();
+        foreach (glob($convert_folder . "/*") as $folder) {
+            foreach (glob($folder."/*.{mov,avi}", GLOB_BRACE) as $file) {
+                $folder_name = str_replace("$convert_folder/", "", $folder);
+                $file_name = str_replace($folder."/", "", $file);
+                $time_file_name = $folder . "/" . $file_name . ".time.txt";
+                if(file_exists($time_file_name)) {
+                    $file_time = file_get_contents($time_file_name);
+                    if ($current_time - $file_time > MAX_FILE_TIMESTAMP_LIMIT) {
+                        file_put_contents($time_file_name, $current_time);
+                        $folder_file = $folder_name . ":" . $file_name;
+                    }
+                } else {
+                    file_put_contents($time_file_name, $current_time);
+                    $folder_file = $folder_name . ":" . $file_name;
+                }
+                if (isset($folder_file)) {
+                    break 2;
+                }
+            }
+        }
+        if(isset($folder_file)) {
+            echo webencode($folder_file);
+        }
+    }
+}
-- 
1.9.4.msysgit.1

