<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org (initial MediaJob class
 *      and subclasses based on work of Pooja Mishra for her master's)
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\media_jobs;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\FetchUrl;
use seekquarry\yioop\library\IndexShard;
use seekquarry\yioop\library\PhraseParser;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\models\GroupModel;
use seekquarry\yioop\controllers\CrawlController;

/**
 * A media job to periodically download Podcasts and store them as resources
 * of a Wiki Page
 */
class PodcastDownloadJob extends MediaJob
{
    /**
     * how long in seconds before a podcast item expires
     */
    const ITEM_EXPIRES_TIME = C\ONE_WEEK;
    /**
     * Mamimum number of feeds to download in one try
     */
    const MAX_PODCASTS_ONE_GO = 100;
    /**
     * Time in current epoch when feeds last updated
     * @var int
     */
    public $update_time;
    /**
     * Whether the current run was triggered by an immediate-update
     * request rather than the periodic interval, in which case only
     * the requested folders' sources are downloaded
     * @var bool
     */
    public $request_driven = false;
    /**
     * Folder keys with a pending immediate-update request for the
     * current run, set by checkPrerequisites
     * @var array
     */
    public $requested_folder_keys = [];
    /**
     * Instance of group model used to get a list of podcasts
     * @var object
     */
    public $group_model;
    /**
     * Datasource object used to run db queries related to fes items
     * (for storing and updating them)
     * @var object
     */
    public $db;
    /**
     * Initializes the last update time to far in the past so, feeds will get
     * immediately updated. Sets up connect to DB to store feeds items, and
     * makes it so the same media job runs both on name server and client
     * Media Updaters
     */
    public function init()
    {
        $this->update_time = 0;
        $this->name_server_does_client_tasks = true;
        $this->name_server_does_client_tasks_only = true;
        $db_class = C\NS_DATASOURCES . ucfirst(C\p('DBMS')). "Manager";
        $this->db = new $db_class();
        $this->db->connect();
        if(!empty($this->controller)) {
            $group_model = $this->controller->model("group");
        } else {
            $group_model = new GroupModel();
        }
        $this->group_model = $group_model;
        C\nsconddefine("PODCAST_UPDATE_INTERVAL", C\ONE_DAY);
    }
    /**
     * Builds the per-folder key that identifies one podcast
     * destination across the download marker and the update-request
     * marker. A destination is a wiki resource folder, fixed by its
     * group, page, and sub-path, so the key combines those three.
     *
     * @param int $group_id group the destination page belongs to
     * @param int $page_id wiki page the podcast downloads to
     * @param string $sub_path resource sub-folder within the page,
     *      empty for the page's top resource folder
     * @return string key naming this destination folder
     */
    public static function podcastFolderKey($group_id, $page_id,
        $sub_path)
    {
        $clean_sub_path = ($sub_path === "" || $sub_path === null) ?
            "" : L\crawlHash($sub_path);
        return "Podcast-" . $group_id . "-" . $page_id . "-" .
            $clean_sub_path;
    }
    /**
     * Path of the marker file the job writes and refreshes while it
     * is downloading podcast items to one destination folder. The
     * file's modification time is the heartbeat the Media List page
     * checks against PODCAST_DOWNLOAD_STALE_TIME.
     *
     * @param string $folder_key key from podcastFolderKey
     * @return string absolute path of the downloading marker file
     */
    public static function podcastDownloadingMarkerFile($folder_key)
    {
        return C\SCHEDULES_DIR . "/" . $folder_key . "Downloading.txt";
    }
    /**
     * Path of the marker file the web UI writes to request an
     * immediate update of one destination folder's podcast sources.
     * The job removes it once it has begun honoring the request.
     *
     * @param string $folder_key key from podcastFolderKey
     * @return string absolute path of the update-request marker file
     */
    public static function podcastRequestMarkerFile($folder_key)
    {
        return C\SCHEDULES_DIR . "/" . $folder_key . "Request.txt";
    }
    /**
     * Whether a destination folder is currently being downloaded to,
     * judged by a downloading marker whose heartbeat is newer than
     * PODCAST_DOWNLOAD_STALE_TIME. A marker left behind by an updater
     * that died mid-download goes stale and reads as not downloading.
     *
     * @param string $folder_key key from podcastFolderKey
     * @return bool true if a fresh downloading marker exists
     */
    public static function isPodcastDownloading($folder_key)
    {
        $marker = self::podcastDownloadingMarkerFile($folder_key);
        clearstatcache(true, $marker);
        if (!file_exists($marker)) {
            return false;
        }
        return (time() - filemtime($marker)) <
            C\PODCAST_DOWNLOAD_STALE_TIME;
    }
    /**
     * Whether an immediate-update request is pending for a folder.
     *
     * @param string $folder_key key from podcastFolderKey
     * @return bool true if an update-request marker exists
     */
    public static function isPodcastUpdateRequested($folder_key)
    {
        $marker = self::podcastRequestMarkerFile($folder_key);
        clearstatcache(true, $marker);
        return file_exists($marker);
    }
    /**
     * Writes an immediate-update request for a destination folder so
     * the next MediaUpdater loop downloads that folder's podcast
     * sources without waiting out the periodic interval. Refuses to
     * write one while the folder is already downloading, so a button
     * press during an active download cannot start a second one.
     *
     * @param string $folder_key key from podcastFolderKey
     * @return bool true if the request was written, false if a
     *      download is already in progress for the folder
     */
    public static function requestPodcastUpdate($folder_key)
    {
        if (self::isPodcastDownloading($folder_key)) {
            return false;
        }
        file_put_contents(
            self::podcastRequestMarkerFile($folder_key), time());
        return true;
    }
    /**
     * Refreshes the heartbeat on a folder's downloading marker so a
     * download that runs longer than PODCAST_DOWNLOAD_STALE_TIME
     * still reads as active. Does nothing when the marker is absent
     * (the download already finished or was never started for this
     * folder), so it is safe to call from the per-item loop.
     *
     * @param string $folder_key key from podcastFolderKey
     */
    public static function heartbeatPodcastDownload($folder_key)
    {
        if ($folder_key === "" || $folder_key === null) {
            return;
        }
        $marker = self::podcastDownloadingMarkerFile($folder_key);
        if (file_exists($marker)) {
            touch($marker);
        }
    }
    /**
     * Only update if its been more than an hour since the last update
     *
     * @return bool whether its been an hour since the last update
     */
    public function checkPrerequisites()
    {
        $time = time();
        $delta = $time - $this->update_time;
        $this->requested_folder_keys = self::pendingPodcastRequests();
        if (!empty($this->requested_folder_keys)) {
            $this->request_driven = true;
            L\crawlLog("Performing requested podcast folder update");
            return true;
        }
        if ($delta > C\PODCAST_UPDATE_INTERVAL) {
            $this->update_time = $time;
            $this->request_driven = false;
            L\crawlLog("Performing media podcasts update");
            return true;
        }
        L\crawlLog("Time since last update not exceeded, skipping wiki
            update");
        return false;
    }
    /**
     * Folder keys with a pending immediate-update request, taken from
     * the request marker files in the schedules directory. A run
     * triggered by a request downloads only these folders' sources.
     *
     * @return array list of folder-key strings with a pending request
     */
    public static function pendingPodcastRequests()
    {
        $keys = [];
        $pattern = C\SCHEDULES_DIR . "/Podcast-*Request.txt";
        clearstatcache();
        foreach (glob($pattern) as $path) {
            $name = basename($path);
            $keys[] = substr($name, 0, -strlen("Request.txt"));
        }
        return $keys;
    }
    /**
     * Get the media sources from the local database and use those to run the
     * the same task as in the distributed setting
     */
    public function nondistributedTasks()
    {
        $db = $this->db;
        $sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='feed_podcast' OR ".
            "TYPE='scrape_podcast')";
        $result = $db->execute($sql);
        $podcasts = [];
        while ($podcast = $db->fetchArray($result)) {
            $this->parsePodcastAuxInfo($podcast);
            if (!isset($podcast["PREVIOUSLY_DOWNLOADED"])) {
                continue;
            }
            if ($this->request_driven && !in_array(
                $podcast["FOLDER_KEY"], $this->requested_folder_keys)) {
                continue;
            }
            $podcasts[] = $podcast;
        }
        if ($this->request_driven) {
            foreach ($this->requested_folder_keys as $folder_key) {
                $request_marker =
                    self::podcastRequestMarkerFile($folder_key);
                if (file_exists($request_marker)) {
                    unlink($request_marker);
                }
            }
        }
        $this->tasks = $podcasts;
        $this->doTasks($podcasts);
    }
    /**
     * For each podcast source downloads the podcast web file, checks which
     * podcast items are  not in the database, adds them.
     *
     * @param array $tasks array of feed info (url to download, paths to
     *  extract etc)
     */
    public function doTasks($tasks)
    {
        if (!is_array($tasks)) {
            L\crawlLog(
                "----This media updater is NOT responsible for any podcasts!");
            return;
        }
        $podcasts = $tasks;
        L\crawlLog("----This media updater is responsible for the podcasts:");
        $i = 1;
        foreach ($podcasts as $podcast) {
            L\crawlLog("----  $i. " . $podcast["NAME"]);
            $i++;
        }
        $num_podcasts = count($podcasts);
        $podcasts_one_go = self::MAX_PODCASTS_ONE_GO;
        $limit = 0;
        while ($limit < $num_podcasts) {
            $podcasts_batch = array_slice($podcasts, $limit,
                $podcasts_one_go);
            $this->updatePodcastsOneGo($podcasts_batch);
            $limit += $podcasts_one_go;
        }
    }
    /**
     * Used to fill in details for an associative arrays containing the
     * details of a  Wiki feed and scrape podcast which should be examined
     * to see if new items should be downloaded to wiki pages. As part of
     * processing expired feed items for the given wiki might be deleted.
     *
     * @param array &$podcast after running will contain an associative
     *  array of details about a particular podcast. The input podcast
     *  is assumed to have at least the NAME, WIKI_PAGE, AUX_PATH, and CATEGORY
     *  fields filled in. The latter with the time in seconds till item
     *  expires. If successful the MAX_AGE (which is esseentially the value
     *  the CATEGORY field), WIKI_FILE_PATTERN, WIKI_PAGE_FOLDERS, and
     *  PREVIOUSLY_DOWNLOADED folders will be filled in.
     * @param boolean $test_mode if true then does not cull expired feed items
     *     from disk, but will return previously downloaded as if it had.
     */
    public function parsePodcastAuxInfo(&$podcast, $test_mode = false)
    {
        $locale_tag = $podcast["LANGUAGE"];
        $aux_parts = explode("###",
                html_entity_decode($podcast['AUX_INFO'], ENT_QUOTES));
        list($podcast['AUX_URL_XPATH'], , , , $podcast['LINK_XPATH'],
            $podcast['WIKI_PAGE']) = $aux_parts;
        $podcast['MAX_AGE'] = (empty($podcast['CATEGORY'])) ?
            self::ITEM_EXPIRES_TIME : $podcast['CATEGORY'];
        $group_model = $this->group_model;
        list($group_id, $page_id, $sub_path, $podcast["WIKI_FILE_PATTERN"]) =
            $group_model->getGroupIdPageIdSubPathFromName($podcast['WIKI_PAGE'],
            $locale_tag);
        $podcast["GROUP_ID"] = $group_id;
        $podcast["PAGE_ID"] = $page_id;
        $podcast["SUB_PATH"] = $sub_path;
        $podcast["FOLDER_KEY"] = self::podcastFolderKey($group_id,
            $page_id, $sub_path);
        $podcast["WIKI_PAGE_FOLDERS"] =
            $group_model->getGroupPageResourcesFolders($group_id, $page_id,
            $sub_path, true);
        if (empty($podcast["WIKI_PAGE_FOLDERS"][1])) {
            return;
        }
        $podcast["PREVIOUSLY_DOWNLOADED"] = [];
        list($resource_folder, $thumb_folder,) = $podcast["WIKI_PAGE_FOLDERS"];
        $podcast_download_file = $thumb_folder . "/" .
            L\crawlHash($podcast["NAME"]) . ".txt";
        if (file_exists($podcast_download_file)) {
            $podcast["PREVIOUSLY_DOWNLOADED"] = unserialize(file_get_contents(
                $podcast_download_file));
            $previously_downloaded = [];
            $max_age = $podcast["MAX_AGE"];
            $time = time();
            foreach ($podcast["PREVIOUSLY_DOWNLOADED"] as $guid => $item) {
                if ($max_age < 0||$item["SAVE_TIMESTAMP"] + $max_age > $time) {
                    $previously_downloaded[$guid] = $item;
                } else if (!$test_mode) {
                    $group_model->deleteResource($item['FILENAME'], $group_id,
                        $page_id, $sub_path);
                }
            }
            $podcast["PREVIOUSLY_DOWNLOADED"] = $previously_downloaded;
        }
    }
    /**
     * For each of a supplied list of podcast associative arrays,
     * downloads the non-expired media for that podcast to the wiki folder
     * specified.
     *
     * @param array &$podcasts an array of associative arrays of info about
     *  how and where to download podcasts to
     * @param int $age oldest age items to consider for download
     * @param boolean $test_mode if true then rather then updating items in
     *  wiki, returns as a string summarizing the results of the downloads
     *  that would occur as part of updating the podcast
     * @return mixed either true, or if $test_mode is true then the results
     *      as a string of the operations involved in downloading the podcasts
     */
    public function updatePodcastsOneGo($podcasts, $age = C\ONE_WEEK,
        $test_mode = false)
    {
        $test_results = "";
        $log_function = function ($msg, $log_tag = "div class='source-test'")
            use (&$test_results, $test_mode) {
            $close_tag= preg_split("/\s+/",$log_tag)[0];
            if ($test_mode) {
                $test_results .= "<$log_tag>$msg</$close_tag>\n";
            } else {
                L\crawlLog($msg);
            }
        };
        $podcasts = FetchUrl::getPages($podcasts, false, C\ONE_GIB,
            null, "SOURCE_URL", CrawlConstants::PAGE, false, null, true);
        foreach ($podcasts as $podcast) {
            if (empty($podcast[CrawlConstants::PAGE])) {
                $log_function(
                    "...No data in {$podcast['NAME']} feed skipping.", "h3");
                continue;
            }
            $log_function("----Updating {$podcast['NAME']}.", "h3");
            // strip namespaces
            $page = preg_replace('@<(/?)(\w+\s*)\:@u', '<$1',
                $podcast[CrawlConstants::PAGE]);
            if (!empty($page)) {
                $podcast[CrawlConstants::PAGE] = $page;
            }
            $mime_type = (empty($podcast[CrawlConstants::TYPE])) ?
                "application/xml" : $podcast[CrawlConstants::TYPE];
            $is_html = preg_match("/html/i", $mime_type);
            if ($test_mode) {
                $log_function("...Downloaded Podcast Data was:",
                    "h3");
                $out_dom = new \DOMDocument('1.0');
                $out_dom->preserveWhiteSpace = false;
                $out_dom->formatOutput = true;
                if ($is_html) {
                    set_error_handler(null);
                    @$out_dom->loadHTML($podcast[CrawlConstants::PAGE]);
                    restore_error_handler();
                    $podcast_contents = $out_dom->saveHTML();
                } else {
                    set_error_handler(null);
                    @$out_dom->loadXML($podcast[CrawlConstants::PAGE]);
                    restore_error_handler();
                    $podcast_contents = $out_dom->saveXML();
                }
                $podcast_contents = "<textarea disabled='disabled'>" .
                    htmlentities($podcast_contents) . "</textarea>";
                $log_function($podcast_contents);
            }
            $marker = "";
            if (!$test_mode && !empty($podcast["FOLDER_KEY"])) {
                $marker = self::podcastDownloadingMarkerFile(
                    $podcast["FOLDER_KEY"]);
                file_put_contents($marker, time());
            }
            if ($is_html && !empty($podcast['LINK_XPATH'])) {
                list($num_added, $test_info) =
                    $this->processHtmlPodcast($podcast, $age, $test_mode);
            } else if (!$is_html) {
                list($num_added, $test_info) =
                    $this->processFeedPodcast($podcast, $age, $test_mode);
            } else {
                $num_added = 0;
                $test_info = "";
            }
            if ($marker !== "" && file_exists($marker)) {
                unlink($marker);
            }
            $test_results .= $test_info;
            if ($num_added > 0 && !$test_mode) {
                $podcast_download_file = $podcast["WIKI_PAGE_FOLDERS"][1] .
                    "/" . L\crawlHash($podcast["NAME"]) . ".txt";
                file_put_contents($podcast_download_file,
                    serialize($podcast["PREVIOUSLY_DOWNLOADED"]));
            } else if ($num_added <= 0) {
                $log_function("----could not parse any podcasts from page.");
            }
        }
        return ($test_mode) ? $test_results : true;
    }
    /**
     * Used to download the media item associated with an HTML scrape podcast
     *
     * @param array &$podcast associative array containing info about the
     *  location, how to handle, and where to download the podcast
     * @param int $age max age of an the media item to be considered for
     *  download
     * @param boolean $test_mode if true then rather then updating items in
     *  wiki, returns as a string summarizing the results of the downloads
     *  that would occur as part of updating the podcast
     * @return array [whether item downloaded, test_mode_info_string if
     *      applicable or "" otherwise]
     */
    public function processHtmlPodcast(&$podcast, $age, $test_mode = false)
    {
        $test_results = "";
        $log_function = function ($msg, $log_tag = "div class='source-test'")
            use (&$test_results, $test_mode) {
            $close_tag= preg_split("/\s+/",$log_tag)[0];
            if ($test_mode) {
                $test_results .= "<$log_tag>$msg</$close_tag>\n";
            } else {
                L\crawlLog($msg);
            }
        };
        $page = $podcast[CrawlConstants::PAGE];
        $dom = L\getDomFromString($page);
        $source_url = $podcast["SOURCE_URL"];
        if (!empty($podcast['AUX_URL_XPATH'])) {
            $sub_aux_xpaths = explode("\n", $podcast['AUX_URL_XPATH']);
            $log_function("...Processing the following AUX PATHS:", "h3");
            $log_function(print_r($sub_aux_xpaths, true));
            foreach ($sub_aux_xpaths as $aux_xpath) {
                $aux_url = $this->getLinkFromQueryPage($aux_xpath,
                    $page, $dom, $source_url);
                if (empty($aux_url)) {
                    $log_function("Downloading aux_url for xpath $aux_xpath ".
                        "was empty, bailing...", "h3");
                    break;
                }
                $log_function("Downloading aux_url $aux_url", "h3");
                $page = FetchUrl::getPage($aux_url);
                if ($test_mode) {
                    $log_function("...Downloaded Aux Data was:",
                        "h3");
                    $out_dom = new \DOMDocument('1.0');
                    $out_dom->preserveWhiteSpace = false;
                    $out_dom->formatOutput = true;
                    set_error_handler(null);
                    @$out_dom->loadHTML($page);
                    restore_error_handler();
                    $aux_contents = $out_dom->saveHTML();
                    $aux_contents = "<textarea disabled='disabled'>" .
                        htmlentities($aux_contents) . "</textarea>";
                    $log_function($aux_contents);
                }
                $dom = L\getDomFromString($page);
            }
        }
        $url = $this->getLinkFromQueryPage($podcast['LINK_XPATH'], $page, $dom,
             $source_url);
        if (empty($aux_url)) {
            $aux_url = $url;
        }
        $log_function("----...done extracting info. Check for new ".
            "podcast item in {$podcast['NAME']}.", "h3");
        if ($test_mode) {
            $log_function("----...Found following item...", "h3");
        }
        $item = ['guid' => L\crawlHash($url), 'link' => $url,
            'pubdate' => time(), 'title' => $aux_url];
        if ($test_mode) {
            $log_function(print_r($item, true));
            $did_add = true;
        } else {
            $did_add = $this->downloadPodcastItemIfNew($item, $podcast,
                $age);
        }
        if ($did_add) {
            return [1, $test_results];
        }
    }
    /**
     * Used to extract a URL from a page either as a string of in dom form
     * and to canonicalize it based on a starting url.
     *
     * @param string $xpath either an xpath to look into a dom object or
     *      a regex to search a page as a string
     * @param string $page source page to search in as a string
     * @param string $dom source page as a dom object
     * @param string $source_url url to use to canonicalize an incomplete
     *  url if the extraction only produces part of a url
     * @return string desired url link
     */
    public function getLinkFromQueryPage($xpath, $page, $dom, $source_url)
    {
        $dom_xpath = new \DOMXPath($dom);
        set_error_handler(null);
        $nodes = @$dom_xpath->evaluate($xpath);
        restore_error_handler();
        $url = false;
        if ($nodes === false) {
            $regex_json_parts = explode("json|", $xpath);
            set_error_handler(null);
            @preg_match_all(trim($regex_json_parts[0]), $page, $matches);
            restore_error_handler();
            if (!empty($matches[1][0])) {
                $url = $matches[1][0];
                if (!empty($regex_json_parts[1])) {
                    $json_data = json_decode(trim($url, ";"), true);
                    $play_list_path = explode("|", $regex_json_parts[1]);
                    foreach ($play_list_path as $path) {
                        if (!empty($json_data[$path])) {
                            $json_data = $json_data[$path];
                        }
                    }
                    $url = $json_data;
                } else if (isset($regex_json_parts[1]) && $url[0] == "\"") {
                    $url = json_decode($url);
                }
                if (is_string($url)) {
                    $url = preg_replace('/\\\u002F/i', "/", $url);
                } else {
                    $url = "";
                }
            }
        } else if ($nodes && $nodes->item(0)) {
            $node_item = $nodes->item(0);
            if (in_array($node_item->nodeName, ["a", "meta", "link"])) {
                $href = $node_item->attributes->getNamedItem("href");
                if(!empty($href)) {
                    $url = $href->nodeValue;
                }
                if (!$url && $node_item->nodeName == 'meta') {
                    $content = $node_item->attributes->getNamedItem("content");
                    if(!empty($content)) {
                        $url = $content->nodeValue;
                    }
                }
            } else if (in_array($node_item->nodeName, ["video", "audio"])) {
                $src = $node_item->attributes->getNamedItem("src");
                if(!empty($src)) {
                    $url = $src->nodeValue;
                }
            }
            if (!$url) {
                $url = $nodes->item(0)->nodeValue;
            }
        }
        if ($url) {
            $url = UrlParser::canonicalLink(
                urldecode(html_entity_decode($url)), $source_url);
        }
        return $url;
    }
    /**
     * Processes the page contents of one podcast feed. Determines which
     * podcast files on that page are fresh and if a podcast is fresh downloads
     * it and moves it to the appropariate wiki folder.
     *
     * @param array &$podcast associative array containing page data
     *  for a podcast feed page (not the video or audio files of a particular
     *  podcast on that page) together with rules for how to process it
     * @param int $age how many seconds ago is still considered a recent
     *  enough podcast to process
     * @param boolean $test_mode if true then rather then updating items in
     *  wiki, returns as a string summarizing the results of the downloads
     *  that would occur as part of updating the podcast
     * @return mixed either true, or if $test_mode is true then the results
     *      as a string of the operations involved in downloading the podcasts
     */
    public function processFeedPodcast(&$podcast, $age, $test_mode = false)
    {
        $test_results = "";
        $log_function = function ($msg, $log_tag = "div class='source-test'")
            use (&$test_results, $test_mode) {
            $close_tag= preg_split("/\s+/",$log_tag)[0];
            if ($test_mode) {
                $test_results .= "<$log_tag>$msg</$close_tag>\n";
            } else {
                L\crawlLog($msg);
            }
        };
        $page = $podcast[CrawlConstants::PAGE];
        $page = preg_replace("@<link@", "<slink", $page);
        $page = preg_replace("@</link@", "</slink", $page);
        $page = preg_replace("@pubDate@i", "pubdate", $page);
        $page = preg_replace("@&lt;@", "<", $page);
        $page = preg_replace("@&gt;@", ">", $page);
        $page = preg_replace("@<!\[CDATA\[(.+?)\]\]>@s", '$1', $page);
        // we also need a hack to make UTF-8 work correctly
        $dom = L\getDomFromString($page);
        $dom->encoding = 'UTF-8';
        $nodes = $dom->getElementsByTagName('item');
        // see above comment on why slink rather than link
        $item_elements = ["title" => "title",
            "description" => "description", "link" =>"slink",
            "guid" => "guid", "pubdate" => "pubdate"];
        if ($nodes->length == 0) {
            // maybe we're dealing with atom rather than rss
            $nodes = $dom->getElementsByTagName('entry');
            $item_elements = [
                "title" => "title",
                "description" => ["summary", "content"],
                "link" => "slink", "guid" => "id",
                "pubdate" => "updated"];
        }
        if (!empty($podcast['LINK_XPATH'])) {
            $item_elements['link'] = $podcast['LINK_XPATH'];
        }
        $log_function("----...done extracting info. Check for new ".
            "podcast items in {$podcast['NAME']}.", "h3");
        if ($test_mode) {
            $log_function("----...Found following items...", "h3");
        }
        $num_added = 0;
        $num_seen = 0;
        foreach ($nodes as $node) {
            $item = [];
            foreach ($item_elements as $db_element => $podcast_element) {
                if (!$test_mode) {
                    L\crawlTimeoutLog(
                        "----still adding podcast items to index.");
                }
                if ($db_element == "link" && substr($podcast_element, 0, 4)
                    == "http") {
                    $element_text = $podcast_element;
                    $item[$db_element] = strip_tags($element_text);
                    continue;
                }
                if (!is_array($podcast_element)) {
                    $podcast_element = [$podcast_element];
                }
                foreach ($podcast_element as $tag_name) {
                    $tag_node = $node->getElementsByTagName(
                            $tag_name)->item(0);
                    $element_text = (is_object($tag_node)) ?
                        $tag_node->nodeValue: "";
                    if ($element_text) {
                        break;
                    }
                }
                if ($db_element == "link" && $tag_node &&
                    empty($element_text)) {
                    $element_text = $tag_node->getAttribute("href");
                    if (empty($element_text)) {
                        $element_text = $tag_node->getAttribute("url");
                    }
                    if (empty($element_text)) {
                        $element_text = $tag_node->getAttribute("src");
                    }
                    $element_text = UrlParser::canonicalLink($element_text,
                        $podcast["SOURCE_URL"]);
                }
                $item[$db_element] = strip_tags($element_text);
            }
            if (!empty($item['link']) && !empty($item['title']) &&
                !empty($item_elements['link'])
                && $item['link'] == $item_elements['link']) {
                $item['link'] .= "#" . L\crawlHash($item['title']);
            }
            if (empty($item['guid'])) {
                $item['guid'] = L\crawlHash($item['link'] . $item['title']);
            }
            if ($test_mode) {
                $log_function(print_r($item, true));
                $did_add = true;
            } else {
                $did_add = $this->downloadPodcastItemIfNew($item, $podcast,
                    $age);
            }
            if ($did_add) {
                $num_added++;
            }
            $num_seen++;
        }
        return [$num_added, $test_results];
    }
    /**
     * Given a podcast item from a podcast feed page determines if it has
     * been downloaded or not and if not whether it is recent enough to
     * download. If it is recent enough, it scrapes the file to download
     * and downloads any other intermediate files need to find the
     * file to download, then finally downloads this podcast item. If the
     * podcast item is built out of multiple videos, it concatenates them
     * and makes a single video. It then moves the podcast item to the
     * appropriate wiki folder.
     *
     * @param array $item an associative array about one item on a podcast
     *  feed page
     * @param array &$podcast a reference to an associate array of the podcast
     *  feed the item is from. This is used for the language etc of the item
     *  and is also used to store updates to what podcasts have already been
     *  downloaded
     * @param int $age how many seconds ago is still considered a recent
     *  enough podcast to process
     * @return bool whether downloaded or not.
     */
    public function downloadPodcastItemIfNew($item, &$podcast, $age)
    {
        if (!empty($podcast["FOLDER_KEY"])) {
            self::heartbeatPodcastDownload($podcast["FOLDER_KEY"]);
        }
        $group_model = $this->group_model;
        $controller = new CrawlController(); //only need for clean() method
        $pubdate = (empty($item['pubdate'])) ? time():
            (is_int($item['pubdate']) ? $item['pubdate'] :
            strtotime($item['pubdate']));
        if ($pubdate + $age < time()) {
            L\crawlLog("Podcast Item: {$item['title']} is too old skipping...");
            return false;
        }
        if (key_exists($item['guid'], $podcast["PREVIOUSLY_DOWNLOADED"])) {
            $still_present = true;
            if ($this->request_driven &&
                !empty($podcast["WIKI_PAGE_FOLDERS"][0])) {
                $existing_name = $podcast["PREVIOUSLY_DOWNLOADED"][
                    $item['guid']]["FILENAME"] ?? "";
                $existing_path = $podcast["WIKI_PAGE_FOLDERS"][0] .
                    "/" . $existing_name;
                clearstatcache(true, $existing_path);
                $still_present = ($existing_name !== "" &&
                    file_exists($existing_path));
            }
            if ($still_present) {
                L\crawlLog("Podcast Item: {$item['title']} already ".
                    "downloaded, skipping...");
                return false;
            }
            L\crawlLog("Podcast Item: {$item['title']} missing from ".
                "folder, re-downloading...");
        }
        $type = UrlParser::getDocumentType($item['link']);
        $file_name = UrlParser::getDocumentFilename($item['link']);
        if (empty($file_name)) {
            $file_name = date("Y-m-d-H-i");
        }
        if (!empty($type)) {
            $file_name .= ".$type";
        }
        list($group_id, $page_id, $sub_path, $file_pattern) =
            $group_model->getGroupIdPageIdSubPathFromName($podcast['WIKI_PAGE'],
            $podcast['LANGUAGE']);
        $file_name = $this->makeFileNamePattern($file_name, $file_pattern,
            substr($item['title'], 0, C\NAME_LEN), $pubdate);
        $file_name = $controller->clean($file_name, "file_name");
        $type = UrlParser::getDocumentType($file_name);
        if (!empty($item['link'])) {
            $data = $this->downloadPodcastItem($item['link'], $type);
            if ($data) {
                $group_model->copyFileToGroupPageResource("", $file_name,
                    L\mimeType($file_name), $group_id, $page_id, $sub_path,
                    $data);
                $podcast["PREVIOUSLY_DOWNLOADED"][$item['guid']] = [
                    "FILENAME" => $file_name,
                    "SAVE_TIMESTAMP" => time()
                ];
                return true;
            }
        }
        return false;
    }
    /**
     * Helper method to @see downloadPodcastItemIfNew called when it is known
     * that a podcast item should be downloaded. It downloads a podcast item.
     * If the podcast item is an intermediate file pointing to several
     * items to download such as video. It downloads these and concatenates
     * them to makes a single video.
     *
     * @param string $url of podcast item to download
     * @param string $type file type of podcast item
     * @param array $audiolist_urls an array of audio urls to download
     *      if this has already been obtained
     * @return string with podcast item if successful or false otherwise
     */
    public function downloadPodcastItem($url, $type = "mp4",
        $audiolist_urls =[])
    {
        L\crawlLog("Downloading ...$url");
        $data = $this->getPage($url);
        if (!str_starts_with($data, "#EXTM3")) {
            return $data;
        }
        $convert_folder = C\SCHEDULES_DIR . self::CONVERT_FOLDER;
        if (!$this->makeFolder($convert_folder)) {
            return false;
        }
        $convert_folder .= "/" . L\crawlHash($url);
        if (!$this->makeFolder($convert_folder)) {
            return false;
        }
        if (in_array($type, ['mp4', 'm4v', 'mov', 'avi']) &&
            preg_match('/\#EXT\-X\-MEDIA\s*\:\s*TYPE\s*=\s*AUDIO[^\n]+/', $data,
            $audio_match)) {
            $audio_line = $audio_match[0];
            if (preg_match("/URI\s*\=\s*[\"\']([^\"\']+)[\"\']/i", $audio_line,
                $audio_uri_match)) {
                $audio_url = $audio_uri_match[1];
                if (preg_match("/GROUP-ID\s*\=\s*[\"\']([^\"\']+)[\"\']/i",
                    $audio_line, $audio_group_id_match)) {
                    $group_id = $audio_group_id_match[1];
                }
                if (!empty($group_id) &&
                    !empty($audio_url = $audio_uri_match[1])) {
                    $audio_url = UrlParser::canonicalLink($audio_url, $url);
                    $audio_data = $this->getPage($audio_url);
                    $audiolist_urls = [];
                    $lines = preg_split('/(\r)?\n/', $audio_data);
                    $num_lines = count($lines);
                    for ($i = 0; $i  < $num_lines; $i++) {
                        if (stristr($lines[$i], "#EXTINF") &&
                            !empty($lines[$i + 1])) {
                            $audiolist_urls[] = UrlParser::canonicalLink(
                                trim($lines[$i + 1]), $audio_url);
                        }
                    }
                }
            }
        }
        if (empty($audiolist_urls)) {
            $group_id = "";
        }
        if (stripos($data, "#EXT-X-STREAM-INF") !== false) {
            $lines = preg_split('/(\r)?\n/', $data);
            $max_url = "";
            $max_res = 0;
            $num_lines = count($lines);
            for ($i = 0; $i  < $num_lines; $i++) {
                if (preg_match('/\s*\#EXT\-X\-STREAM\-INF(.*)' .
                    preg_quote($group_id) . '.*/i', $lines[$i],
                    $matches)) {
                    if (empty($max_url) && !empty($lines[$i + 1])) {
                        $max_url = $lines[$i + 1];
                    }
                }
                if (!empty($matches[1]) &&
                    preg_match('/RESOLUTION\=(.+)x(.+)/', $matches[1],
                    $sub_matches)) {
                    $width = intval($sub_matches[1]);
                    $height = intval($sub_matches[2]);
                    $res = $width * $height;
                    if ($max_res < $res && !empty($lines[$i + 1])) {
                        $max_url = $lines[$i + 1];
                        $max_res = $res;
                    }
                }
            }
            if (empty($max_url)) {
                return false;
            }
            $max_url = UrlParser::canonicalLink($max_url, $url);
            return $this->downloadPodcastItem($max_url, $type,
                $audiolist_urls);
        }
        $segments_txt_file = $convert_folder . "/segments.txt";
        file_put_contents($segments_txt_file, "");
        // find the video segments file for today's news
        $seen = [];
        $playlists = preg_split("/\#EXT\-X\-MEDIA\-SEQUENCE\s*\:\s*\d*/",
            $data);
        if (count($playlists) > 1) {
            array_shift($playlists);
        }
        $playlist = $playlists[0];
        //get any encryption file if needed
        preg_match_all('/\#EXT\-X\-KEY\:METHOD\=AES\-128,URI\=\"(.*)\"/',
            $playlist, $key_file_matches);
        if (!empty($key_file_matches[1][0])) {
            $key_file = $key_file_matches[1][0];
            $key_url = UrlParser::canonicalLink($key_file, $url);
            L\crawlLog("Downloading encrypt key url... $key_url");
            $key = FetchUrl::getPage($key_url);
        }
        // Now get playlist urls to download
        $playlist_urls = [];
        $lines = preg_split('/(\r)?\n/', $playlist);
        $num_lines = count($lines);
        for ($i = 0; $i  < $num_lines; $i++) {
            if (stristr($lines[$i], "#EXTINF") && !empty($lines[$i + 1])) {
                $playlist_urls[] = trim($lines[$i + 1]);
            }
        }
        $i = 0;
        foreach ($playlist_urls as $playlist_url) {
            $play_name = UrlParser::getDocumentFilename($playlist_url);
            $file_name = $play_name . "." . UrlParser::getDocumentType(
                $playlist_url);
            if (!file_exists("$convert_folder/$file_name")) {
                $item_url = UrlParser::canonicalLink($playlist_url, $url);
                L\crawlLog("Downloading item url... $item_url");
                $data = $this->getPage($item_url);
                if (!empty($key)) {
                    $data = openssl_decrypt($data, "AES-128-CBC", $key,
                        OPENSSL_RAW_DATA);
                }
                $download_item_name = "$convert_folder/$file_name";
                L\crawlLog("...Writing downloaded file to $download_item_name");
                file_put_contents($download_item_name, $data);
                chmod($download_item_name, 0777);
                if (!empty($audiolist_urls[$i])) {
                    $audio_url = $audiolist_urls[$i];
                    $audio_name = UrlParser::getDocumentFilename($audio_url) .
                        "." . UrlParser::getDocumentType($audio_url);
                    $download_audio_name = "$convert_folder/audio-$audio_name";
                    if (!file_exists($download_audio_name)) {
                        L\crawlLog("Downloading audio url... $audio_url");
                        $audio_data = $this->getPage($audio_url);
                        if (!empty($key)) {
                            $audio_data  = openssl_decrypt($audio_data ,
                                "AES-128-CBC", $key, OPENSSL_RAW_DATA);
                        }
                        L\crawlLog("...Writing downloaded file to " .
                            "$download_audio_name");
                        file_put_contents($download_audio_name, $audio_data);
                        chmod($download_audio_name, 0777);
                    }
                    if (file_exists($download_item_name) &&
                        file_exists($download_audio_name)) {
                        $combine_name = "$convert_folder/".
                            "combine-$play_name.mp4";
                        $combine_ffmpeg = "ffmpeg -i $download_item_name " .
                            " -i $download_audio_name -c:v copy -c:a aac ".
                            "-strict experimental $combine_name";
                        L\crawlLog("...Combinining audio and video to  " .
                            "$combine_name with command\n $combine_ffmpeg");
                        exec($combine_ffmpeg);
                        chmod($combine_name, 0777);
                        $file_name = "combine-$play_name.mp4";
                    }
                }
            }
            file_put_contents($segments_txt_file,
                "file '$convert_folder/$file_name'\n", FILE_APPEND);
            $i++;
        }
        $out_file = "$convert_folder/out.$type";
        $ffmpeg = "ffmpeg -f concat -safe 0 -i $segments_txt_file " .
            " -c copy $out_file";
        L\crawlLog($ffmpeg);
        exec($ffmpeg);
        chmod($out_file, 0777);
        $data = file_get_contents($out_file);
        $model = new GroupModel();
        $model->db->unlinkRecursive($convert_folder);
        return $data;
    }
    /**
     * Used to construct a filename for a downloaded podcast item suitable
     * to be used when stored in a wiki page's resource folder
     *
     * @param string $file_name name of file
     * @param string $file_pattern string which can contain %F for previous
     *   filename, %T for title, and date %date_command, for example,
     *   %Y for year,  %m for month, %d for day, etc. These will be substituted
     *   with their values when wriitng out the wiki name for the downloaded
     *   podcast item.
     * @param string $title a title string for wiki item
     * @param int $pubdate when the wiki item was published as a Unix timestamp.
     *  The value of this is used when computing values for the $file_pattern
     * @return string output filename for wiki item
     */
    private function makeFileNamePattern($file_name, $file_pattern,
        $title = "", $pubdate = null)
    {
        $translates = ["'" => "", "\"" => "", "/" => "-", "|" => "-",
            '$' => '',];
        $file_name_parts = explode("?", $file_name);
        $file_name = strtr(basename($file_name_parts[0]), $translates);
        $title = strtr($title, $translates);
        if (empty($file_pattern)) {
            return $file_name;
        }
        if (!$pubdate) {
            $pubdate = time();
        }
        $pattern_parts = preg_split('/(\%\w)/u', $file_pattern, -1,
            PREG_SPLIT_DELIM_CAPTURE);
        $out_name = "";
        foreach ($pattern_parts as $pattern_part) {
            if (!empty($pattern_part[0]) && $pattern_part[0] == '%') {
                if ($pattern_part[1] == 'F') {
                    $out_name .= $file_name;
                } else if ($pattern_part[1] == 'T') {
                    $out_name .= $title;
                } else {
                    $out_name .= date($pattern_part[1], $pubdate);
                }
            } else {
                $out_name .= $pattern_part;
            }
        }
        //just in case the name came from a url with extra garbbage
        return $out_name;
    }
    /**
     * Makes a directory in a way compatible with yioop's error handling.
     *
     * @param string $folder name of directory/folder to create.
     * @return boolean whether directory was created
     */
    private function makeFolder($folder)
    {
        if (!file_exists($folder)) {
            set_error_handler(null);
            @mkdir($folder);
            restore_error_handler();
            if (!file_exists($folder)) {
                L\crawlLog("----Unable to create folder. Bailing!");
                return false;
            }
        }
        return true;
    }
    /**
     * Downloads the internet page with the give url.
     *
     * @param $url The url want to download
     * @return string contents of downloaded page
     */
    private function getPage($url)
    {
        return FetchUrl::getPage($url, null, false, null,
            4 * C\SINGLE_PAGE_TIMEOUT);
    }
}
