seek_quarry
[ class tree: seek_quarry ] [ index: seek_quarry ] [ all elements ]

Source for file admin_controller.php

Documentation is available at admin_controller.php

  1. <?php
  2. /**
  3.  *  SeekQuarry/Yioop --
  4.  *  Open Source Pure PHP Search Engine, Crawler, and Indexer
  5.  *
  6.  *  Copyright (C) 2009 - 2013  Chris Pollett chris@pollett.org
  7.  *
  8.  *  LICENSE:
  9.  *
  10.  *  This program is free software: you can redistribute it and/or modify
  11.  *  it under the terms of the GNU General Public License as published by
  12.  *  the Free Software Foundation, either version 3 of the License, or
  13.  *  (at your option) any later version.
  14.  *
  15.  *  This program is distributed in the hope that it will be useful,
  16.  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  17.  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18.  *  GNU General Public License for more details.
  19.  *
  20.  *  You should have received a copy of the GNU General Public License
  21.  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22.  *
  23.  *  END LICENSE
  24.  *
  25.  * @author Chris Pollett chris@pollett.org
  26.  * @package seek_quarry
  27.  * @subpackage controller
  28.  * @license http://www.gnu.org/licenses/ GPL3
  29.  * @link http://www.seekquarry.com/
  30.  * @copyright 2009 - 2013
  31.  * @filesource
  32.  */
  33.  
  34. if(!defined('BASE_DIR')) {echo "BAD REQUEST"exit();}
  35.  
  36. /** Load base controller class if needed */
  37. require_once BASE_DIR."/controllers/controller.php";
  38. /** Loads common constants for web crawling */
  39. require_once BASE_DIR."/lib/crawl_constants.php";
  40. /** Need get host for search filter admin */
  41. require_once BASE_DIR."/lib/url_parser.php";
  42. /** Used in rule parser test in page options */
  43. require_once BASE_DIR."/lib/page_rule_parser.php";
  44. /** Used to create, update, and delete user-trained classifiers. */
  45. require_once BASE_DIR."/lib/classifiers/classifier.php";
  46. /** Loads crawl_daemon to manage news_updater */
  47. require_once BASE_DIR."/lib/crawl_daemon.php";
  48. /** get processors for different file types */
  49. foreach(glob(BASE_DIR."/lib/processors/*_processor.php"as $filename{
  50.     require_once $filename;
  51. }
  52. /**
  53.  * Controller used to handle admin functionalities such as
  54.  * modify login and password, CREATE, UPDATE,DELETE operations
  55.  * for users, roles, locale, and crawls
  56.  *
  57.  * @author Chris Pollett
  58.  * @package seek_quarry
  59.  * @subpackage controller
  60.  */
  61.  
  62. class AdminController extends Controller implements CrawlConstants
  63. {
  64.     /**
  65.      * Says which views to load for this controller
  66.      * admin is the main one, signin has the login screen crawlstatus
  67.      * is used to see how many pages crawled by the current crawl
  68.      * @var array 
  69.      */
  70.     var $views = array("admin","signin","crawlstatus""machinestatus");
  71.     /**
  72.      * Says which models to load for this controller.
  73.      * @var array 
  74.      */
  75.     var $models = array(
  76.         "signin""user""activity""crawl""role""locale""profile",
  77.         "searchfilters""source""machine""cron""group");
  78.     /**
  79.      * Says which activities (roughly methods invoke from the web) this
  80.      * controller will respond to
  81.      * @var array 
  82.      */
  83.     var $activities = array("signin""manageAccount""manageUsers",
  84.         "manageRoles""manageGroups","manageCrawls","pageOptions",
  85.         "manageClassifiers","resultsEditor""manageMachines""manageLocales",
  86.         "crawlStatus","mixCrawls","machineStatus","searchSources","configure");
  87.     /**
  88.      * An array of activities which are periodically updated within other
  89.      * activities that they live. For example, within manage crawl,
  90.      * the current crawl status is updated every 20 or so seconds.
  91.      * @var array 
  92.      */
  93.     var $status_activities = array("crawlStatus""machineStatus");
  94.  
  95.  
  96.  
  97.     /**
  98.      * This is the main entry point for handling requests to administer the
  99.      * Yioop/SeekQuarry site
  100.      *
  101.      * ProcessRequest determines the type of request (signin , manageAccount,
  102.      * etc) is being made.  It then calls the appropriate method to handle the
  103.      * given activity. Finally, it draws the relevant admin screen
  104.      */
  105.     function processRequest()
  106.     {
  107.         $data array();
  108.  
  109.         if(!PROFILE{
  110.             return $this->configureRequest();
  111.         }
  112.         $view "signin";
  113.  
  114.         if(isset($_SESSION['USER_ID'])) {
  115.             $user $_SESSION['USER_ID'];
  116.         else {
  117.             $user $_SERVER['REMOTE_ADDR'];
  118.         }
  119.         $data[CSRF_TOKEN$this->generateCSRFToken($user);
  120.         $token_okay $this->checkCSRFToken(CSRF_TOKEN$user);
  121.         if($token_okay{
  122.             if(isset($_SESSION['USER_ID']&& !isset($_REQUEST['u'])) {
  123.                 $data array_merge($data$this->processSession());
  124.                 if(!isset($data['REFRESH'])) {
  125.                     $view "admin";
  126.                 else {
  127.                     $view $data['REFRESH'];
  128.                 }
  129.             else if ($this->checkSignin()){
  130.                 $user_id $this->signinModel->getUserId(
  131.                     $this->clean($_REQUEST['u']"string"));
  132.                 $session $this->userModel->getUserSession($user_id);
  133.                 if(is_array($session)) {
  134.                     $_SESSION $session;
  135.                 }
  136.                 $_SESSION['USER_ID'$user_id;
  137.                 $data[CSRF_TOKEN$this->generateCSRFToken(
  138.                     $_SESSION['USER_ID']);
  139.                 // now don't want to use remote address anymore
  140.                 $data['SCRIPT'"doMessage('<h1 class=\"red\" >".
  141.                     tl('admin_controller_login_successful')."</h1>')";
  142.                 $data array_merge($data$this->processSession());
  143.                 $view "admin";
  144.             else {
  145.                 $data['SCRIPT'"doMessage('<h1 class=\"red\" >".
  146.                     tl('admin_controller_login_failed')."</h1>')";
  147.                 unset($_SESSION['USER_ID']);
  148.             }
  149.         else if($this->checkCSRFToken(CSRF_TOKEN"config")) {
  150.             $data['SCRIPT'"doMessage('<h1 class=\"red\" >".
  151.                 tl('admin_controller_login_to_config')."</h1>')";
  152.         else if(isset($_REQUEST['a']&&
  153.             in_array($_REQUEST['a']$this->status_activities)) {
  154.             e("<p class='red'>".
  155.                 tl('admin_controller_status_updates_stopped')."</p>");
  156.             exit();
  157.         }
  158.         if($token_okay && isset($_SESSION["USER_ID"])) {
  159.             $data["ADMIN"true;
  160.         else {
  161.             $data["ADMIN"false;
  162.         }
  163.         if($view == 'signin'{
  164.             unset($_SESSION['USER_ID']);
  165.             $data[CSRF_TOKEN$this->generateCSRFToken(
  166.                 $_SERVER['REMOTE_ADDR']);
  167.             $data['SCRIPT'"var u; if ((u = elt('username')) && u.focus) ".
  168.                "u.focus();";
  169.         }
  170.         $this->displayView($view$data);
  171.     }
  172.  
  173.     /**
  174.      * If there is no profile/work directory set up then this method
  175.      * get called to by pass any login and go to the configure screen.
  176.      * The configure screen is only displayed if the user is connected
  177.      * from localhost in this case
  178.      */
  179.     function configureRequest()
  180.     {
  181.         $data $this->processSession();
  182.         $data[CSRF_TOKEN$this->generateCSRFToken("config");
  183.         $this->displayView("admin"$data);
  184.     }
  185.  
  186.     /**
  187.      * Checks whether the user name and password sent presumably by the signin
  188.      * form match a user in the database
  189.      *
  190.      * @return bool whether they do or not
  191.      */
  192.     function checkSignin()
  193.     {
  194.         $result $this->signinModel->checkValidSignin(
  195.         $this->clean($_REQUEST['u']"string"),
  196.         $this->clean($_REQUEST['p']"string") );
  197.         return $result;
  198.     }
  199.  
  200.     /**
  201.      * Determines the user's current allowed activities and current activity,
  202.      * then calls the method for the latter.
  203.      *
  204.      * This is called from {@link processRequest()} once a user is logged in.
  205.      *
  206.      * @return array $data the results of doing the activity for display in the
  207.      *       view
  208.      */
  209.     function processSession()
  210.     {
  211.         if(!PROFILE || (defined("FIX_NAME_SERVER"&& FIX_NAME_SERVER)) {
  212.             $activity "configure";
  213.         else if(isset($_REQUEST['a']&&
  214.             in_array($_REQUEST['a']$this->activities)) {
  215.             $activity $_REQUEST['a'];
  216.         else {
  217.             $activity "manageAccount";
  218.         }
  219.         $allowed true;
  220.         if(!PROFILE{
  221.             $allowed_activities arrayarray(
  222.                 "ACTIVITY_NAME" =>
  223.                 $this->activityModel->getActivityNameFromMethodName($activity),
  224.                 'METHOD_NAME' => $activity));
  225.             $allowed true;
  226.         else {
  227.             $allowed_activities =
  228.                  $this->userModel->getUserActivities($_SESSION['USER_ID']);
  229.         }
  230.         foreach($allowed_activities as $allowed_activity{
  231.             if($activity == $allowed_activity['METHOD_NAME']{
  232.                  $allowed true;
  233.             }
  234.             if($allowed_activity['METHOD_NAME'== "manageCrawls" &&
  235.                 $activity == "crawlStatus"{
  236.                 $allowed true;
  237.             }
  238.             if($allowed_activity['METHOD_NAME'== "manageMachines" &&
  239.                 $activity == "machineStatus"{
  240.                 $allowed true;
  241.             }
  242.         }
  243.  
  244.         //for now we allow anyone to get crawlStatus
  245.         if($allowed{
  246.             $data $this->$activity();
  247.             if(!is_array($data)) {
  248.                 $data array();
  249.             }
  250.             $data['ACTIVITIES'$allowed_activities;
  251.         }
  252.         if(!in_array($activity$this->status_activities)) {
  253.             $data['CURRENT_ACTIVITY'=
  254.                 $this->activityModel->getActivityNameFromMethodName($activity);
  255.         }
  256.         return $data;
  257.     }
  258.  
  259.     /**
  260.      * This method is data to signin a user and initialize the data to be
  261.      * display in a view
  262.      *
  263.      * @return array empty array of data to show so far in view
  264.      */
  265.     function signin()
  266.     {
  267.         $data array();
  268.         $_SESSION['USER_ID'=
  269.             $this->signinModel->getUserId($_REQUEST['username']);
  270.         return $data;
  271.     }
  272.  
  273.     /**
  274.      * Used to handle crawlStatus REST activities requesting the status of the
  275.      * current web crawl
  276.      *
  277.      * @return array $data contains crawl status of current crawl as well as
  278.      *       info about prior crawls and which crawl is being used for default
  279.      *       search results
  280.      */
  281.     function crawlStatus()
  282.     {
  283.         $data array();
  284.         $data['REFRESH'"crawlstatus";
  285.  
  286.         $crawl_time $this->crawlModel->getCurrentIndexDatabaseName();
  287.         if(isset($crawl_time) ) {
  288.             $data['CURRENT_INDEX'= (int)$crawl_time;
  289.         else {
  290.             $data['CURRENT_INDEX'= -1;
  291.         }
  292.  
  293.         $machine_urls $this->machineModel->getQueueServerUrls();
  294.         list($stalled$status$data['RECENT_CRAWLS']=
  295.             $this->crawlModel->combinedCrawlInfo($machine_urls);
  296.  
  297.         if($stalled{
  298.             $this->crawlModel->sendStopCrawlMessage($machine_urls);
  299.         }
  300.  
  301.         $data array_merge($data$status);
  302.  
  303.         $data["CRAWL_RUNNING"false;
  304.         if(isset($data['CRAWL_TIME']&& $data["CRAWL_TIME"!= 0{
  305.             //erase from previous crawl list any active crawl
  306.             $num_crawls count($data['RECENT_CRAWLS']);
  307.             for($i 0$i $num_crawls$i++{
  308.                 if($data['RECENT_CRAWLS'][$i]['CRAWL_TIME'==
  309.                     $data['CRAWL_TIME']{
  310.                     $data['RECENT_CRAWLS'][$ifalse;
  311.                 }
  312.             }
  313.             $data["CRAWL_RUNNING"true;
  314.             $data['RECENT_CRAWLS']array_filter($data['RECENT_CRAWLS']);
  315.         }
  316.         if(isset($data['RECENT_CRAWLS'][0])) {
  317.             rorderCallback($data['RECENT_CRAWLS'][0]$data['RECENT_CRAWLS'][0],
  318.                 'CRAWL_TIME');
  319.             usort($data['RECENT_CRAWLS']"rorderCallback");
  320.         }
  321.  
  322.         return $data;
  323.     }
  324.  
  325.     /**
  326.      * Gets data from the machineModel concerning the on/off states
  327.      * of the machines managed by this Yioop instance and then passes
  328.      * this data the the machinestatus view.
  329.      * @return array $data MACHINES field has information about each
  330.      *       machine managed by this Yioop instance as well the on off
  331.      *       status of its queue_servers and fetchers.
  332.      *       The REFRESH field is used to tell the controller that the
  333.      *       view shouldn't have its own sidemenu.
  334.      */
  335.     function machineStatus()
  336.     {
  337.         $data array();
  338.         $data['REFRESH'"machinestatus";
  339.         $data['MACHINES'$this->machineModel->getMachineStatuses();
  340.         $profile =  $this->profileModel->getProfile(WORK_DIRECTORY);
  341.         $data['NEWS_MODE'= isset($profile['NEWS_MODE']?
  342.             $profile['NEWS_MODE']"";
  343.         return $data;
  344.     }
  345.  
  346.     /**
  347.      * Used to handle the change current user password admin activity
  348.      *
  349.      * @return array $data SCRIPT field contains success or failure message
  350.      */
  351.     function manageAccount()
  352.     {
  353.         $possible_arguments array("changepassword");
  354.  
  355.         $data["ELEMENT""manageaccountElement";
  356.         $data['SCRIPT'"";
  357.  
  358.         if(isset($_REQUEST['arg']&&
  359.             in_array($_REQUEST['arg']$possible_arguments)) {
  360.             switch($_REQUEST['arg'])
  361.             {
  362.                 case "changepassword":
  363.                     if($_REQUEST['retypepassword'!= $_REQUEST['newpassword']){
  364.                         $data["MESSAGE"=
  365.                             tl('admin_controller_passwords_dont_match');
  366.                         $data['SCRIPT'.=
  367.                             "doMessage('<h1 class=\"red\" >"$data["MESSAGE"].
  368.                             "</h1>')";
  369.                         return $data;
  370.                     }
  371.                     $username =
  372.                         $this->signinModel->getUserName($_SESSION['USER_ID']);
  373.                     $result $this->signinModel->checkValidSignin($username,
  374.                     $this->clean($_REQUEST['oldpassword']"string") );
  375.                     if(!$result{
  376.                         $data["MESSAGE"=
  377.                             tl('admin_controller_invalid_old_password');
  378.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  379.                             $data["MESSAGE"]."</h1>')";
  380.                         return $data;
  381.                     }
  382.                     $this->signinModel->changePassword($username,
  383.                         $this->clean($_REQUEST['newpassword']"string"));
  384.                     $data["MESSAGE"tl('admin_controller_change_password');
  385.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  386.                         $data["MESSAGE"]."</h1>')";
  387.                 break;
  388.                 }
  389.         }
  390.  
  391.         return $data;
  392.     }
  393.  
  394.     /**
  395.      * Used to handle the manage user activity.
  396.      *
  397.      * This activity allows new users to be added, old users to be
  398.      * deleted and allows roles to be added to/deleted from a user
  399.      *
  400.      * @return array $data infomation about users of the system, roles, etc.
  401.      *       as well as status messages on performing a given sub activity
  402.      */
  403.     function manageUsers()
  404.     {
  405.         $possible_arguments array("adduser",
  406.             "deleteuser""adduserrole""deleteuserrole");
  407.  
  408.         $data["ELEMENT""manageusersElement";
  409.         $data['SCRIPT'=
  410.             "selectUser = elt('select-user'); ".
  411.             "selectUser.onchange = submitViewUserRole;";
  412.  
  413.         $usernames $this->userModel->getUserList();
  414.         if(isset($_REQUEST['username'])) {
  415.             $username $this->clean($_REQUEST['username']"string" );
  416.         }
  417.         $base_option tl('admin_controller_select_username');
  418.         $data['USER_NAMES'array();
  419.         $data['USER_NAMES'][""$base_option;
  420.  
  421.         foreach($usernames as $name{
  422.             $data['USER_NAMES'][$name]$name;
  423.         }
  424.  
  425.         if(isset($_REQUEST['selectuser'])) {
  426.             $select_user $this->clean($_REQUEST['selectuser']"string" );
  427.         else {
  428.             $select_user "";
  429.         }
  430.         if($select_user != "" {
  431.             $userid $this->signinModel->getUserId($select_user);
  432.             $data['SELECT_USER'$select_user;
  433.             $data['SELECT_ROLES'$this->userModel->getUserRoles($userid);
  434.             $all_roles $this->roleModel->getRoleList();
  435.             $role_ids array();
  436.             if(isset($_REQUEST['selectrole'])) {
  437.                 $select_role $this->clean($_REQUEST['selectrole']"string" );
  438.             else {
  439.                 $select_role "";
  440.             }
  441.  
  442.             foreach($all_roles as $role{
  443.                 $role_ids[$role['ROLE_ID'];
  444.                 if($select_role == $role['ROLE_ID']{
  445.                     $select_rolename $role['ROLE_NAME'];
  446.                 }
  447.             }
  448.  
  449.             $select_role_ids array();
  450.             foreach($data['SELECT_ROLES'as $role{
  451.                 $select_role_ids[$role['ROLE_ID'];
  452.             }
  453.             $available_roles array();
  454.             $tmp array();
  455.             foreach($all_roles as $role{
  456.                 if(!in_array($role['ROLE_ID']$select_role_ids&&
  457.                     !isset($tmp[$role['ROLE_ID']])) {
  458.                     $tmp[$role['ROLE_ID']] true;
  459.                     $available_roles[$role;
  460.                 }
  461.             }
  462.  
  463.             $data['AVAILABLE_ROLES'][-1=
  464.                 tl('admin_controller_select_rolename');
  465.  
  466.             foreach($available_roles as $role{
  467.                 $data['AVAILABLE_ROLES'][$role['ROLE_ID']]$role['ROLE_NAME'];
  468.             }
  469.  
  470.             if($select_role != ""{
  471.                 $data['SELECT_ROLE'$select_role;
  472.             else {
  473.                 $data['SELECT_ROLE'= -1;
  474.             }
  475.         else {
  476.             $data['SELECT_USER'= -1;
  477.         }
  478.  
  479.         if(isset($_REQUEST['arg']&&
  480.             in_array($_REQUEST['arg']$possible_arguments)) {
  481.  
  482.             switch($_REQUEST['arg'])
  483.             {
  484.                 case "adduser":
  485.                     $data['SELECT_ROLE'= -1;
  486.                     unset($data['AVAILABLE_ROLES']);
  487.                     unset($data['SELECT_ROLES']);
  488.                     if($_REQUEST['retypepassword'!= $_REQUEST['password']{
  489.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  490.                             tl('admin_controller_passwords_dont_match').
  491.                             "</h1>')";
  492.                         return $data;
  493.                     }
  494.  
  495.                     if($this->signinModel->getUserId($username0{
  496.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  497.                             tl('admin_controller_username_exists')."</h1>')";
  498.                         return $data;
  499.                     }
  500.                     $this->userModel->addUser($username,
  501.                         $this->clean($_REQUEST['password']"string"));
  502.                     $data['USER_NAMES'][$username$username;
  503.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  504.                         tl('admin_controller_username_added')."</h1>')";
  505.                 break;
  506.  
  507.                 case "deleteuser":
  508.                     $data['SELECT_ROLE'= -1;
  509.                     unset($data['AVAILABLE_ROLES']);
  510.                     unset($data['SELECT_ROLES']);
  511.                     if(!($this->signinModel->getUserId($username0)) {
  512.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  513.                             tl('admin_controller_username_doesnt_exists').
  514.                             "</h1>')";
  515.                         return $data;
  516.                     }
  517.                     $this->userModel->deleteUser($username);
  518.                     unset($data['USER_NAMES'][$username]);
  519.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  520.                         tl('admin_controller_username_deleted')."</h1>')";
  521.  
  522.                 break;
  523.  
  524.                 case "adduserrole":
  525.                     if$userid <= {
  526.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  527.                             tl('admin_controller_username_doesnt_exists').
  528.                             "</h1>')";
  529.                         return $data;
  530.                     }
  531.                     if(!in_array($select_role$role_ids)) {
  532.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  533.                             tl('admin_controller_rolename_doesnt_exists').
  534.                             "</h1>')";
  535.                         return $data;
  536.                     }
  537.                     $this->userModel->addUserRole($userid$select_role);
  538.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  539.                         tl('admin_controller_rolename_added').
  540.                         "</h1>')";
  541.                     unset($data['AVAILABLE_ROLES'][$select_role]);
  542.                     $data['SELECT_ROLE'= -1;
  543.                     $data['SELECT_ROLES'=
  544.                         $this->userModel->getUserRoles($userid);
  545.                 break;
  546.  
  547.                 case "deleteuserrole":
  548.                     if($userid <= 0{
  549.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  550.                             tl('admin_controller_username_doesnt_exists').
  551.                             "</h1>')";
  552.                         return $data;
  553.                     }
  554.                     if(!in_array($select_role$role_ids)) {
  555.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  556.                             tl('admin_controller_rolename_doesnt_exists').
  557.                             "</h1>')";
  558.                         return $data;
  559.                     }
  560.                     $this->userModel->deleteUserRole($userid$select_role);
  561.                     $data['SELECT_ROLES'=
  562.                         $this->userModel->getUserRoles($userid);
  563.                     $data['AVAILABLE_ROLES'][$select_role$select_rolename;
  564.                     $data['SELECT_ROLE'= -1;
  565.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  566.                         tl('admin_controller_rolename_deleted')."</h1>')";
  567.                 break;
  568.             }
  569.         }
  570.  
  571.         return $data;
  572.     }
  573.  
  574.     /**
  575.      * Used to handle the manage role activity.
  576.      *
  577.      * This activity allows new roles to be added, old roles to be
  578.      * deleted and allows activities to be added to/deleted from a role
  579.      *
  580.      * @return array $data information about roles in the system, activities,
  581.      *       etc. as well as status messages on performing a given sub activity
  582.      *
  583.      */
  584.     function manageRoles()
  585.     {
  586.         $possible_arguments =
  587.             array("addrole""deleterole""addactivity""deleteactivity");
  588.  
  589.         $data["ELEMENT""managerolesElement";
  590.         $data['SCRIPT'=
  591.             "selectRole = elt('select-role'); selectRole.onchange =".
  592.             " submitViewRoleActivities;";
  593.  
  594.         $roles $this->roleModel->getRoleList();
  595.         $role_ids array();
  596.         $base_option tl('admin_controller_select_rolename');
  597.         $data['ROLE_NAMES'array();
  598.         $data['ROLE_NAMES'][-1$base_option;
  599.         if(isset($_REQUEST['rolename'])) {
  600.             $rolename $this->clean($_REQUEST['rolename']"string" );
  601.         }
  602.         foreach($roles as $role{
  603.             $data['ROLE_NAMES'][$role['ROLE_ID']]$role['ROLE_NAME'];
  604.             $role_ids[$role['ROLE_ID'];
  605.         }
  606.         $data['SELECT_ROLE'= -1;
  607.  
  608.  
  609.         if(isset($_REQUEST['selectrole'])) {
  610.             $select_role $this->clean($_REQUEST['selectrole']"string" );
  611.         else {
  612.             $select_role "";
  613.         }
  614.  
  615.         if($select_role != "" {
  616.             $data['SELECT_ROLE'$select_role;
  617.             $data['ROLE_ACTIVITIES'=
  618.                 $this->roleModel->getRoleActivities($select_role);
  619.             $all_activities $this->activityModel->getActivityList();
  620.             $activity_ids array();
  621.             $activity_names array();
  622.             foreach($all_activities as $activity{
  623.                 $activity_ids[$activity['ACTIVITY_ID'];
  624.                 $activity_names[$activity['ACTIVITY_ID']] =
  625.                     $activity['ACTIVITY_NAME'];
  626.             }
  627.  
  628.             $available_activities array();
  629.             $role_activity_ids array();
  630.             foreach($data['ROLE_ACTIVITIES'as $activity{
  631.                 $role_activity_ids[$activity["ACTIVITY_ID"];
  632.             }
  633.             $tmp array();
  634.             foreach($all_activities as $activity{
  635.                 if(!in_array($activity["ACTIVITY_ID"]$role_activity_ids&&
  636.                     !isset($tmp[$activity["ACTIVITY_ID"]])) {
  637.                     $tmp[$activity["ACTIVITY_ID"]] true;
  638.                     $available_activities[$activity;
  639.                 }
  640.             }
  641.             $data['AVAILABLE_ACTIVITIES'][-1=
  642.                 tl('admin_controller_select_activityname');
  643.  
  644.  
  645.             foreach($available_activities as $activity{
  646.                 $data['AVAILABLE_ACTIVITIES'][$activity['ACTIVITY_ID']] =
  647.                     $activity['ACTIVITY_NAME'];
  648.             }
  649.  
  650.             if(isset($_REQUEST['selectactivity'])) {
  651.                 $select_activity =
  652.                     $this->clean($_REQUEST['selectactivity']"int" );
  653.  
  654.             else {
  655.                 $select_activity "";
  656.             }
  657.             if($select_activity != ""{
  658.                 $data['SELECT_ACTIVITY'$select_activity;
  659.             else {
  660.                 $data['SELECT_ACTIVITY'= -1;
  661.             }
  662.  
  663.         }
  664.         if(isset($_REQUEST['arg']&&
  665.             in_array($_REQUEST['arg']$possible_arguments)) {
  666.  
  667.             switch($_REQUEST['arg'])
  668.             {
  669.                 case "addrole":
  670.                     unset($data['ROLE_ACTIVITIES']);
  671.                     unset($data['AVAILABLE_ACTIVITIES']);
  672.                     $data['SELECT_ROLE'= -1;
  673.                     if($this->roleModel->getRoleId($rolename0{
  674.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  675.                             tl('admin_controller_rolename_exists').
  676.                             "</h1>')";
  677.                         return $data;
  678.                     }
  679.  
  680.                     $this->roleModel->addRole($rolename);
  681.                     $roleid $this->roleModel->getRoleId($rolename);
  682.                     $data['ROLE_NAMES'][$roleid$rolename;
  683.  
  684.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  685.                         tl('admin_controller_rolename_added').
  686.                         "</h1>')";
  687.                 break;
  688.  
  689.                 case "deleterole":
  690.                     $data['SELECT_ROLE'= -1;
  691.                     unset($data['ROLE_ACTIVITIES']);
  692.                     unset($data['AVAILABLE_ACTIVITIES']);
  693.  
  694.                     if(!in_array($select_role$role_ids)) {
  695.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  696.                             tl('admin_controller_rolename_doesnt_exists').
  697.                             "</h1>')";
  698.                         return $data;
  699.                     }
  700.                     $this->roleModel->deleteRole($select_role);
  701.                     unset($data['ROLE_NAMES'][$select_role]);
  702.  
  703.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  704.                         tl('admin_controller_rolename_deleted')."</h1>')";
  705.                 break;
  706.  
  707.                 case "addactivity":
  708.                     if(!in_array($select_role$role_ids)) {
  709.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  710.                             tl('admin_controller_rolename_doesnt_exists').
  711.                             "</h1>')";
  712.                         return $data;
  713.                     }
  714.                     if(!in_array($select_activity$activity_ids)) {
  715.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  716.                             tl('admin_controller_activityname_doesnt_exists').
  717.                             "</h1>')";
  718.                         return $data;
  719.                     }
  720.                     $this->roleModel->addActivityRole(
  721.                         $select_role$select_activity);
  722.                     unset($data['AVAILABLE_ACTIVITIES'][$select_activity]);
  723.                     $data['ROLE_ACTIVITIES'=
  724.                         $this->roleModel->getRoleActivities($select_role);
  725.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  726.                         tl('admin_controller_activity_added')."</h1>')";
  727.                 break;
  728.  
  729.                 case "deleteactivity":
  730.                     if(!in_array($select_role$role_ids)) {
  731.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  732.                             tl('admin_controller_rolename_doesnt_exists').
  733.                             "</h1>')";
  734.                         return $data;
  735.                     }
  736.  
  737.                     if(!in_array($select_activity$activity_ids)) {
  738.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  739.                             tl('admin_controller_activityname_doesnt_exists').
  740.                             "</h1>')";
  741.                         return $data;
  742.                     }
  743.                     $this->roleModel->deleteActivityRole(
  744.                         $select_role$select_activity);
  745.                     $data['ROLE_ACTIVITIES'=
  746.                         $this->roleModel->getRoleActivities($select_role);
  747.                     $data['AVAILABLE_ACTIVITIES'][$select_activity=
  748.                         $activity_names[$select_activity];
  749.                     $data['SELECT_ACTIVITY'= -1;
  750.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  751.                         tl('admin_controller_activity_deleted')."</h1>')";
  752.                 break;
  753.             }
  754.         }
  755.  
  756.         return $data;
  757.     }
  758.  
  759.     /**
  760.      * Used to handle the manage group activity.
  761.      *
  762.      * This activity allows new groups to be created out of a set of users.
  763.      * It allows admin rights for the group to be transfered and it allows roles
  764.      * to be added to a group. One can also delete groups and roles from groups.
  765.      *
  766.      * @return array $data information about groups in the system
  767.      */
  768.     function manageGroups()
  769.     {
  770.         $possible_arguments array("addgroup""deletegroup",
  771.             "viewgroups""selectgroup""adduser""deleteuser",
  772.             "addrole""deleterole""updategroup");
  773.  
  774.         $data["ELEMENT""managegroupsElement";
  775.         $data['SCRIPT'=
  776.             "selectGroup = elt('select-group'); selectGroup.onchange =".
  777.             " submitViewGroups;";
  778.  
  779.         if(isset($_REQUEST['groupname'])) {
  780.             $groupname $this->clean($_REQUEST['groupname']"string" );
  781.         }
  782.  
  783.         if($_SESSION['USER_ID'== '1'{
  784.             $groups $this->groupModel->getGroupList();
  785.         else {
  786.             $groups $this->groupModel->getGroupListbyUser(
  787.                 $_SESSION['USER_ID']);
  788.         }
  789.  
  790.         $group_ids array();
  791.         $base_option tl('admin_controller_select_groupname');
  792.         $data['GROUP_NAMES'array();
  793.         $data['GROUP_NAMES'][-1$base_option;
  794.  
  795.         if(isset($_REQUEST['groupname'])) {
  796.             $groupname $this->clean($_REQUEST['groupname']"string" );
  797.         }
  798.  
  799.         foreach($groups as $group{
  800.             $data['GROUP_NAMES'][$group['GROUP_ID']]$group['GROUP_NAME'];
  801.             $group_ids[$group['GROUP_ID'];
  802.         }
  803.  
  804.         if($_SESSION['USER_ID'== '1'{
  805.             $groups $this->groupModel->getGroupList();
  806.         else {
  807.             $groups $this->groupModel->getGroupListbyCreator(
  808.                 $_SESSION['USER_ID']);
  809.         }
  810.  
  811.         $group_ids array();
  812.         $base_option tl('admin_controller_select_groupname');
  813.         $data['DELETE_GROUP_NAMES'array();
  814.         $data['DELETE_GROUP_NAMES'][-1$base_option;
  815.  
  816.         if(isset($_REQUEST['groupname'])) {
  817.             $groupname $this->clean($_REQUEST['groupname']"string" );
  818.         }
  819.  
  820.         foreach($groups as $deletegroup{
  821.             $data['DELETE_GROUP_NAMES'][$deletegroup['GROUP_ID']]=
  822.                 $deletegroup['GROUP_NAME'];
  823.             $group_ids[$deletegroup['GROUP_ID'];
  824.         }
  825.  
  826.         $data['SELECT_GROUP'= -1;
  827.         $data['SELECT_USER'= -1;
  828.  
  829.         if(isset($_REQUEST['selectgroup'])) {
  830.             $select_group $this->clean($_REQUEST['selectgroup']"string" );
  831.         else {
  832.             $select_group "";
  833.         }
  834.  
  835.         if(isset($_REQUEST['arg']&& !($_REQUEST['arg'== 'deletegroup'
  836.             || $_REQUEST['arg'== 'addgroup')) {
  837.  
  838.             $data['SELECT_GROUP'$select_group;
  839.             $grouproles $this->groupModel->getGroupRoles($select_group);
  840.             $data['GROUP_ROLES'$grouproles;
  841.             $rolenames $this->roleModel->getRoleList();
  842.             $base_option tl('admin_controller_select_rolename');
  843.  
  844.             if(isset($_REQUEST['arg']&& !($_REQUEST['arg'== 'deletegroup'
  845.                 || $_REQUEST['arg'== 'addgroup')) {
  846.                 $data['ROLE_NAMES'array();
  847.                 $role_ids array();
  848.                 $data['ROLE_NAMES'][-1$base_option;
  849.  
  850.                 foreach($rolenames as $rolename{
  851.                     $data['ROLE_NAMES'][$rolename['ROLE_ID']] =
  852.                         $rolename['ROLE_NAME'];
  853.                     $role_ids[$rolename['ROLE_ID'];
  854.                 }
  855.             }
  856.  
  857.             if(isset($_REQUEST['selectrole'])) {
  858.                 $select_role $this->clean($_REQUEST['selectrole']"string");
  859.             else {
  860.                 $select_role "";
  861.             }
  862.  
  863.             $usergroups $this->groupModel->getGroupUsers($select_group);
  864.             $data['GROUP_USERS'=$usergroups;
  865.  
  866.             if(isset($_REQUEST['selectuser'])) {
  867.                 $select_user $this->clean($_REQUEST['selectuser']"string" );
  868.             else {
  869.                 $select_user "";
  870.             }
  871.  
  872.             if($select_group != "-1"{
  873.                 $usernames $this->userModel->getUserListGroups();
  874.                 $base_option tl('admin_controller_select_username');
  875.                 $data['USER_NAMES'array();
  876.                 $user_ids array();
  877.                 $data['USER_NAMES'][-1$base_option;
  878.                 foreach($usernames as $user{
  879.                     $user_ids[$user['USER_ID'];
  880.                     if($select_user == $user['USER_ID']{
  881.                         $select_username $user['USER_NAME'];
  882.                     }
  883.                 }
  884.                 foreach($usernames as $username{
  885.                     $data['USER_NAMES'[$username['USER_ID']] =
  886.                         $username['USER_NAME'];
  887.                     $user_ids[$username['USER_ID'];
  888.                 }
  889.             }
  890.  
  891.  
  892.             if(isset($_REQUEST['selectactivity'])) {
  893.                 $select_activity =
  894.                     $this->clean($_REQUEST['selectactivity']"int" );
  895.             else {
  896.                 $select_activity "";
  897.             }
  898.  
  899.             if($select_activity != ""{
  900.                 $data['SELECT_ACTIVITY'$select_activity;
  901.             else {
  902.                 $data['SELECT_ACTIVITY'= -1;
  903.             }
  904.         }
  905.  
  906.         if(!isset($_REQUEST['arg']|| !in_array($_REQUEST['arg'],
  907.             $possible_arguments)) {
  908.             return $data;
  909.         }
  910.         $data['SELECT_ROLE'= -1;
  911.         switch($_REQUEST['arg'])
  912.         {
  913.             case "addgroup":
  914.                 $data['SELECT_GROUP'= -1;
  915.                 if($this->groupModel->getGroupId($groupname0{
  916.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  917.                         tl('admin_controller_groupname_exists').
  918.                         "</h1>')";
  919.                     return $data;
  920.                 }
  921.                 $this->groupModel->addGroup(
  922.                     $groupname$_SESSION['USER_ID']);
  923.                 $groupid $this->groupModel->getGroupId($groupname);
  924.                 $data['GROUP_NAMES'][$groupid$groupname;
  925.                 $data['DELETE_GROUP_NAMES'][$groupid$groupname;
  926.                 $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  927.                     tl('admin_controller_groupname_added').
  928.                     "</h1>')";
  929.             break;
  930.             case "deletegroup":
  931.                 if($select_group == -1){
  932.                 $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  933.                   tl('admin_controller_groupname_doesnt_exists')."</h1>')";
  934.                 }else{
  935.                  $this->groupModel->deleteGroup($select_group);
  936.                 unset($data['GROUP_NAMES'][$select_group]);
  937.                 unset($data['DELETE_GROUP_NAMES'][$select_group]);
  938.                 $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  939.                     tl('admin_controller_groupname_deleted')."</h1>')";
  940.                 }
  941.             break;
  942.             case "adduser":
  943.                 if(!in_array($select_user$user_ids)) {
  944.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  945.                         tl('admin_controller_username_doesnt_exists').
  946.                         "</h1>')";
  947.                     return $data;
  948.                 }
  949.                 $this->groupModel->addUserGroup(
  950.                     $select_group$select_user);
  951.                 unset($data['AVAILABLE_USERS'][$select_user]);
  952.                 $data['GROUP_USERS'=
  953.                     $this->groupModel->getGroupUsers($select_group);
  954.                 $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  955.                     tl('admin_controller_user_added')."</h1>')";
  956.             break;
  957.             case "deleteuser":
  958.                 $this->groupModel->deleteUserGroup(
  959.                     $select_group$select_user);
  960.                 $data['GROUP_USERS'=
  961.                     $this->groupModel->getGroupUsers($select_group);
  962.                 $data['AVAILABLE_USERS'][$select_user=
  963.                     $user_names[$select_user];
  964.                 $data['SELECT_USER'= -1;
  965.                 unset($data['GROUP_NAMES'][$select_group]);
  966.                 unset($data['GROUP_USERS']);
  967.                 $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  968.                     tl('admin_controller_user_deleted')."</h1>')";
  969.             break;
  970.             case "addrole":
  971.                 if(!in_array($select_group$group_ids)) {
  972.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  973.                         tl('admin_controller_groupname_doesnt_exists').
  974.                         "</h1>')";
  975.                     return $data;
  976.                 }
  977.                 if(!in_array($select_role$role_ids)) {
  978.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  979.                         tl('admin_controller_rolename_doesnt_exists').
  980.                         "</h1>')";
  981.                     return $data;
  982.                 }
  983.                 $this->groupModel->addGroupRole($select_group$select_role);
  984.                 unset($data['AVAILABLE_ROLES'][$select_role]);
  985.                 $data['GROUP_ROLES'=
  986.                     $this->groupModel->getGroupRoles($select_group);
  987.                 $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  988.                     tl('admin_controller_rolename_added')."</h1>')";
  989.             break;
  990.             case "deleterole":
  991.                 if(!in_array($select_group$group_ids)) {
  992.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  993.                         tl('admin_controller_groupname_doesnt_exists').
  994.                         "</h1>')";
  995.                     return $data;
  996.                 }
  997.                 if(!in_array($select_role$role_ids)) {
  998.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  999.                         tl('admin_controller_rolename_doesnt_exists').
  1000.                         "</h1>')";
  1001.                     return $data;
  1002.                 }
  1003.                 $this->groupModel->deleteGroupRole($select_group$select_role);
  1004.                 $data['GROUP_ROLES'=
  1005.                     $this->groupModel->getGroupRoles($select_group);
  1006.                 if(isset($rolenames[$select_role])) {
  1007.                     $data['AVAILABLE_ACTIVITIES'][$select_role=
  1008.                         $rolenames[$select_role];
  1009.                 }
  1010.                 $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1011.                     tl('admin_controller_role_deleted')."</h1>')";
  1012.             break;
  1013.             case "updategroup":
  1014.                 $this->groupModel->updateUserGroup($select_user$select_group);
  1015.             break;
  1016.         }
  1017.  
  1018.         return $data;
  1019.     }
  1020.  
  1021.     /**
  1022.      * Used to handle the manage crawl activity.
  1023.      *
  1024.      * This activity allows new crawls to be started, statistics about old
  1025.      * crawls to be seen. It allows a user to stop the current crawl or
  1026.      * restart an old crawl. It also allows a user to configure the options
  1027.      * by which a crawl is conducted
  1028.      *
  1029.      * @return array $data information and statistics about crawls in the system
  1030.      *       as well as status messages on performing a given sub activity
  1031.  
  1032.      */
  1033.     function manageCrawls()
  1034.     {
  1035.         $possible_arguments =
  1036.             array("start""resume""delete""stop""index""options");
  1037.  
  1038.         $data["ELEMENT""managecrawlsElement";
  1039.         $data['SCRIPT'"doUpdate();";
  1040.  
  1041.         if(isset($_REQUEST['arg']&&
  1042.             in_array($_REQUEST['arg']$possible_arguments)) {
  1043.  
  1044.             $machine_urls $this->machineModel->getQueueServerUrls();
  1045.             $num_machines count($machine_urls);
  1046.             if($num_machines <  || ($num_machines ==  &&
  1047.                 UrlParser::isLocalhostUrl($machine_urls[0]))) {
  1048.                 $machine_urls NULL;
  1049.             }
  1050.  
  1051.             switch($_REQUEST['arg'])
  1052.             {
  1053.                 case "start":
  1054.                     $this->startCrawl($data$machine_urls);
  1055.                 break;
  1056.  
  1057.                 case "stop":
  1058.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1059.                         tl('admin_controller_stop_crawl')."</h1>')";
  1060.                     @unlink(CRAWL_DIR."/schedules/crawl_params.txt");
  1061.  
  1062.                     $info array();
  1063.                     $info[self::STATUS"STOP_CRAWL";
  1064.                     $filename CRAWL_DIR.
  1065.                         "/schedules/name_server_messages.txt";
  1066.                     file_put_contents($filenameserialize($info));
  1067.  
  1068.                     $this->crawlModel->sendStopCrawlMessage($machine_urls);
  1069.                 break;
  1070.  
  1071.                 case "resume":
  1072.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1073.                         tl('admin_controller_resume_crawl')."</h1>')";
  1074.                     $crawl_params array();
  1075.                     $crawl_params[self::STATUS"RESUME_CRAWL";
  1076.                     $crawl_params[self::CRAWL_TIME=
  1077.                         $this->clean($_REQUEST['timestamp']"int");
  1078.                     $seed_info $this->crawlModel->getCrawlSeedInfo(
  1079.                         $crawl_params[self::CRAWL_TIME]$machine_urls);
  1080.                     $this->getCrawlParametersFromSeedInfo($crawl_params,
  1081.                         $seed_info);
  1082.                    /*
  1083.                        Write the new crawl parameters to the name server, so
  1084.                        that it can pass them along in the case of a new archive
  1085.                        crawl.
  1086.                     */
  1087.                     $filename CRAWL_DIR.
  1088.                         "/schedules/name_server_messages.txt";
  1089.                     file_put_contents($filenameserialize($crawl_params));
  1090.                     chmod($filename0777);
  1091.                     $this->crawlModel->sendStartCrawlMessage($crawl_params,
  1092.                         NULL$machine_urls);
  1093.                 break;
  1094.  
  1095.                 case "delete":
  1096.                     if(isset($_REQUEST['timestamp'])) {
  1097.                          $timestamp =
  1098.                             $this->clean($_REQUEST['timestamp']"int");
  1099.                          $this->crawlModel->deleteCrawl($timestamp,
  1100.                             $machine_urls);
  1101.  
  1102.                          $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1103.                             tl('admin_controller_delete_crawl_success').
  1104.                             "</h1>'); crawlStatusUpdate(); ";
  1105.                      else {
  1106.                         $data['SCRIPT'.= "crawlStatusUpdate(); ".
  1107.                             "doMessage('<h1 class=\"red\" >".
  1108.                             tl('admin_controller_delete_crawl_fail').
  1109.                             "</h1>')";
  1110.                      }
  1111.                 break;
  1112.  
  1113.                 case "index":
  1114.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1115.                         tl('admin_controller_set_index')."</h1>')";
  1116.  
  1117.                     $timestamp $this->clean($_REQUEST['timestamp']"int");
  1118.                     $this->crawlModel->setCurrentIndexDatabaseName($timestamp);
  1119.                 break;
  1120.  
  1121.                 case "options":
  1122.                     $this->editCrawlOption($data$machine_urls);
  1123.                 break;
  1124.             }
  1125.         }
  1126.         return $data;
  1127.     }
  1128.  
  1129.     /**
  1130.      * Called from @see manageCrawls to start a new crawl on the machines
  1131.      * $machine_urls. Updates $data array with crawl start message
  1132.      *
  1133.      * @param array &$data an array of info to supply to AdminView
  1134.      * @param array $machine_urls string urls of machines managed by this
  1135.      *   Yioop name server on which to perform the crawl
  1136.      */
  1137.     function startCrawl(&$data$machine_urls$seed_info NULL)
  1138.     {
  1139.         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1140.             tl('admin_controller_starting_new_crawl')."</h1>')";
  1141.  
  1142.         $crawl_params array();
  1143.         $crawl_params[self::STATUS"NEW_CRAWL";
  1144.         $crawl_params[self::CRAWL_TIMEtime();
  1145.         $seed_info $this->crawlModel->getSeedInfo();
  1146.         $this->getCrawlParametersFromSeedInfo($crawl_params$seed_info);
  1147.         if(isset($_REQUEST['description'])) {
  1148.             $description $this->clean($_REQUEST['description']"string");
  1149.         else {
  1150.             $description tl('admin_controller_no_description');
  1151.         }
  1152.         $crawl_params['DESCRIPTION'$description;
  1153.         $crawl_params[self::VIDEO_SOURCESarray();
  1154.         $sources =
  1155.             $this->sourceModel->getMediaSources('video');
  1156.         foreach($sources as $source{
  1157.             $url $source['SOURCE_URL'];
  1158.             $url_parts explode("{}"$url);
  1159.             $crawl_params[self::VIDEO_SOURCES][$url_parts[0];
  1160.         }
  1161.         if(isset($crawl_params[self::INDEXING_PLUGINS]&&
  1162.             is_array($crawl_params[self::INDEXING_PLUGINS])) {
  1163.             foreach($crawl_params[self::INDEXING_PLUGINSas $plugin_prefix{
  1164.                 $plugin_name $plugin_prefix."Plugin";
  1165.                 $plugin new $plugin_name();
  1166.                 if(method_exists($plugin_name"loadConfiguration")) {
  1167.                     $crawl_params[self::INDEXING_PLUGINS_DATA][$plugin_prefix=
  1168.                         $plugin->loadConfiguration();
  1169.                 }
  1170.             }
  1171.         }
  1172.  
  1173.         /*
  1174.            Write the new crawl parameters to the name server, so
  1175.            that it can pass them along in the case of a new archive
  1176.            crawl.
  1177.         */
  1178.         $filename CRAWL_DIR.
  1179.             "/schedules/name_server_messages.txt";
  1180.         file_put_contents($filenameserialize($crawl_params));
  1181.         chmod($filename0777);
  1182.  
  1183.         $this->crawlModel->sendStartCrawlMessage($crawl_params,
  1184.             $seed_info$machine_urls);
  1185.     }
  1186.  
  1187.     /**
  1188.      * Reads the parameters for a crawl from an array gotten from a crawl.ini
  1189.      * file
  1190.      *
  1191.      * @param array &$crawl_params parameters to write to queue_server
  1192.      * @param array $seed_info data from crawl.ini file
  1193.      */
  1194.     function getCrawlParametersFromSeedInfo(&$crawl_params$seed_info)
  1195.     {
  1196.         $crawl_params[self::CRAWL_TYPE$seed_info['general']['crawl_type'];
  1197.         $crawl_params[self::CRAWL_INDEX=
  1198.             (isset($seed_info['general']['crawl_index'])) ?
  1199.             $seed_info['general']['crawl_index''';
  1200.         $crawl_params[self::ARC_DIR]=(isset($seed_info['general']['arc_dir'])) ?
  1201.             $seed_info['general']['arc_dir''';
  1202.         $crawl_params[self::ARC_TYPE=
  1203.             (isset($seed_info['general']['arc_type'])) ?
  1204.             $seed_info['general']['arc_type''';
  1205.         $crawl_params[self::CACHE_PAGES=
  1206.             (isset($seed_info['general']['cache_pages'])) ?
  1207.             intval($seed_info['general']['cache_pages']:
  1208.             true;
  1209.         $crawl_params[self::PAGE_RANGE_REQUEST=
  1210.             (isset($seed_info['general']['page_range_request'])) ?
  1211.             intval($seed_info['general']['page_range_request']:
  1212.             PAGE_RANGE_REQUEST;
  1213.         $crawl_params[self::MAX_DESCRIPTION_LEN=
  1214.             (isset($seed_info['general']['max_description_len'])) ?
  1215.             intval($seed_info['general']['max_description_len']:
  1216.             MAX_DESCRIPTION_LEN;
  1217.         $crawl_params[self::PAGE_RECRAWL_FREQUENCY=
  1218.             (isset($seed_info['general']['page_recrawl_frequency'])) ?
  1219.             intval($seed_info['general']['page_recrawl_frequency']:
  1220.             PAGE_RECRAWL_FREQUENCY;
  1221.         $crawl_params[self::TO_CRAWL$seed_info['seed_sites']['url'];
  1222.         $crawl_params[self::CRAWL_ORDER$seed_info['general']['crawl_order'];
  1223.         $crawl_params[self::RESTRICT_SITES_BY_URL=
  1224.             $seed_info['general']['restrict_sites_by_url'];
  1225.         $crawl_params[self::ALLOWED_SITES=
  1226.             isset($seed_info['allowed_sites']['url']?
  1227.             $seed_info['allowed_sites']['url'array();
  1228.         $crawl_params[self::DISALLOWED_SITES=
  1229.             isset($seed_info['disallowed_sites']['url']?
  1230.             $seed_info['disallowed_sites']['url'array();
  1231.         if(isset($seed_info['indexed_file_types']['extensions'])) {
  1232.             $crawl_params[self::INDEXED_FILE_TYPES=
  1233.                 $seed_info['indexed_file_types']['extensions'];
  1234.         }
  1235.         if(isset($seed_info['active_classifiers']['label'])) {
  1236.             // Note that 'label' is actually an array of active class labels.
  1237.             $crawl_params[self::ACTIVE_CLASSIFIERS=
  1238.                 $seed_info['active_classifiers']['label'];
  1239.         }
  1240.         if(isset($seed_info['active_rankers']['label'])) {
  1241.             // Note that 'label' is actually an array of active class labels.
  1242.             $crawl_params[self::ACTIVE_RANKERS=
  1243.                 $seed_info['active_rankers']['label'];
  1244.         }
  1245.         if(isset($seed_info['indexing_plugins']['plugins'])) {
  1246.             $crawl_params[self::INDEXING_PLUGINS=
  1247.                 $seed_info['indexing_plugins']['plugins'];
  1248.         }
  1249.         $crawl_params[self::PAGE_RULES=
  1250.             isset($seed_info['page_rules']['rule']?
  1251.             $seed_info['page_rules']['rule'array();
  1252.     }
  1253.  
  1254.     /**
  1255.      * Called from @see manageCrawls to edit the parameters for the next
  1256.      * crawl (or current crawl) to be carried out by the machines
  1257.      * $machine_urls. Updates $data array to be supplied to AdminView
  1258.      *
  1259.      * @param array &$data an array of info to supply to AdminView
  1260.      * @param array $machine_urls string urls of machines managed by this
  1261.      *   Yioop name server on which to perform the crawl
  1262.      */
  1263.     function editCrawlOption(&$data$machine_urls)
  1264.     {
  1265.         $data["leftorright"(getLocaleDirection(== 'ltr'?
  1266.             "right""left";
  1267.         $data["ELEMENT""crawloptionsElement";
  1268.         $crawls $this->crawlModel->getCrawlList(falsefalse$machine_urls);
  1269.         $indexes $this->crawlModel->getCrawlList(truetrue$machine_urls);
  1270.         $mixes $this->crawlModel->getMixList(false);
  1271.         foreach($mixes as $mix{
  1272.             $tmp array();
  1273.             $tmp["DESCRIPTION""MIX::".$mix["MIX_NAME"];
  1274.             $tmp["CRAWL_TIME"$mix["MIX_TIMESTAMP"];
  1275.             $tmp["ARC_DIR""MIX";
  1276.             $tmp["ARC_TYPE""MixArchiveBundle";
  1277.             $indexes[$tmp;
  1278.         }
  1279.  
  1280.         $indexes_by_crawl_time array();
  1281.         $update_flag false;
  1282.         $data['available_options'array(
  1283.             tl('admin_controller_use_below'),
  1284.             tl('admin_controller_use_defaults'));
  1285.         $data['available_crawl_indexes'array();
  1286.         $data['options_default'tl('admin_controller_use_below');
  1287.         foreach($crawls as $crawl{
  1288.             if(strlen($crawl['DESCRIPTION']{
  1289.                 $data['available_options'][$crawl['CRAWL_TIME']] =
  1290.                     tl('admin_controller_previous_crawl')." ".
  1291.                     $crawl['DESCRIPTION'];
  1292.             }
  1293.         }
  1294.         foreach($indexes as $i => $crawl{
  1295.             $data['available_crawl_indexes'][$crawl['CRAWL_TIME']]
  1296.                 = $crawl['DESCRIPTION'];
  1297.             $indexes_by_crawl_time[$crawl['CRAWL_TIME']] =$indexes[$i];
  1298.         }
  1299.         $no_further_changes false;
  1300.         $seed_current $this->crawlModel->getSeedInfo();
  1301.         if(isset($_REQUEST['load_option']&&
  1302.             $_REQUEST['load_option'== 1{
  1303.             $seed_info $this->crawlModel->getSeedInfo(true);
  1304.             if(isset(
  1305.                 $seed_current['general']['page_range_request'])) {
  1306.                 $seed_info['general']['page_range_request'=
  1307.                     $seed_current['general']['page_range_request'];
  1308.             }
  1309.             if(isset(
  1310.                 $seed_current['general']['page_recrawl_frequency'])
  1311.                 ){
  1312.                 $seed_info['general']['page_recrawl_frequency'=
  1313.                 $seed_current['general']['page_recrawl_frequency'];
  1314.             }
  1315.             if(isset(
  1316.                 $seed_current['general']['max_description_len'])) {
  1317.                 $seed_info['general']['max_description_len'=
  1318.                     $seed_current['general']['max_description_len'];
  1319.             }
  1320.             $update_flag true;
  1321.             $no_further_changes true;
  1322.         else if (isset($_REQUEST['load_option']&&
  1323.             $_REQUEST['load_option'{
  1324.             $timestamp =
  1325.                 $this->clean($_REQUEST['load_option']"int");
  1326.             $seed_info $this->crawlModel->getCrawlSeedInfo(
  1327.                 $timestamp$machine_urls);
  1328.             $update_flag true;
  1329.             $no_further_changes true;
  1330.         else if(isset($_REQUEST['ts'])) {
  1331.             $timestamp =
  1332.                 $this->clean($_REQUEST['ts']"int");
  1333.             $seed_info $this->crawlModel->getCrawlSeedInfo(
  1334.                 $timestamp$machine_urls);
  1335.             $data['ts'$timestamp;
  1336.         else {
  1337.             $seed_info $this->crawlModel->getSeedInfo();
  1338.         }
  1339.         $page_options_properties array('indexed_file_types',
  1340.             'active_classifiers''page_rules''indexing_plugins');
  1341.         //these properties should be changed under page_options not here
  1342.         foreach($page_options_properties as $property{
  1343.             if(isset($seed_current[$property])) {
  1344.                 $seed_info[$property$seed_current[$property];
  1345.             }
  1346.         }
  1347.  
  1348.         if(!$no_further_changes && isset($_REQUEST['crawl_indexes'])
  1349.             && in_array($_REQUEST['crawl_indexes'],
  1350.             array_keys($data['available_crawl_indexes']))) {
  1351.             $seed_info['general']['crawl_index'$_REQUEST['crawl_indexes'];
  1352.             $index_data $indexes_by_crawl_time[$_REQUEST['crawl_indexes']];
  1353.             if(isset($index_data['ARC_DIR'])) {
  1354.                 $seed_info['general']['arc_dir'$index_data['ARC_DIR'];
  1355.                 $seed_info['general']['arc_type'$index_data['ARC_TYPE'];
  1356.             else {
  1357.                 $seed_info['general']['arc_dir''';
  1358.                 $seed_info['general']['arc_type''';
  1359.             }
  1360.             $update_flag true;
  1361.         }
  1362.         $data['crawl_index'=  (isset($seed_info['general']['crawl_index'])) ?
  1363.             $seed_info['general']['crawl_index''';
  1364.         $data['available_crawl_types'array(self::WEB_CRAWL,
  1365.             self::ARCHIVE_CRAWL);
  1366.         if(!$no_further_changes && isset($_REQUEST['crawl_type']&&
  1367.             in_array($_REQUEST['crawl_type']$data['available_crawl_types'])) {
  1368.             $seed_info['general']['crawl_type'=
  1369.                 $_REQUEST['crawl_type'];
  1370.             $update_flag true;
  1371.         }
  1372.         $data['crawl_type'$seed_info['general']['crawl_type'];
  1373.         if($data['crawl_type'== self::WEB_CRAWL{
  1374.             $data['web_crawl_active'"active";
  1375.             $data['archive_crawl_active'"";
  1376.         else {
  1377.             $data['archive_crawl_active'"active";
  1378.             $data['web_crawl_active'"";
  1379.         }
  1380.  
  1381.         $data['available_crawl_orders'array(
  1382.             self::BREADTH_FIRST =>
  1383.                 tl('admin_controller_breadth_first'),
  1384.             self::PAGE_IMPORTANCE =>
  1385.                 tl('admin_controller_page_importance'));
  1386.  
  1387.         if(!$no_further_changes && isset($_REQUEST['crawl_order'])
  1388.             &&  in_array($_REQUEST['crawl_order'],
  1389.                 array_keys($data['available_crawl_orders']))) {
  1390.             $seed_info['general']['crawl_order'=
  1391.                 $_REQUEST['crawl_order'];
  1392.             $update_flag true;
  1393.         }
  1394.         $data['crawl_order'$seed_info['general']['crawl_order'];
  1395.  
  1396.         if(!$no_further_changes && isset($_REQUEST['posted'])) {
  1397.             $seed_info['general']['restrict_sites_by_url'=
  1398.                 (isset($_REQUEST['restrict_sites_by_url'])) ?
  1399.                 true false;
  1400.             $update_flag true;
  1401.         }
  1402.         $data['restrict_sites_by_url'=
  1403.             $seed_info['general']['restrict_sites_by_url'];
  1404.         $site_types =
  1405.             array('allowed_sites' => 'url''disallowed_sites' => 'url',
  1406.                 'seed_sites' => 'url');
  1407.         foreach($site_types as $type => $field{
  1408.             if(!$no_further_changes && isset($_REQUEST[$type])) {
  1409.                 $seed_info[$type][$field=
  1410.                     $this->convertStringCleanArray(
  1411.                     $_REQUEST[$type]$field);
  1412.                     $update_flag true;
  1413.             }
  1414.             if(isset($seed_info[$type][$field])) {
  1415.                 $data[$type$this->convertArrayLines(
  1416.                     $seed_info[$type][$field]);
  1417.             else {
  1418.                 $data[$type"";
  1419.             }
  1420.         }
  1421.         $data['TOGGLE_STATE'=
  1422.             ($data['restrict_sites_by_url']?
  1423.             "checked='checked'" "";
  1424.  
  1425.         $data['SCRIPT'"setDisplay('toggle', ".
  1426.             "'{$data['restrict_sites_by_url']}');";
  1427.         if(!isset($_REQUEST['ts'])) {
  1428.             $data['SCRIPT'.=
  1429.             " elt('load-options').onchange = ".
  1430.             "function() { if(elt('load-options').selectedIndex !=".
  1431.             " 0) { elt('crawloptionsForm').submit();  }};";
  1432.         }
  1433.         if($data['crawl_type'== CrawlConstants::WEB_CRAWL{
  1434.             $data['SCRIPT'.=
  1435.                 "switchTab('webcrawltab', 'archivetab');";
  1436.         else {
  1437.             $data['SCRIPT'.=
  1438.                 "switchTab('archivetab', 'webcrawltab');";
  1439.         }
  1440.         $add_message "";
  1441.         if(isset($_REQUEST['ts']&&
  1442.             isset($_REQUEST['inject_sites'])) {
  1443.                 $timestamp $this->clean($_REQUEST['ts'],
  1444.                     "string");
  1445.                 $inject_urls =
  1446.                     $this->convertStringCleanArray(
  1447.                     $_REQUEST['inject_sites']);
  1448.                 if($this->crawlModel->injectUrlsCurrentCrawl(
  1449.                     $timestamp$inject_urls$machine_urls)) {
  1450.                     $add_message "<br />".
  1451.                         tl('admin_controller_urls_injected');
  1452.                 }
  1453.         }
  1454.         if($update_flag{
  1455.             if(isset($_REQUEST['ts'])) {
  1456.                 $this->crawlModel->setCrawlSeedInfo($timestamp,
  1457.                     $seed_info$machine_urls);
  1458.             else {
  1459.                 $this->crawlModel->setSeedInfo($seed_info);
  1460.             }
  1461.             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1462.                 tl('admin_controller_update_seed_info').
  1463.                 "$add_message</h1>');";
  1464.         }
  1465.         return $data;
  1466.     }
  1467.  
  1468.     /**
  1469.      * Converts an array of lines of strings into a single string with
  1470.      * proper newlines, each line having been trimmed and potentially
  1471.      * cleaned
  1472.      *
  1473.      * @param array $arr the array of lines to be process
  1474.      * @param string $endline_string what string should be used to indicate
  1475.      *       the end of a line
  1476.      * @param bool $clean whether to clean each line
  1477.      * @return string a concatenated string of cleaned lines
  1478.      */
  1479.     function convertArrayLines($arr$endline_string="\n"$clean false)
  1480.     {
  1481.         $output "";
  1482.         $eol "";
  1483.         foreach($arr as $line{
  1484.             $output .= $eol;
  1485.             $out_line trim($line);
  1486.             if($clean{
  1487.                 $out_line $this->clean($out_line"string");
  1488.             }
  1489.             $output .= trim($out_line);
  1490.             $eol $endline_string;
  1491.         }
  1492.         return $output;
  1493.     }
  1494.     /**
  1495.      * Cleans a string consisting of lines, typically of urls into an array of
  1496.      * clean lines. This is used in handling data from the crawl options
  1497.      * text areas.
  1498.      *
  1499.      * @param string $str contains the url data
  1500.      * @param string $line_type does additional cleaning depending on the type
  1501.      *       of the lines. For instance, if is "url" then a line not beginning
  1502.      *       with a url scheme will have http:// prepended.
  1503.      * @return $lines an array of clean lines
  1504.      */
  1505.     function convertStringCleanArray($str$line_type="url")
  1506.     {
  1507.         if($line_type == "url"{
  1508.             $pre_lines preg_split("/(\s)+/"$str);
  1509.         else {
  1510.             $pre_lines preg_split('/\n+/'$str);
  1511.         }
  1512.         $lines array();
  1513.         foreach($pre_lines as $line{
  1514.             $pre_line trim($this->clean($line"string"));
  1515.             if(strlen($pre_line0{
  1516.                 if($line_type == "url"{
  1517.                     $start_line substr($pre_line06);
  1518.                     if(!in_array($start_line,
  1519.                         array("file:/""http:/""domain""https:"))) {
  1520.                         $pre_line "http://"$pre_line;
  1521.                     }
  1522.                 }
  1523.                 $lines[$pre_line;
  1524.             }
  1525.         }
  1526.         return $lines;
  1527.     }
  1528.  
  1529.     /**
  1530.      * Handles admin request related to the crawl mix activity
  1531.      *
  1532.      * The crawl mix activity allows a user to create/edit crawl mixes:
  1533.      * weighted combinations of search indexes
  1534.      *
  1535.      * @return array $data info about available crawl mixes and changes to them
  1536.      *       as well as any messages about the success or failure of a
  1537.      *       sub activity.
  1538.      */
  1539.     function mixCrawls()
  1540.     {
  1541.         $possible_arguments array(
  1542.             "createmix""deletemix""editmix""index");
  1543.  
  1544.         $data["ELEMENT""mixcrawlsElement";
  1545.  
  1546.         $data['mix_default'0;
  1547.         $machine_urls $this->machineModel->getQueueServerUrls();
  1548.         $num_machines count($machine_urls);
  1549.         if($num_machines <  || ($num_machines ==  &&
  1550.             UrlParser::isLocalhostUrl($machine_urls[0]))) {
  1551.             $machine_urls NULL;
  1552.         }
  1553.         $crawls $this->crawlModel->getCrawlList(falsetrue$machine_urls);
  1554.         $data['available_crawls'][0tl('admin_controller_select_crawl');
  1555.         $data['available_crawls'][1tl('admin_controller_default_crawl');
  1556.         $data['SCRIPT'"c = [];c[0]='".
  1557.             tl('admin_controller_select_crawl')."';";
  1558.         $data['SCRIPT'.= "c[1]='".
  1559.             tl('admin_controller_default_crawl')."';";
  1560.         foreach($crawls as $crawl{
  1561.             $data['available_crawls'][$crawl['CRAWL_TIME']] =
  1562.                 $crawl['DESCRIPTION'];
  1563.             $data['SCRIPT'.= 'c['.$crawl['CRAWL_TIME'].']="'.
  1564.                 $crawl['DESCRIPTION'].'";';
  1565.         }
  1566.         $mixes $this->crawlModel->getMixList(true);
  1567.         if(count($mixes{
  1568.             $data['available_mixes']$mixes;
  1569.             $mix_ids array();
  1570.             foreach($mixes as $mix{
  1571.                 $mix_ids[$mix['MIX_TIMESTAMP'];
  1572.             }
  1573.         }
  1574.  
  1575.         $mix array();
  1576.         if(isset($_REQUEST['arg']&&
  1577.             in_array($_REQUEST['arg']$possible_arguments)) {
  1578.             switch($_REQUEST['arg'])
  1579.             {
  1580.                 case "createmix":
  1581.                     $mix['MIX_TIMESTAMP'time();
  1582.                     if(isset($_REQUEST['MIX_NAME'])) {
  1583.                         $mix['MIX_NAME'$this->clean($_REQUEST['MIX_NAME'],
  1584.                             'string');
  1585.                     else {
  1586.                         $mix['MIX_NAME'tl('admin_controller_unnamed');
  1587.                     }
  1588.                     $mix['GROUPS'array();
  1589.                     $this->crawlModel->setCrawlMix($mix);
  1590.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1591.                         tl('admin_controller_mix_created')."</h1>');";
  1592.  
  1593.                 case "editmix":
  1594.                     //$data passed by reference
  1595.                     $this->editMix($data$mix_ids$mix);
  1596.                 break;
  1597.  
  1598.                 case "index":
  1599.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1600.                         tl('admin_controller_set_index')."</h1>')";
  1601.  
  1602.                     $timestamp $this->clean($_REQUEST['timestamp']"int");
  1603.                     $this->crawlModel->setCurrentIndexDatabaseName($timestamp);
  1604.                 break;
  1605.  
  1606.                 case "deletemix":
  1607.                     if(!isset($_REQUEST['timestamp'])|| !isset($mix_ids||
  1608.                         !in_array($_REQUEST['timestamp']$mix_ids)) {
  1609.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1610.                             tl('admin_controller_mix_doesnt_exists').
  1611.                             "</h1>')";
  1612.                         return $data;
  1613.                     }
  1614.                     $this->crawlModel->deleteCrawlMix($_REQUEST['timestamp']);
  1615.                     $data['available_mixes'=
  1616.                         $this->crawlModel->getMixList(true);
  1617.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1618.                         tl('admin_controller_mix_deleted')."</h1>')";
  1619.                 break;
  1620.             }
  1621.         }
  1622.  
  1623.         $crawl_time $this->crawlModel->getCurrentIndexDatabaseName();
  1624.         if(isset($crawl_time) ) {
  1625.             $data['CURRENT_INDEX'= (int)$crawl_time;
  1626.         else {
  1627.             $data['CURRENT_INDEX'= -1;
  1628.         }
  1629.  
  1630.         return $data;
  1631.     }
  1632.  
  1633.     /**
  1634.      * Handles admin request related to the editing a crawl mix activity
  1635.      *
  1636.      * @return array $data info about the groups and their contents for a
  1637.      *       particular crawl mix
  1638.      */
  1639.     function editMix(&$data&$mix_ids$mix)
  1640.     {
  1641.         $data["leftorright"=
  1642.             (getLocaleDirection(== 'ltr'"right""left";
  1643.         $data["ELEMENT""editmixElement";
  1644.  
  1645.         if(isset($_REQUEST['timestamp'])) {
  1646.             $mix $this->crawlModel->getCrawlMix(
  1647.                 $_REQUEST['timestamp']);
  1648.         }
  1649.         $data['MIX'$mix;
  1650.         $data['INCLUDE_SCRIPTS'array("mix");
  1651.  
  1652.         //set up an array of translation for javascript-land
  1653.         $data['SCRIPT'.= "tl = {".
  1654.             'editmix_element_add_crawls:"'tl('editmix_element_add_crawls'.
  1655.             '",' 'editmix_element_num_results:"'.
  1656.                 tl('editmix_element_num_results').'",'.
  1657.             'editmix_element_del_grp:"'.tl('editmix_element_del_grp').'",'.
  1658.             'editmix_element_weight:"'.tl('editmix_element_weight').'",'.
  1659.             'editmix_element_name:"'.tl('editmix_element_name').'",'.
  1660.             'editmix_add_keywords:"'.tl('editmix_add_keywords').'",'.
  1661.             'editmix_element_actions:"'.tl('editmix_element_actions').'",'.
  1662.             'editmix_add_query:"'.tl('editmix_add_query').'",'.
  1663.             'editmix_element_delete:"'.tl('editmix_element_delete').'"'.
  1664.             '};';
  1665.         //clean and save the crawl mix sent from the browser
  1666.         if(isset($_REQUEST['update']&& $_REQUEST['update'==
  1667.             "update"{
  1668.             $mix $_REQUEST['mix'];
  1669.             $mix['MIX_TIMESTAMP'=
  1670.                 $this->clean($mix['MIX_TIMESTAMP']"int");
  1671.             $mix['MIX_NAME'=$this->clean($mix['MIX_NAME'],
  1672.                 "string");
  1673.             $comp array();
  1674.             if(isset($mix['GROUPS'])) {
  1675.  
  1676.                 if($mix['GROUPS'!= NULL{
  1677.                     foreach($mix['GROUPS'as $group_id => $group_data{
  1678.                         if(isset($group_data['RESULT_BOUND'])) {
  1679.                             $mix['GROUPS'][$group_id]['RESULT_BOUND'=
  1680.                                 $this->clean($group_data['RESULT_BOUND'],
  1681.                                     "int");
  1682.                         else {
  1683.                             $mix['GROUPS']['RESULT_BOUND'0;
  1684.                         }
  1685.                         if(isset($group_data['COMPONENTS'])) {
  1686.                             $comp array();
  1687.                             foreach($group_data['COMPONENTS'as $component{
  1688.                                 $row array();
  1689.                                 $row['CRAWL_TIMESTAMP'=
  1690.                                     $this->clean($component['CRAWL_TIMESTAMP'],
  1691.                                     "int");
  1692.                                 $row['WEIGHT'$this->clean(
  1693.                                     $component['WEIGHT']"float");
  1694.                                 $row['KEYWORDS'$this->clean(
  1695.                                     $component['KEYWORDS'],
  1696.                                     "string");
  1697.                                 $comp[=$row;
  1698.                             }
  1699.                             $mix['GROUPS'][$group_id]['COMPONENTS'$comp;
  1700.                         else {
  1701.                             $mix['GROUPS'][$group_id]['COMPONENTS'array();
  1702.                         }
  1703.                     }
  1704.                 else {
  1705.                     $mix['COMPONENTS'array();
  1706.                 }
  1707.  
  1708.             else {
  1709.                 $mix['GROUPS'$data['MIX']['GROUPS'];
  1710.             }
  1711.  
  1712.             $data['MIX'$mix;
  1713.             $this->crawlModel->setCrawlMix($mix);
  1714.             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  1715.                 tl('admin_controller_mix_saved')."</h1>');";
  1716.         }
  1717.  
  1718.         $data['SCRIPT'.= 'groups = [';
  1719.         $not_first "";
  1720.         foreach($mix['GROUPS'as $group_id => $group_data{
  1721.             $data['SCRIPT'.= $not_first.'{';
  1722.             $not_first",";
  1723.             if(isset($group_data['RESULT_BOUND'])) {
  1724.                 $data['SCRIPT'.= "num_results:".$group_data['RESULT_BOUND'];
  1725.             else {
  1726.                 $data['SCRIPT'.= "num_results:1 ";
  1727.             }
  1728.             $data['SCRIPT'.= ", components:[";
  1729.             if(isset($group_data['COMPONENTS'])) {
  1730.                 $comma "";
  1731.                 foreach($group_data['COMPONENTS'as $component{
  1732.                     $crawl_ts $component['CRAWL_TIMESTAMP'];
  1733.                     $crawl_name $data['available_crawls'][$crawl_ts];
  1734.                     $data['SCRIPT'.= $comma." [$crawl_ts, '$crawl_name', ".
  1735.                         $component['WEIGHT'].", ";
  1736.                     $comma ",";
  1737.                     $keywords (isset($component['KEYWORDS'])) ?
  1738.                         $component['KEYWORDS'"";
  1739.                     $data['SCRIPT'.= "'$keywords'] ";
  1740.                 }
  1741.             }
  1742.             $data['SCRIPT'.= "] }";
  1743.         }
  1744.         $data['SCRIPT'.= ']; drawGroups();';
  1745.     }
  1746.  
  1747.     /**
  1748.      * Handles admin request related to controlling file options to be used
  1749.      * in a crawl
  1750.      *
  1751.      * This activity allows a user to specify the page range size to be
  1752.      * be used during a crawl as well as which file types can be downloaded
  1753.      */
  1754.     function pageOptions()
  1755.     {
  1756.         global $INDEXED_FILE_TYPES;
  1757.         $data["ELEMENT""pageoptionsElement";
  1758.         $data['SCRIPT'"";
  1759.         $machine_urls $this->machineModel->getQueueServerUrls();
  1760.         $num_machines count($machine_urls);
  1761.         if($num_machines <  || ($num_machines ==  &&
  1762.             UrlParser::isLocalhostUrl($machine_urls[0]))) {
  1763.             $machine_urls NULL;
  1764.         }
  1765.         $data['available_options'array(
  1766.             tl('admin_controller_use_below'),
  1767.             tl('admin_controller_use_defaults'));
  1768.         $crawls $this->crawlModel->getCrawlList(falsetrue$machine_urls);
  1769.         $data['options_default'tl('admin_controller_use_below');
  1770.         foreach($crawls as $crawl{
  1771.             if(strlen($crawl['DESCRIPTION']{
  1772.                 $data['available_options'][$crawl['CRAWL_TIME']] =
  1773.                     tl('admin_controller_previous_crawl')." ".
  1774.                     $crawl['DESCRIPTION'];
  1775.             }
  1776.         }
  1777.         $seed_info $this->crawlModel->getSeedInfo();
  1778.         $data['RECRAWL_FREQS'array(-1=>tl('admin_controller_recrawl_never'),
  1779.             1=>tl('admin_controller_recrawl_1day'),
  1780.             2=>tl('admin_controller_recrawl_2day'),
  1781.             3=>tl('admin_controller_recrawl_3day'),
  1782.             7=>tl('admin_controller_recrawl_7day'),
  1783.             14=>tl('admin_controller_recrawl_14day'));
  1784.         $data['SIZE_VALUES'array(10000=>1000050000=>50000,
  1785.             100000=>100000500000=>5000001000000=>1000000,
  1786.             5000000=>500000010000000=>10000000);
  1787.         $data['LEN_VALUES'array(2000=>200010000=>1000050000=>50000,
  1788.             100000=>100000500000=>5000001000000=>1000000,
  1789.             5000000=>500000010000000=>10000000);
  1790.         if(!isset($seed_info["indexed_file_types"]["extensions"])) {
  1791.             $seed_info["indexed_file_types"]["extensions"=
  1792.                 $INDEXED_FILE_TYPES;
  1793.         }
  1794.         $loaded false;
  1795.         if(isset($_REQUEST['load_option']&&
  1796.             $_REQUEST['load_option'0{
  1797.             if($_REQUEST['load_option'== 1{
  1798.                 $seed_loaded $this->crawlModel->getSeedInfo(true);
  1799.             else {
  1800.                 $timestamp $this->clean($_REQUEST['load_option']"int");
  1801.                 $seed_loaded $this->crawlModel->getCrawlSeedInfo(
  1802.                     $timestamp$machine_urls);
  1803.             }
  1804.             $copy_options array("general" => array("page_recrawl_frequency",
  1805.                 "page_range_request""max_description_len""cache_pages"),
  1806.                 "indexed_file_types" => array("extensions"),
  1807.                 "indexing_plugins" => array("plugins"));
  1808.             foreach($copy_options as $main_option => $sub_options{
  1809.                 foreach($sub_options as $sub_option{
  1810.                     if(isset($seed_loaded[$main_option][$sub_option])) {
  1811.                         $seed_info[$main_option][$sub_option=
  1812.                             $seed_loaded[$main_option][$sub_option];
  1813.                     }
  1814.                 }
  1815.             }
  1816.             if(isset($seed_loaded['page_rules'])) {
  1817.                 $seed_info['page_rules'=
  1818.                     $seed_loaded['page_rules'];
  1819.             }
  1820.             if(isset($seed_loaded['active_classifiers'])) {
  1821.                 $seed_info['active_classifiers'=
  1822.                     $seed_loaded['active_classifiers'];
  1823.             else {
  1824.                 $seed_info['active_classifiers'array();
  1825.                 $seed_info['active_classifiers']['label'array();
  1826.             }
  1827.             $update_flag true;
  1828.             $loaded true;
  1829.         else {
  1830.             $seed_info $this->crawlModel->getSeedInfo();
  1831.             if(isset($_REQUEST["page_recrawl_frequency"]&&
  1832.                 in_array($_REQUEST["page_recrawl_frequency"],
  1833.                     array_keys($data['RECRAWL_FREQS']))) {
  1834.                 $seed_info["general"]["page_recrawl_frequency"=
  1835.                     $_REQUEST["page_recrawl_frequency"];
  1836.             }
  1837.             if(isset($_REQUEST["page_range_request"]&&
  1838.                 in_array($_REQUEST["page_range_request"],$data['SIZE_VALUES'])){
  1839.                 $seed_info["general"]["page_range_request"=
  1840.                     $_REQUEST["page_range_request"];
  1841.             }
  1842.             if(isset($_REQUEST["max_description_len"]&&
  1843.                 in_array($_REQUEST["max_description_len"],$data['LEN_VALUES'])){
  1844.                 $seed_info["general"]["max_description_len"=
  1845.                     $_REQUEST["max_description_len"];
  1846.             }
  1847.            if(isset($_REQUEST["cache_pages"]) ) {
  1848.                 $seed_info["general"]["cache_pages"true;
  1849.            else if(isset($_REQUEST['posted'])) {
  1850.                 //form sent but check box unchecked
  1851.                 $seed_info["general"]["cache_pages"false;
  1852.            }
  1853.  
  1854.            if(isset($_REQUEST['page_rules'])) {
  1855.                 $seed_info['page_rules']['rule'=
  1856.                     $this->convertStringCleanArray(
  1857.                     $_REQUEST['page_rules']'rule');
  1858.                     $update_flag true;
  1859.             }
  1860.         }
  1861.         if(!isset($seed_info["general"]["page_recrawl_frequency"])) {
  1862.             $seed_info["general"]["page_recrawl_frequency"=
  1863.                 PAGE_RECRAWL_FREQUENCY;
  1864.         }
  1865.         $data['PAGE_RECRAWL_FREQUENCY'=
  1866.             $seed_info["general"]["page_recrawl_frequency"];
  1867.         if(!isset($seed_info["general"]["cache_pages"])) {
  1868.             $seed_info["general"]["cache_pages"false;
  1869.         }
  1870.         $data["CACHE_PAGES"$seed_info["general"]["cache_pages"];
  1871.         if(!isset($seed_info["general"]["page_range_request"])) {
  1872.             $seed_info["general"]["page_range_request"PAGE_RANGE_REQUEST;
  1873.         }
  1874.         $data['PAGE_SIZE'$seed_info["general"]["page_range_request"];
  1875.         if(!isset($seed_info["general"]["max_description_len"])) {
  1876.             $seed_info["general"]["max_description_len"MAX_DESCRIPTION_LEN;
  1877.         }
  1878.         $data['MAX_LEN'$seed_info["general"]["max_description_len"];
  1879.  
  1880.         $data['INDEXING_PLUGINS'array();
  1881.         $included_plugins array();
  1882.         if(isset($_REQUEST["posted"])) {
  1883.             $seed_info['indexing_plugins']['plugins'=
  1884.                 (isset($_REQUEST["INDEXING_PLUGINS"])) ?
  1885.                 $_REQUEST["INDEXING_PLUGINS"array();
  1886.         }
  1887.         $included_plugins =
  1888.             (isset($seed_info['indexing_plugins']['plugins'])) ?
  1889.                 $seed_info['indexing_plugins']['plugins']
  1890.                 : array();
  1891.         foreach($this->indexing_plugins as $plugin{
  1892.             $plugin_name ucfirst($plugin);
  1893.             $data['INDEXING_PLUGINS'][$plugin_name]['checked'=
  1894.                 (in_array($plugin_name$included_plugins)) ?
  1895.                 "checked='checked'" "";
  1896.             $class_name $plugin_name."Plugin";
  1897.             if(method_exists($class_name'configureHandler'&&
  1898.                 method_exists($class_name'configureView')) {
  1899.                 $data['INDEXING_PLUGINS'][$plugin_name]['configure'true;
  1900.                 $plugin_object new $class_name();
  1901.                 $plugin_object->configureHandler($data);
  1902.             else {
  1903.                 $data['INDEXING_PLUGINS'][$plugin_name]['configure'false;
  1904.             }
  1905.         }
  1906.  
  1907.         $profile =  $this->profileModel->getProfile(WORK_DIRECTORY);
  1908.         if(!isset($_REQUEST['load_option'])) {
  1909.             $data array_merge($data$profile);
  1910.         else {
  1911.             $this->updateProfileFields($data$profile,
  1912.                 array('IP_LINK','CACHE_LINK''SIMILAR_LINK''IN_LINK',
  1913.                     'SIGNIN_LINK''SUBSEARCH_LINK','WORD_SUGGEST'));
  1914.         }
  1915.         $weights array('TITLE_WEIGHT' => 4,
  1916.             'DESCRIPTION_WEIGHT' => 1'LINK_WEIGHT' => 2,
  1917.             'MIN_RESULTS_TO_GROUP' => 200'SERVER_ALPHA' => 1.6);
  1918.         $change false;
  1919.         foreach($weights as $weight => $value{
  1920.             if(isset($_REQUEST[$weight])) {
  1921.                 $data[$weight$this->clean($_REQUEST[$weight]'float'1
  1922.                     );
  1923.                 $profile[$weight$data[$weight];
  1924.                 $change true;
  1925.             else if(isset($profile[$weight]&& $profile[$weight!= ""){
  1926.                 $data[$weight$profile[$weight];
  1927.             else {
  1928.                 $data[$weight$value;
  1929.                 $profile[$weight$data[$weight];
  1930.                 $change true;
  1931.             }
  1932.         }
  1933.         if($change == true{
  1934.             $this->profileModel->updateProfile(WORK_DIRECTORYarray(),
  1935.                 $profile);
  1936.         }
  1937.  
  1938.         $data['INDEXED_FILE_TYPES'array();
  1939.         $filetypes array();
  1940.         foreach($INDEXED_FILE_TYPES as $filetype{
  1941.             $ison =false;
  1942.             if(isset($_REQUEST["filetype"]&& !$loaded{
  1943.                 if(isset($_REQUEST["filetype"][$filetype])) {
  1944.                     $filetypes[$filetype;
  1945.                     $ison true;
  1946.                     $change true;
  1947.                 }
  1948.             else {
  1949.                 if(in_array($filetype,
  1950.                     $seed_info["indexed_file_types"]["extensions"])) {
  1951.                     $filetypes[$filetype;
  1952.                     $ison true;
  1953.                 }
  1954.             }
  1955.             $data['INDEXED_FILE_TYPES'][$filetype($ison?
  1956.                 "checked='checked'" :'';
  1957.         }
  1958.         $seed_info["indexed_file_types"]["extensions"$filetypes;
  1959.  
  1960.         $data['CLASSIFIERS'array();
  1961.         $data['RANKERS'array();
  1962.         $active_classifiers array();
  1963.         $active_rankers array();
  1964.  
  1965.         foreach (Classifier::getClassifierList(as $classifier{
  1966.             $label $classifier->class_label;
  1967.             $ison false;
  1968.             if (isset($_REQUEST['classifier']&& !$loaded{
  1969.                 if (isset($_REQUEST['classifier'][$label])) {
  1970.                     $ison true;
  1971.                 }
  1972.             else if ($loaded || !isset($_REQUEST['posted']&&
  1973.                 isset($seed_info['active_classifiers']['label'])) {
  1974.                 if (in_array($label,
  1975.                     $seed_info['active_classifiers']['label'])) {
  1976.                     $ison true;
  1977.                 }
  1978.             }
  1979.             if ($ison{
  1980.                 $data['CLASSIFIERS'][$label'checked="checked"';
  1981.                 $active_classifiers[$label;
  1982.             else {
  1983.                 $data['CLASSIFIERS'][$label'';
  1984.             }
  1985.             $ison false;
  1986.             if (isset($_REQUEST['ranker']&& !$loaded{
  1987.                 if (isset($_REQUEST['ranker'][$label])) {
  1988.                     $ison true;
  1989.                 }
  1990.             else if ($loaded || !isset($_REQUEST['posted']&&
  1991.                 isset($seed_info['active_rankers']['label'])) {
  1992.                 if (isset($seed_info['active_rankers']['label']&&
  1993.                     in_array($label$seed_info['active_rankers']['label'])) {
  1994.                     $ison true;
  1995.                 }
  1996.             }
  1997.             if ($ison{
  1998.                 $data['RANKERS'][$label'checked="checked"';
  1999.                 $active_rankers[$label;
  2000.             else {
  2001.                 $data['RANKERS'][$label'';
  2002.             }
  2003.         }
  2004.         $seed_info['active_classifiers']['label'$active_classifiers;
  2005.         $seed_info['active_rankers']['label'$active_rankers;
  2006.  
  2007.         if(isset($seed_info['page_rules']['rule'])) {
  2008.             if(isset($seed_info['page_rules']['rule']['rule'])) {
  2009.                 $data['page_rules'$this->convertArrayLines(
  2010.                     $seed_info['page_rules']['rule']['rule']);
  2011.             else {
  2012.                 $data['page_rules'$this->convertArrayLines(
  2013.                     $seed_info['page_rules']['rule']);
  2014.             }
  2015.         else {
  2016.             $data['page_rules'"";
  2017.         }
  2018.         $allowed_options array('crawl_time''search_time''test_options');
  2019.         if(isset($_REQUEST['option_type']&&
  2020.             in_array($_REQUEST['option_type']$allowed_options)) {
  2021.             $data['option_type'$_REQUEST['option_type'];
  2022.         else {
  2023.             $data['option_type''crawl_time';
  2024.         }
  2025.         if($data['option_type'== 'crawl_time'{
  2026.             $data['crawl_time_active'"active";
  2027.             $data['search_time_active'"";
  2028.             $data['test_options_active'"";
  2029.             $data['SCRIPT'.= "\nswitchTab('crawltimetab',".
  2030.                 "'searchtimetab', 'testoptionstab')\n";
  2031.         else if($data['option_type'== 'search_time'{
  2032.             $data['search_time_active'"active";
  2033.             $data['crawl_time_active'"";
  2034.             $data['test_options_active'"";
  2035.             $data['SCRIPT'.= "\nswitchTab('searchtimetab',".
  2036.                 "'crawltimetab', 'testoptionstab')\n";
  2037.         else {
  2038.             $data['search_time_active'"";
  2039.             $data['crawl_time_active'"";
  2040.             $data['test_options_active'"active";
  2041.             $data['SCRIPT'.= "\nswitchTab('testoptionstab',".
  2042.                 "'crawltimetab', 'searchtimetab');\n";
  2043.         }
  2044.  
  2045.         $this->crawlModel->setSeedInfo($seed_info);
  2046.         if($change == true && $data['option_type'!= 'test_options'{
  2047.             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2048.                 tl('admin_controller_page_options_updated')."</h1>')";
  2049.         }
  2050.         $test_processors array(
  2051.             "text/html" => "HtmlProcessor",
  2052.             "text/asp" => "HtmlProcessor",
  2053.             "text/xml" => "XmlProcessor",
  2054.             "text/robot" => "RobotProcessor",
  2055.             "application/xml" => "XmlProcessor",
  2056.             "application/xhtml+xml" => "HtmlProcessor",
  2057.             "application/rss+xml" => "RssProcessor",
  2058.             "application/atom+xml" => "RssProcessor",
  2059.             "text/rtf" => "RtfProcessor",
  2060.             "text/plain" => "TextProcessor",
  2061.             "text/csv" => "TextProcessor",
  2062.             "text/tab-separated-values" => "TextProcessor",
  2063.         );
  2064.         $data['MIME_TYPES'array_keys($test_processors);
  2065.         $data['page_type'"text/html";
  2066.         if(isset($_REQUEST['page_type']&& in_array($_REQUEST['page_type'],
  2067.             $data['MIME_TYPES'])) {
  2068.             $data['page_type'$_REQUEST['page_type'];
  2069.         }
  2070.         $data['TESTPAGE'(isset($_REQUEST['TESTPAGE'])) ?
  2071.             $this->clean($_REQUEST['TESTPAGE']'string'"";
  2072.         if($data['option_type'== 'test_options' && $data['TESTPAGE'!=""{
  2073.             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2074.                 tl('admin_controller_page_options_running_tests')."</h1>')";
  2075.             $site array();
  2076.             $site[self::ENCODING"UTF-8";
  2077.             $site[self::URL"http://test-site.yioop.com/";
  2078.             $site[self::IP_ADDRESSESarray("1.1.1.1");
  2079.             $site[self::HTTP_CODE200;
  2080.             $site[self::MODIFIEDdate("U"time());
  2081.             $site[self::TIMESTAMPtime();
  2082.             $site[self::TYPE"text/html";
  2083.             $site[self::HEADER"page options test extractor";
  2084.             $site[self::SERVER"unknown";
  2085.             $site[self::SERVER_VERSION"unknown";
  2086.             $site[self::OPERATING_SYSTEM"unknown";
  2087.             $site[self::LANG'en';
  2088.             $site[self::JUST_METASfalse;
  2089.             if(isset($_REQUEST['page_type']&&
  2090.                 in_array($_REQUEST['page_type']$data['MIME_TYPES'])) {
  2091.                 $site[self::TYPE$_REQUEST['page_type'];
  2092.             }
  2093.             if($site[self::TYPE== 'text/html'{
  2094.                 $site[self::ENCODING=
  2095.                     guessEncodingHtml($_REQUEST['TESTPAGE']);
  2096.             }
  2097.             $processor_name $test_processors[$site[self::TYPE]];
  2098.             $plugin_processors array();
  2099.             if (isset($seed_info['indexing_plugins']['plugins'])) {
  2100.                 foreach($seed_info['indexing_plugins']['plugins'as $plugin{
  2101.                     $plugin_name $plugin."Plugin";
  2102.                     $supported_processors $plugin_name::getProcessors();
  2103.                     foreach($supported_processors as $supported_processor{
  2104.                         if ($supported_processor == $processor_name{
  2105.                             $plugin_processors[$plugin_name;
  2106.                         }
  2107.                     }
  2108.                 }
  2109.             }
  2110.             $page_processor new $processor_name($plugin_processors,
  2111.                 $seed_info["general"]["max_description_len"]);
  2112.             $doc_info $page_processor->handle($_REQUEST['TESTPAGE'],
  2113.                 $site[self::URL]);
  2114.  
  2115.             if($page_processor != "RobotProcessor" &&
  2116.                 !isset($doc_info[self::JUST_METAS])) {
  2117.                 $doc_info[self::LINKSUrlParser::pruneLinks(
  2118.                     $doc_info[self::LINKS]);
  2119.             }
  2120.             foreach($doc_info as $key => $value{
  2121.                 $site[$key$value;
  2122.             }
  2123.             if(isset($site[self::PAGE])) {
  2124.                 unset($site[self::PAGE]);
  2125.             }
  2126.             if(isset($site[self::ROBOT_PATHS])) {
  2127.                 $site[self::JUST_METAStrue;
  2128.             }
  2129.             $reflect new ReflectionClass("CrawlConstants");
  2130.             $crawl_constants $reflect->getConstants();
  2131.             $crawl_keys array_keys($crawl_constants);
  2132.             $crawl_values array_values($crawl_constants);
  2133.             $inverse_constants array_combine($crawl_values$crawl_keys);
  2134.             $after_process array();
  2135.             foreach($site as $key => $value{
  2136.                 $out_key (isset($inverse_constants[$key])) ?
  2137.                     $inverse_constants[$key$key;
  2138.                 $after_process[$out_key$value;
  2139.             }
  2140.             $data["AFTER_PAGE_PROCESS"wordwrap($this->clean(
  2141.                 print_r($after_processtrue)"string")75"\n"true);
  2142.             $rule_string implode("\n"$seed_info['page_rules']['rule']);
  2143.             $rule_string html_entity_decode($rule_stringENT_QUOTES);
  2144.             $page_rule_parser =
  2145.                 new PageRuleParser($rule_string);
  2146.             $page_rule_parser->executeRuleTrees($site);
  2147.             $after_process array();
  2148.             foreach($site as $key => $value{
  2149.                 $out_key (isset($inverse_constants[$key])) ?
  2150.                     $inverse_constants[$key$key;
  2151.                 $after_process[$out_key$value;
  2152.             }
  2153.             $data["AFTER_RULE_PROCESS"wordwrap($this->clean(
  2154.                 print_r($after_processtrue)"string")75"\n"true);
  2155.             $lang NULL;
  2156.             if(isset($site[self::LANG])) {
  2157.                 $lang $site[self::LANG];
  2158.             }
  2159.             $meta_ids PhraseParser::calculateMetas($site);
  2160.             if(!$site[self::JUST_METAS]{
  2161.                 $host_words UrlParser::getWordsIfHostUrl($site[self::URL]);
  2162.                 $path_words UrlParser::getWordsLastPathPartUrl(
  2163.                     $site[self::URL]);
  2164.                 $phrase_string $host_words." ".$site[self::TITLE.
  2165.                     " "$path_words " "$site[self::DESCRIPTION];
  2166.                 if($site[self::TITLE!= "" {
  2167.                     $lang guessLocaleFromString($site[self::TITLE]$lang);
  2168.                 else {
  2169.                     $lang guessLocaleFromString(
  2170.                         substr($site[self::DESCRIPTION]0,
  2171.                         AD_HOC_TITLE_LENGTH)$lang);
  2172.                 }
  2173.                 $word_lists =
  2174.                     PhraseParser::extractPhrasesInLists($phrase_string,
  2175.                         $lang);
  2176.                 $len strlen($phrase_string);
  2177.                 if(PhraseParser::computeSafeSearchScore($word_lists$len<
  2178.                     0.012{
  2179.                     $meta_ids["safe:true";
  2180.                     $safe true;
  2181.                 else {
  2182.                     $meta_ids["safe:false";
  2183.                     $safe false;
  2184.                 }
  2185.             }
  2186.             if(!isset($word_lists)) {
  2187.                 $word_lists array();
  2188.             }
  2189.             $data["EXTRACTED_WORDS"wordwrap($this->clean(
  2190.                 print_r($word_liststrue)"string")75"\n"true);;
  2191.             $data["EXTRACTED_META_WORDS"wordwrap($this->clean(
  2192.                 print_r($meta_idstrue)"string")75"\n"true);
  2193.         }
  2194.         return $data;
  2195.     }
  2196.  
  2197.     /**
  2198.      * Handles admin requests for creating, editing, and deleting classifiers.
  2199.      *
  2200.      * This activity implements the logic for the page that lists existing
  2201.      * classifiers, including the actions that can be performed on them.
  2202.      */
  2203.     function manageClassifiers()
  2204.     {
  2205.         $possible_arguments array('createclassifier''editclassifier',
  2206.             'finalizeclassifier''deleteclassifier');
  2207.  
  2208.         $data['ELEMENT''manageclassifiersElement';
  2209.         $data['SCRIPT''';
  2210.  
  2211.         $machine_urls $this->machineModel->getQueueServerUrls();
  2212.         $num_machines count($machine_urls);
  2213.         if ($num_machines || ($num_machines == &&
  2214.             UrlParser::isLocalhostUrl($machine_urls[0]))) {
  2215.             $machine_urls NULL;
  2216.         }
  2217.  
  2218.         $data['leftorright'=
  2219.             (getLocaleDirection(== 'ltr''right''left';
  2220.  
  2221.         $classifiers Classifier::getClassifierList();
  2222.         $start_finalizing false;
  2223.         if (isset($_REQUEST['arg']&&
  2224.             in_array($_REQUEST['arg']$possible_arguments)) {
  2225.             $label $this->clean($_REQUEST['class_label']'string');
  2226.             $label Classifier::cleanLabel($label);
  2227.             switch ($_REQUEST['arg'])
  2228.             {
  2229.                 case 'createclassifier':
  2230.                     if (!isset($classifiers[$label])) {
  2231.                         $classifier new Classifier($label);
  2232.                         Classifier::setClassifier($classifier);
  2233.                         $classifiers[$label$classifier;
  2234.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2235.                             tl('admin_controller_new_classifier').'</h1>\');';
  2236.                     else {
  2237.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2238.                             tl('admin_controller_classifier_exists').
  2239.                             '</h1>\');';
  2240.                     }
  2241.                     break;
  2242.  
  2243.                 case 'editclassifier':
  2244.                     if (isset($classifiers[$label])) {
  2245.                         $data['class_label'$label;
  2246.                         $this->editClassifier($data$classifiers,
  2247.                             $machine_urls);
  2248.                     else {
  2249.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2250.                             tl('admin_controller_no_classifier').
  2251.                             '</h1>\');';
  2252.                     }
  2253.                     break;
  2254.  
  2255.                 case 'finalizeclassifier':
  2256.                     /*
  2257.                        Finalizing is too expensive to be done directly in the
  2258.                        controller that responds to the web request. Instead, a
  2259.                        daemon is launched to finalize the classifier
  2260.                        asynchronously and save it back to disk when it's done.
  2261.                        In the meantime, a flag is set to indicate the current
  2262.                        finalizing state.
  2263.                      */
  2264.                     CrawlDaemon::start("classifier_trainer"$label''-1);
  2265.                     $classifier $classifiers[$label];
  2266.                     $classifier->finalized Classifier::FINALIZING;
  2267.                     $start_finalizing true;
  2268.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2269.                         tl('admin_controller_finalizing_classifier').
  2270.                         '</h1>\');';
  2271.                     break;
  2272.  
  2273.                 case 'deleteclassifier':
  2274.                     /*
  2275.                        In addition to deleting the classifier, we also want to
  2276.                        delete the associated crawl mix (if one exists) used to
  2277.                        iterate over existing indexes in search of new training
  2278.                        examples.
  2279.                      */
  2280.                     if (isset($classifiers[$label])) {
  2281.                         unset($classifiers[$label]);
  2282.                         Classifier::deleteClassifier($label);
  2283.                         $mix_name Classifier::getCrawlMixName($label);
  2284.                         $mix_time $this->crawlModel->getCrawlMixTimestamp(
  2285.                             $mix_name);
  2286.                         if ($mix_time{
  2287.                             $this->crawlModel->deleteCrawlMixIteratorState(
  2288.                                 $mix_time);
  2289.                             $this->crawlModel->deleteCrawlMix($mix_time);
  2290.                         }
  2291.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2292.                             tl('admin_controller_classifier_deleted').
  2293.                             '</h1>\');';
  2294.                     else {
  2295.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2296.                             tl('admin_controller_no_classifier').
  2297.                             '</h1>\');';
  2298.                     }
  2299.                     break;
  2300.             }
  2301.         }
  2302.  
  2303.         $data['classifiers'$classifiers;
  2304.         $data['reload'false;
  2305.         foreach($classifiers as $label => $classifier{
  2306.             if($classifier->finalized == Classifier::FINALIZING{
  2307.                 $data['reload'true;
  2308.                 break;
  2309.             }
  2310.         }
  2311.         if($data['reload'&& !$start_finalizing{
  2312.             $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2313.                 tl('admin_controller_finalizing_classifier')'</h1>\');';
  2314.         }
  2315.         return $data;
  2316.     }
  2317.  
  2318.     /**
  2319.      * Handles the particulars of editing a classifier, which includes changing
  2320.      * its label and adding training examples.
  2321.      *
  2322.      * This activity directly handles changing the class label, but not adding
  2323.      * training examples. The latter activity is done interactively without
  2324.      * reloading the page via XmlHttpRequests, coordinated by the classifier
  2325.      * controller dedicated to that task.
  2326.      *
  2327.      * @param array $data data to be passed on to the view
  2328.      * @param array $classifiers map from class labels to their associated
  2329.      *      classifiers
  2330.      * @param array $machine_urls string urls of machines managed by this
  2331.      *      Yioop name server
  2332.      */
  2333.     function editClassifier(&$data$classifiers$machine_urls)
  2334.     {
  2335.         $data['ELEMENT''editclassifierElement';
  2336.         $data['INCLUDE_SCRIPTS'array('classifiers');
  2337.  
  2338.         // We want recrawls, but not archive crawls.
  2339.         $crawls $this->crawlModel->getCrawlList(falsetrue$machine_urls);
  2340.         $data['CRAWLS'$crawls;
  2341.  
  2342.         $classifier $classifiers[$data['class_label']];
  2343.  
  2344.         if (isset($_REQUEST['update']&& $_REQUEST['update'== 'update'{
  2345.             if (isset($_REQUEST['rename_label'])) {
  2346.                 $new_label $this->clean($_REQUEST['rename_label']'string');
  2347.                 $new_label preg_replace('/[^a-zA-Z0-9_]/'''$new_label);
  2348.                 if (!isset($classifiers[$new_label])) {
  2349.                     $old_label $classifier->class_label;
  2350.                     $classifier->class_label $new_label;
  2351.                     Classifier::setClassifier($classifier);
  2352.                     Classifier::deleteClassifier($old_label);
  2353.                     $data['class_label'$new_label;
  2354.                 else {
  2355.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\">".
  2356.                         tl('admin_controller_classifier_exists').
  2357.                         '</h1>\');';
  2358.                 }
  2359.             }
  2360.         }
  2361.  
  2362.         $data['classifier'$classifier;
  2363.  
  2364.         // Translations for the classification javascript.
  2365.         $data['SCRIPT'.= "window.tl = {".
  2366.             'editclassifier_load_failed:"'.
  2367.                 tl('editclassifier_load_failed').'",'.
  2368.             'editclassifier_loading:"'.
  2369.                 tl('editclassifier_loading').'",'.
  2370.             'editclassifier_added_examples:"'.
  2371.                 tl('editclassifier_added_examples').'",'.
  2372.             'editclassifier_label_update_failed:"'.
  2373.                 tl('editclassifier_label_update_failed').'",'.
  2374.             'editclassifier_updating:"'.
  2375.                 tl('editclassifier_updating').'",'.
  2376.             'editclassifier_acc_update_failed:"'.
  2377.                 tl('editclassifier_acc_update_failed').'",'.
  2378.             'editclassifier_na:"'.
  2379.                 tl('editclassifier_na').'",'.
  2380.             'editclassifier_no_docs:"'.
  2381.                 tl('editclassifier_no_docs').'",'.
  2382.             'editclassifier_num_docs:"'.
  2383.                 tl('editclassifier_num_docs').'",'.
  2384.             'editclassifier_in_class:"'.
  2385.                 tl('editclassifier_in_class').'",'.
  2386.             'editclassifier_not_in_class:"'.
  2387.                 tl('editclassifier_not_in_class').'",'.
  2388.             'editclassifier_skip:"'.
  2389.                 tl('editclassifier_skip').'",'.
  2390.             'editclassifier_prediction:"'.
  2391.                 tl('editclassifier_prediction').'",'.
  2392.             'editclassifier_scores:"'.
  2393.                 tl('editclassifier_scores').'"'.
  2394.             '};';
  2395.  
  2396.         /*
  2397.            We pass along authentication information to the client, so that it
  2398.            can authenticate any XmlHttpRequests that it makes in order to label
  2399.            documents.
  2400.          */
  2401.         $time strval(time());
  2402.         $session md5($time.AUTH_KEY);
  2403.         $data['SCRIPT'.=
  2404.             "Classifier.initialize(".
  2405.                 "'{$data['class_label']}',".
  2406.                 "'{$session}',".
  2407.                 "'{$time}');";
  2408.     }
  2409.  
  2410.     /**
  2411.      * Handles admin request related to the search filter activity
  2412.      *
  2413.      * This activity allows a user to specify hosts whose web pages are to be
  2414.      * filtered out the search results
  2415.      *
  2416.      * @return array $data info about the groups and their contents for a
  2417.      *       particular crawl mix
  2418.      */
  2419.     function resultsEditor()
  2420.     {
  2421.         $data["ELEMENT""resultseditorElement";
  2422.         $data['SCRIPT'"";
  2423.  
  2424.         if(isset($_REQUEST['disallowed_sites'])) {
  2425.             $sites $this->convertStringCleanArray(
  2426.                 $_REQUEST['disallowed_sites']);
  2427.             $disallowed_sites array();
  2428.             foreach($sites as $site{
  2429.                 $site UrlParser::getHost($site);
  2430.                 if(strlen($site0{
  2431.                     $disallowed_sites[$site."/";
  2432.                 }
  2433.             }
  2434.             $data['disallowed_sites'implode("\n"$disallowed_sites);
  2435.             $this->searchfiltersModel->set($disallowed_sites);
  2436.             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2437.                 tl('admin_controller_results_editor_update')."</h1>')";
  2438.         }
  2439.         if(!isset($data['disallowed_sites'])) {
  2440.             $data['disallowed_sites'=
  2441.                 implode("\n"$this->searchfiltersModel->getUrls());
  2442.         }
  2443.         foreach (array("URL""TITLE""DESCRIPTION"as $field{
  2444.             $data[$field(isset($_REQUEST[$field])) ?
  2445.                 $this->clean($_REQUEST[$field]"string":
  2446.                  ((isset($data[$field]) ) $data[$field"");
  2447.         }
  2448.         if($data["URL"!= ""{
  2449.             $data["URL"UrlParser::canonicalLink($data["URL"],"");
  2450.         }
  2451.         $tmp tl('admin_controller_edited_pages');
  2452.         $data["URL_LIST"array ($tmp => $tmp);
  2453.         $summaries $this->searchfiltersModel->getEditedPageSummaries();
  2454.         foreach($summaries as $hash => $summary{
  2455.             $data["URL_LIST"][$summary[self::URL]] $summary[self::URL];
  2456.         }
  2457.         if(isset($_REQUEST['arg']) ) {
  2458.             switch($_REQUEST['arg'])
  2459.             {
  2460.                 case "save_page":
  2461.                     $missing_page_field ($data["URL"== ""truefalse;
  2462.                     if($missing_page_field{
  2463.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2464.                             tl('admin_controller_results_editor_need_url').
  2465.                             "</h1>')";
  2466.                     else {
  2467.                         $this->searchfiltersModel->updateResultPage(
  2468.                             $data["URL"]$data["TITLE"]$data["DESCRIPTION"]);
  2469.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2470.                             tl('admin_controller_results_editor_page_updated').
  2471.                             "</h1>')";
  2472.                     }
  2473.                 break;
  2474.                 case "load_url":
  2475.                     $hash_url crawlHash($_REQUEST['LOAD_URL']true);
  2476.                     if(isset($summaries[$hash_url])) {
  2477.                         $data["URL"$this->clean($_REQUEST['LOAD_URL'],
  2478.                             "string");
  2479.                         $data["TITLE"$summaries[$hash_url][self::TITLE];
  2480.                         $data["DESCRIPTION"$summaries[$hash_url][
  2481.                             self::DESCRIPTION];
  2482.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2483.                             tl('admin_controller_results_editor_page_loaded').
  2484.                             "</h1>')";
  2485.                     }
  2486.                 break;
  2487.             }
  2488.         }
  2489.  
  2490.         return $data;
  2491.     }
  2492.  
  2493.  
  2494.     /**
  2495.      * Handles admin request related to the managing the machines which perform
  2496.      *  crawls
  2497.      *
  2498.      * With this activity an admin can add/delete machines to manage. For each
  2499.      * managed machine, the admin can stop and start fetchers/queue_servers
  2500.      * as well as look at their log files
  2501.      *
  2502.      * @return array $data MACHINES, their MACHINE_NAMES, data for
  2503.      *       FETCHER_NUMBERS drop-down
  2504.      */
  2505.     function manageMachines()
  2506.     {
  2507.         $data array();
  2508.         $data["ELEMENT""managemachinesElement";
  2509.         $possible_arguments array("addmachine""deletemachine",
  2510.             "newsmode""log""update");
  2511.         $data['SCRIPT'"doUpdate();";
  2512.         $data["leftorright"]=(getLocaleDirection(== 'ltr'"right""left";
  2513.         $data['MACHINES'array();
  2514.         $data['MACHINE_NAMES'array();
  2515.         $urls array();
  2516.         $data['FETCHER_NUMBERS'array(
  2517.             => 0,
  2518.             => 1,
  2519.             => 2,
  2520.             => 3,
  2521.             => 4,
  2522.             => 5,
  2523.             => 6,
  2524.             => 7,
  2525.             => 8,
  2526.             16 => 16
  2527.         );
  2528.         $machines $this->machineModel->getMachineList();
  2529.         $tmp tl('admin_controller_select_machine');
  2530.         $data['DELETABLE_MACHINES'array(
  2531.             $tmp => $tmp
  2532.         );
  2533.         $data['REPLICATABLE_MACHINES'array(
  2534.             $tmp => $tmp
  2535.         );
  2536.         foreach($machines as $machine{
  2537.             $data['MACHINE_NAMES'][$machine["NAME"];
  2538.             $urls[$machine["URL"];
  2539.             $data['DELETABLE_MACHINES'][$machine["NAME"]] $machine["NAME"];
  2540.             if(!isset($machine["PARENT"]|| $machine["PARENT"== ""{
  2541.                 $data['REPLICATABLE_MACHINES'][$machine["NAME"]]
  2542.                     = $machine["NAME"];
  2543.             }
  2544.         }
  2545.  
  2546.         if(!isset($_REQUEST["has_queue_server"]||
  2547.             isset($_REQUEST['is_replica'])) {
  2548.             $_REQUEST["has_queue_server"false;
  2549.         }
  2550.         if(isset($_REQUEST['is_replica'])) {
  2551.             $_REQUEST['num_fetchers'0;
  2552.         else {
  2553.             $_REQUEST['parent'"";
  2554.         }
  2555.         $request_fields array(
  2556.             "name" => "string",
  2557.             "url" => "string",
  2558.             "has_queue_server" => "bool",
  2559.             "num_fetchers" => "int",
  2560.             "parent" => "string"
  2561.         );
  2562.         $r array();
  2563.  
  2564.         $allset true;
  2565.         foreach($request_fields as $field => $type{
  2566.             if(isset($_REQUEST[$field])) {
  2567.                 $r[$field$this->clean($_REQUEST[$field]$type);
  2568.                 if($field == "url" && $r[$field][strlen($r[$field])-1]
  2569.                     != "/"{
  2570.                     $r[$field.= "/";
  2571.                 }
  2572.             else {
  2573.                 $allset false;
  2574.             }
  2575.         }
  2576.         if(isset($r["num_fetchers"]&&
  2577.             in_array($r["num_fetchers"]$data['FETCHER_NUMBERS'])) {
  2578.             $data['FETCHER_NUMBER'$r["num_fetchers"];
  2579.         else {
  2580.             $data['FETCHER_NUMBER'0;
  2581.             if(isset($r["num_fetchers"])) {
  2582.                 $r["num_fetchers"0;
  2583.             }
  2584.         }
  2585.         $machine_exists (isset($r["name"]&& in_array($r["name"],
  2586.             $data['MACHINE_NAMES']) ) || (isset($r["url"]&&
  2587.             in_array($r["url"]$urls));
  2588.  
  2589.         if(isset($_REQUEST['arg']&&
  2590.             in_array($_REQUEST['arg']$possible_arguments)) {
  2591.             switch($_REQUEST['arg'])
  2592.             {
  2593.                 case "addmachine":
  2594.                     if($allset == true && !$machine_exists{
  2595.                         $this->machineModel->addMachine(
  2596.                             $r["name"]$r["url"]$r["has_queue_server"],
  2597.                             $r["num_fetchers"]$r["parent"]);
  2598.  
  2599.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2600.                             tl('admin_controller_machine_added').
  2601.                             "</h1>');";
  2602.                         $data['MACHINE_NAMES'][$r["name"];
  2603.                         $data['DELETABLE_MACHINES'][$r["name"]] $r["name"];
  2604.                         sort($data['MACHINE_NAMES']);
  2605.                     else if ($allset && $machine_exists {
  2606.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2607.                             tl('admin_controller_machine_exists').
  2608.                             "</h1>');";
  2609.                     else {
  2610.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2611.                             tl('admin_controller_machine_incomplete').
  2612.                             "</h1>');";
  2613.                     }
  2614.                 break;
  2615.  
  2616.                 case "deletemachine":
  2617.                     if(!isset($r["name"]||
  2618.                         !in_array($r["name"]$data['MACHINE_NAMES'])) {
  2619.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2620.                             tl('admin_controller_machine_doesnt_exists').
  2621.                             "</h1>');";
  2622.                     else {
  2623.                         $machines $this->machineModel->getMachineStatuses();
  2624.                         $service_in_use false;
  2625.                         foreach($machines as $machine{
  2626.                             if($machine['NAME'== $r["name"]{
  2627.                                 if($machine['STATUSES'!= array()) {
  2628.                                     $service_in_use true;
  2629.                                     break;
  2630.                                 else {
  2631.                                     break;
  2632.                                 }
  2633.                             }
  2634.                         }
  2635.                         if($service_in_use{
  2636.                             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2637.                            tl('admin_controller_stop_service_first')."</h1>');";
  2638.                             break;
  2639.                         }
  2640.                         $this->machineModel->deleteMachine($r["name"]);
  2641.                         $tmp_array array($r["name"]);
  2642.                         $diff =
  2643.                             array_diff($data['MACHINE_NAMES'],  $tmp_array);
  2644.                         $data['MACHINE_NAMES'array_merge($diff);
  2645.                         $tmp_array array($r["name"=> $r["name"]);
  2646.                         $diff =
  2647.                             array_diff($data['DELETABLE_MACHINES']$tmp_array);
  2648.                         $data['DELETABLE_MACHINES'array_merge($diff);
  2649.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2650.                             tl('admin_controller_machine_deleted')."</h1>');";
  2651.                     }
  2652.                 break;
  2653.  
  2654.                 case "newsmode":
  2655.                     $profile =  $this->profileModel->getProfile(WORK_DIRECTORY);
  2656.                     $news_modes array("news_off""news_web""news_process");
  2657.                     if(isset($_REQUEST['news_mode']&& in_array(
  2658.                         $_REQUEST['news_mode']$news_modes)) {
  2659.                         $profile["NEWS_MODE"$_REQUEST['news_mode'];
  2660.                         if($profile["NEWS_MODE"!= "news_process"{
  2661.                             CrawlDaemon::stop("news_updater"""false);
  2662.                             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2663.                                 tl('admin_controller_news_mode_updated').
  2664.                                 "</h1>');";
  2665.                         else {
  2666.                             CrawlDaemon::start("news_updater"'none'"",
  2667.                                 -1);
  2668.                             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2669.                                 tl('admin_controller_news_mode_updated').
  2670.                                 "</h1>');";
  2671.                         }
  2672.                         $this->profileModel->updateProfile(
  2673.                             WORK_DIRECTORYarray()$profile);
  2674.                     else {
  2675.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2676.                             tl('admin_controller_news_update_failed').
  2677.                             "</h1>');";
  2678.                     }
  2679.                 break;
  2680.  
  2681.                 case "log":
  2682.                     if(isset($_REQUEST["fetcher_num"])) {
  2683.                         $r["fetcher_num"=
  2684.                             $this->clean($_REQUEST["fetcher_num"]"int");
  2685.                     }
  2686.                     if(isset($_REQUEST["mirror_name"])) {
  2687.                         $r["mirror_name"=
  2688.                             $this->clean($_REQUEST["mirror_name"]"string");
  2689.                     }
  2690.                     if(isset($_REQUEST["time"])) {
  2691.                         $data["time"=
  2692.                             $this->clean($_REQUEST["time"]"int"30;
  2693.                     else {
  2694.                         $data["time"30;
  2695.                     }
  2696.                     if(isset($_REQUEST["NO_REFRESH"])) {
  2697.                         $data["NO_REFRESH"$this->clean(
  2698.                             $_REQUEST["NO_REFRESH"]"bool");
  2699.                     else {
  2700.                         $data["NO_REFRESH"false;
  2701.                     }
  2702.                     $data["ELEMENT""machinelogElement";
  2703.                     $filter"";
  2704.                     if(isset($_REQUEST['f'])) {
  2705.                         $filter =
  2706.                             $this->clean($_REQUEST['f']"string");
  2707.                     }
  2708.                     $data['filter'$filter;
  2709.                     $data["REFRESH_LOG""&time="$data["time"];
  2710.                     $data["LOG_TYPE""";
  2711.                     if(isset($r['fetcher_num']&& isset($r['name'])) {
  2712.                         $data["LOG_FILE_DATA"$this->machineModel->getLog(
  2713.                             $r["name"]$r["fetcher_num"]$filter);
  2714.                         $data["LOG_TYPE"$r['name'].
  2715.                             " fetcher ".$r["fetcher_num"];
  2716.                         $data["REFRESH_LOG".= "&arg=log&name=".$r['name'].
  2717.                             "&fetcher_num=".$r['fetcher_num'];
  2718.                     else if(isset($r["mirror_name"])) {
  2719.                         $data["LOG_TYPE"$r['mirror_name']." mirror";
  2720.                         $data["LOG_FILE_DATA"$this->machineModel->getLog(
  2721.                             $r["mirror_name"]NULL$filter,  true);
  2722.                     else if(isset($r['name'])) {
  2723.                         $data["LOG_TYPE"$r['name']." queue_server";
  2724.                         if($r['name'== "news"{
  2725.                             $data["LOG_TYPE""Name Server News Updater";
  2726.                         }
  2727.                         $data["LOG_FILE_DATA"$this->machineModel->getLog(
  2728.                             $r["name"]NULL$filter);
  2729.                         $data["REFRESH_LOG".=
  2730.                             "&arg=log&name=".$r['name'];
  2731.                     }
  2732.                     if($data["time">= 1200{
  2733.                         $data["REFRESH_LOG""";
  2734.                     }
  2735.  
  2736.                     if(!isset($data["LOG_FILE_DATA"])
  2737.                         || $data["LOG_FILE_DATA"== ""){
  2738.                         $data["LOG_FILE_DATA"=
  2739.                             tl('admin_controller_no_machine_log');
  2740.                     }
  2741.                     $lines =array_reverse(explode("\n",$data["LOG_FILE_DATA"]));
  2742.                     $data["LOG_FILE_DATA"implode("\n"$lines);
  2743.                 break;
  2744.  
  2745.                 case "update":
  2746.                     if(isset($_REQUEST["fetcher_num"])) {
  2747.                         $r["fetcher_num"=
  2748.                             $this->clean($_REQUEST["fetcher_num"]"int");
  2749.                     else {
  2750.                         $r["fetcher_num"NULL;
  2751.                     }
  2752.                     $available_actions array("start""stop",
  2753.                         "mirror_start""mirror_stop");
  2754.                     if(isset($r["name"]&& isset($_REQUEST["action"]&&
  2755.                         in_array($_REQUEST["action"]$available_actions)) {
  2756.                         $action $_REQUEST["action"];
  2757.                         $is_mirror false;
  2758.                         if($action == "mirror_start"{
  2759.                             $action "start";
  2760.                             $is_mirror true;
  2761.                         else if ($action == "mirror_stop"{
  2762.                             $action "stop";
  2763.                             $is_mirror true;
  2764.                         }
  2765.                         $this->machineModel->update($r["name"],
  2766.                             $action$r["fetcher_num"]$is_mirror);
  2767.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2768.                             tl('admin_controller_machine_servers_updated').
  2769.                             "</h1>');";
  2770.                     else {
  2771.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2772.                             tl('admin_controller_machine_no_action').
  2773.                             "</h1>');";
  2774.                     }
  2775.  
  2776.                 break;
  2777.  
  2778.             }
  2779.         }
  2780.         if(!isset($_REQUEST['arg']|| $_REQUEST['arg'!= 'log'{
  2781.             $data['SCRIPT'.= "toggleReplica(false);";
  2782.         }
  2783.         return $data;
  2784.     }
  2785.  
  2786.     /**
  2787.      * Handles admin request related to the manage locale activity
  2788.      *
  2789.      * The manage locale activity allows a user to add/delete locales, view
  2790.      * statistics about a locale as well as edit the string for that locale
  2791.      *
  2792.      * @return array $data info about current locales, statistics for each
  2793.      *       locale as well as potentially the currently set string of a
  2794.      *       locale and any messages about the success or failure of a
  2795.      *       sub activity.
  2796.      */
  2797.     function manageLocales()
  2798.     {
  2799.         $possible_arguments array("addlocale""deletelocale""editlocale");
  2800.  
  2801.         $data['SCRIPT'"";
  2802.         $data["ELEMENT""managelocalesElement";
  2803.  
  2804.         $data["LOCALES"$this->localeModel->getLocaleList();
  2805.         $data['LOCALE_NAMES'][-1tl('admin_controller_select_localename');
  2806.  
  2807.         $locale_ids array();
  2808.  
  2809.         foreach ($data["LOCALES"as $locale{
  2810.             $data["LOCALE_NAMES"][$locale["LOCALE_TAG"]] =
  2811.                 $locale["LOCALE_NAME"];
  2812.             $locale_ids[$locale["LOCALE_TAG"];
  2813.         }
  2814.  
  2815.         if(isset($_REQUEST['arg']&&
  2816.             in_array($_REQUEST['arg']$possible_arguments)) {
  2817.             if(isset($_REQUEST['localename'])) {
  2818.                 $localename $this->clean($_REQUEST['localename']"string" );
  2819.             else {
  2820.                 $localename "";
  2821.             }
  2822.             if(isset($_REQUEST['localetag'])) {
  2823.                 $localetag $this->clean($_REQUEST['localetag']"string" );
  2824.             else {
  2825.                 $localetag "";
  2826.             }
  2827.             if(isset($_REQUEST['writingmode'])) {
  2828.                 $writingmode =
  2829.                     $this->clean($_REQUEST['writingmode']"string" );
  2830.             else {
  2831.                 $writingmode "";
  2832.             }
  2833.             if(isset($_REQUEST['selectlocale'])) {
  2834.                 $select_locale =
  2835.                     $this->clean($_REQUEST['selectlocale']"string" );
  2836.             else {
  2837.                 $select_locale "";
  2838.             }
  2839.  
  2840.             switch($_REQUEST['arg'])
  2841.             {
  2842.                 case "addlocale":
  2843.                     $this->localeModel->addLocale(
  2844.                         $localename$localetag$writingmode);
  2845.                     $this->localeModel->extractMergeLocales();
  2846.                     $data["LOCALES"$this->localeModel->getLocaleList();
  2847.                     $data['LOCALE_NAMES'][$localetag$localename;
  2848.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2849.                         tl('admin_controller_locale_added')."</h1>')";
  2850.                 break;
  2851.  
  2852.                 case "deletelocale":
  2853.  
  2854.                     if(!in_array($select_locale$locale_ids)) {
  2855.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2856.                             tl('admin_controller_localename_doesnt_exists').
  2857.                             "</h1>')";
  2858.                         return $data;
  2859.                     }
  2860.                     $this->localeModel->deleteLocale($select_locale);
  2861.                     $data["LOCALES"$this->localeModel->getLocaleList();
  2862.                     unset($data['LOCALE_NAMES'][$select_locale]);
  2863.  
  2864.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2865.                         tl('admin_controller_localename_deleted')."</h1>')";
  2866.                 break;
  2867.  
  2868.                 case "editlocale":
  2869.                     if(!isset($select_locale)) break;
  2870.                     $data["leftorright"=
  2871.                         (getLocaleDirection(== 'ltr'"right""left";
  2872.                     $data["ELEMENT""editlocalesElement";
  2873.                     $data['STATIC_PAGES'][-1]=
  2874.                         tl('admin_controller_select_staticpages');
  2875.                     $data['STATIC_PAGES'+=
  2876.                         $this->localeModel->getStaticPageList($select_locale);
  2877.                     $data['CURRENT_LOCALE_NAME'=
  2878.                         $data['LOCALE_NAMES'][$select_locale];
  2879.                     $data['CURRENT_LOCALE_TAG'$select_locale;
  2880.                     $tmp_pages $data['STATIC_PAGES'];
  2881.                     array_shift($tmp_pages);
  2882.                     $page_keys array_keys($tmp_pages);
  2883.                     if(isset($_REQUEST['static_page']&&
  2884.                         in_array($_REQUEST['static_page']$page_keys)) {
  2885.                         $data["ELEMENT""editstaticElement";
  2886.                         $data['STATIC_PAGE'$_REQUEST['static_page'];
  2887.                         if(isset($_REQUEST['PAGE_DATA'])) {
  2888.                             $this->localeModel->setStaticPage(
  2889.                                 $_REQUEST['static_page'],
  2890.                                 $data['CURRENT_LOCALE_TAG'],
  2891.                                 $_REQUEST['PAGE_DATA']);
  2892.                             $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2893.                                 tl('admin_controller_staticpage_updated').
  2894.                                 "</h1>')";
  2895.                         }
  2896.                         $data['PAGE_NAME'=
  2897.                             $data['STATIC_PAGES'][$data['STATIC_PAGE']];
  2898.                         $data['PAGE_DATA'=
  2899.                             $this->localeModel->getStaticPage(
  2900.                                 $_REQUEST['static_page'],
  2901.                                 $data['CURRENT_LOCALE_TAG']);
  2902.                         /*since page data can contain tags we clean it
  2903.                           htmlentities it just before displaying*/
  2904.                         $data['PAGE_DATA'$this->clean($data['PAGE_DATA'],
  2905.                             "string");
  2906.                         break;
  2907.                     }
  2908.                     $data['SCRIPT'.= "selectPage = elt('static-pages');".
  2909.                         "selectPage.onchange = submitStaticPageForm;";
  2910.                     if(isset($_REQUEST['STRINGS'])) {
  2911.                         $safe_strings array();
  2912.                         foreach($_REQUEST['STRINGS'as $key => $value{
  2913.                             $clean_key $this->clean($key"string" );
  2914.                             $clean_value $this->clean($value"string" );
  2915.                             $safe_strings[$clean_key$clean_value;
  2916.                         }
  2917.                         $this->localeModel->updateStringData(
  2918.                             $select_locale$safe_strings);
  2919.                         $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  2920.                             tl('admin_controller_localestrings_updated').
  2921.                             "</h1>')";
  2922.                     else {
  2923.                         $this->localeModel->extractMergeLocales();
  2924.                     }
  2925.                     $data['STRINGS'=
  2926.                         $this->localeModel->getStringData($select_locale);
  2927.                     $data['DEFAULT_STRINGS'=
  2928.                         $this->localeModel->getStringData(DEFAULT_LOCALE);
  2929.                 break;
  2930.             }
  2931.         }
  2932.         return $data;
  2933.     }
  2934.  
  2935.     /**
  2936.      * Checks to see if the current machine has php configured in a way
  2937.      * Yioop! can run.
  2938.      *
  2939.      * @return string a message indicatign which required and optional
  2940.      *       components are missing; or "Passed" if nothing missing.
  2941.      */
  2942.      function systemCheck()
  2943.      {
  2944.         $required_items array(
  2945.             array("name" => "Multi-Curl",
  2946.                 "check"=>"curl_multi_init""type"=>"function"),
  2947.             array("name" => "GD Graphics Library",
  2948.                 "check"=>"imagecreate""type"=>"function"),
  2949.             array("name" => "SQLite3 Library",
  2950.                 "check"=>"SQLite3|PDO""type"=>"class"),
  2951.             array("name" => "Multibyte Character Library",
  2952.                 "check"=>"mb_internal_encoding""type"=>"function"),
  2953.         );
  2954.         $optional_items array(
  2955.          /* as an example of what this array could contain...
  2956.             array("name" => "Memcache", "check" => "Memcache",
  2957.                 "type"=> "class"), */
  2958.         );
  2959.  
  2960.         $missing_required "";
  2961.         $comma "";
  2962.         foreach($required_items as $item{
  2963.             $check_function $item["type"]."_exists";
  2964.             $check_parts explode("|"$item["check"]);
  2965.             $check_flag true;
  2966.             foreach($check_parts as $check{
  2967.                 if($check_function($check)) {
  2968.                     $check_flag false;
  2969.                 }
  2970.             }
  2971.             if($check_flag{
  2972.                 $missing_required .= $comma.$item["name"];
  2973.                 $comma ", ";
  2974.             }
  2975.         }
  2976.         if(!defined('PHP_VERSION_ID'|| PHP_VERSION_ID 50300{
  2977.             $missing_required .= $comma.tl("admin_controller_php_version");
  2978.             $comma ", ";
  2979.         }
  2980.  
  2981.         $out "";
  2982.         $br "";
  2983.  
  2984.         if(!is_writable(BASE_DIR."/configs/config.php")) {
  2985.             $out .= tl('admin_controller_no_write_config_php');
  2986.             $br "<br />";
  2987.         }
  2988.  
  2989.         if(defined(WORK_DIRECTORY&& !is_writable(WORK_DIRECTORY)) {
  2990.             $out .= $brtl('admin_controller_no_write_work_dir');
  2991.             $br "<br />";
  2992.         }
  2993.  
  2994.         if(intval(ini_get("post_max_size")) 2{
  2995.             $out .= $brtl('admin_controller_post_size_small');
  2996.             $br "<br />";
  2997.         }
  2998.  
  2999.         if($missing_required != ""{
  3000.             $out .= $br.
  3001.                 tl('admin_controller_missing_required'$missing_required);
  3002.             $br "<br />";
  3003.         }
  3004.  
  3005.         $missing_optional "";
  3006.         $comma "";
  3007.         foreach($optional_items as $item{
  3008.             $check_function $item["type"]."_exists";
  3009.             $check_parts explode("|"$item["check"]);
  3010.             $check_flag true;
  3011.             foreach($check_parts as $check{
  3012.                 if($check_function($check)) {
  3013.                     $check_flag false;
  3014.                 }
  3015.             }
  3016.             if($check_flag{
  3017.                 $missing_optional .= $comma.$item["name"];
  3018.                 $comma ", ";
  3019.             }
  3020.         }
  3021.  
  3022.         if($missing_optional != ""{
  3023.             $out .= $br.
  3024.                 tl('admin_controller_missing_optional'$missing_optional);
  3025.             $br "<br />";
  3026.         }
  3027.  
  3028.         if($out == ""{
  3029.             $out tl('admin_controller_check_passed');
  3030.         else {
  3031.             $out "<span class='red'>$out</span>";
  3032.         }
  3033.         if(file_exists(BASE_DIR."/configs/local_config.php")) {
  3034.             $out .= "<br />".tl('admin_controller_using_local_config');
  3035.         }
  3036.  
  3037.         return $out;
  3038.  
  3039.      }
  3040.  
  3041.     /**
  3042.      * Handles admin request related to the search sources activity
  3043.       *
  3044.      * The search sources activity allows a user to add/delete search sources
  3045.      * for video and news, it also allows a user to control which subsearches
  3046.      * appear on the SearchView page
  3047.      *
  3048.      * @return array $data info about current search sources, and current
  3049.      *       sub-searches
  3050.      */
  3051.     function searchSources()
  3052.     {
  3053.         $possible_arguments array("addsource""deletesource",
  3054.             "addsubsearch""deletesubsearch");
  3055.  
  3056.         $data array();
  3057.         $data["ELEMENT""searchsourcesElement";
  3058.         $data['SCRIPT'"";
  3059.         $data['SOURCE_TYPES'array(-=> tl('admin_controller_media_kind'),
  3060.             "video" => tl('admin_controller_video'),
  3061.             "rss" => tl('admin_controller_rss_feed'));
  3062.         $source_type_flag false;
  3063.         if(isset($_REQUEST['sourcetype']&&
  3064.             in_array($_REQUEST['sourcetype'],
  3065.             array_keys($data['SOURCE_TYPES']))) {
  3066.             $data['SOURCE_TYPE'$_REQUEST['sourcetype'];
  3067.             $source_type_flag true;
  3068.         else {
  3069.             $data['SOURCE_TYPE'= -1;
  3070.         }
  3071.         $machine_urls $this->machineModel->getQueueServerUrls();
  3072.         $search_lists $this->crawlModel->getCrawlList(falsetrue,
  3073.             $machine_urls);
  3074.         $data["SEARCH_LISTS"array(-=>
  3075.             tl('admin_controller_sources_indexes'));
  3076.         foreach($search_lists as $item{
  3077.             $data["SEARCH_LISTS"]["i:".$item["CRAWL_TIME"]] =
  3078.                 $item["DESCRIPTION"];
  3079.         }
  3080.         $search_lists=  $this->crawlModel->getMixList();
  3081.         foreach($search_lists as $item{
  3082.             $data["SEARCH_LISTS"]["m:".$item["MIX_TIMESTAMP"]] =
  3083.                 $item["MIX_NAME"];
  3084.         }
  3085.         $n NUM_RESULTS_PER_PAGE;
  3086.         $data['PER_PAGE'=
  3087.             array($n => $n2*$n => 2*$n5*$n=> 5*$n10*$n=>10*$n);
  3088.         if(isset($_REQUEST['perpage']&&
  3089.             in_array($_REQUEST['perpage']array_keys($data['PER_PAGE']))) {
  3090.             $data['PER_PAGE_SELECTED'$_REQUEST['perpage'];
  3091.         else {
  3092.             $data['PER_PAGE_SELECTED'NUM_RESULTS_PER_PAGE;
  3093.         }
  3094.         $locales $this->localeModel->getLocaleList();
  3095.         $data["LANGUAGES"array();
  3096.         foreach($locales as $locale{
  3097.             $data["LANGUAGES"][$locale['LOCALE_TAG']] $locale['LOCALE_NAME'];
  3098.         }
  3099.         if(isset($_REQUEST['sourcelocaletag']&&
  3100.             in_array($_REQUEST['sourcelocaletag'],
  3101.                 array_keys($data["LANGUAGES"]))) {
  3102.             $data['SOURCE_LOCALE_TAG'=
  3103.                 $_REQUEST['sourcelocaletag'];
  3104.         else {
  3105.             $data['SOURCE_LOCALE_TAG'DEFAULT_LOCALE;
  3106.         }
  3107.  
  3108.         if(isset($_REQUEST['arg']&&
  3109.             in_array($_REQUEST['arg']$possible_arguments)) {
  3110.             switch($_REQUEST['arg'])
  3111.             {
  3112.                 case "addsource":
  3113.                     if(!$source_type_flagbreak;
  3114.                     $must_have array("sourcename""sourcetype",
  3115.                         'sourceurl');
  3116.                     $to_clean array_merge($must_have,
  3117.                         array('sourcethumbnail','sourcelocaletag'));
  3118.                     foreach ($to_clean as $clean_me{
  3119.                         $r[$clean_me(isset($_REQUEST[$clean_me])) ?
  3120.                             $this->clean($_REQUEST[$clean_me]"string" "";
  3121.                         if(in_array($clean_me$must_have&&
  3122.                             $r[$clean_me== "" break 2;
  3123.                     }
  3124.                     $this->sourceModel->addMediaSource(
  3125.                         $r['sourcename']$r['sourcetype']$r['sourceurl'],
  3126.                         $r['sourcethumbnail']$r['sourcelocaletag']);
  3127.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  3128.                         tl('admin_controller_media_source_added').
  3129.                         "</h1>');";
  3130.                 break;
  3131.                 case "deletesource":
  3132.                     if(!isset($_REQUEST['ts'])) break;
  3133.                     $timestamp $this->clean($_REQUEST['ts']"string");
  3134.                     $this->sourceModel->deleteMediaSource($timestamp);
  3135.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  3136.                         tl('admin_controller_media_source_deleted').
  3137.                         "</h1>');";
  3138.                 break;
  3139.                 case "addsubsearch":
  3140.                     $to_clean array("foldername"'indexsource');
  3141.                     $must_have $to_clean;
  3142.                     foreach ($to_clean as $clean_me{
  3143.                         $r[$clean_me(isset($_REQUEST[$clean_me])) ?
  3144.                             $this->clean($_REQUEST[$clean_me]"string" "";
  3145.                         if(in_array($clean_me$must_have&&
  3146.                             $r[$clean_me== "" break 2;
  3147.                     }
  3148.                     $this->sourceModel->addSubsearch(
  3149.                         $r['foldername']$r['indexsource'],
  3150.                         $data['PER_PAGE_SELECTED']);
  3151.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  3152.                         tl('admin_controller_subsearch_added').
  3153.                         "</h1>');";
  3154.                 break;
  3155.                 case "deletesubsearch":
  3156.                     if(!isset($_REQUEST['fn'])) break;
  3157.                     $folder_name $this->clean($_REQUEST['fn']"string");
  3158.                     $this->sourceModel->deleteSubsearch($folder_name);
  3159.                     $data['SCRIPT'.= "doMessage('<h1 class=\"red\" >".
  3160.                         tl('admin_controller_subsearch_deleted').
  3161.                         "</h1>');";
  3162.                 break;
  3163.             }
  3164.         }
  3165.         $data["MEDIA_SOURCES"$this->sourceModel->getMediaSources();
  3166.         $subsearches $this->sourceModel->getSubsearches();
  3167.         $data["SUBSEARCHES"array();
  3168.         foreach($subsearches as $search{
  3169.             if(isset($data["SEARCH_LISTS"][$search['INDEX_IDENTIFIER']])) {
  3170.                 $data["SUBSEARCHES"][$search;
  3171.             else {
  3172.                 $this->sourceModel->deleteSubsearch($search["FOLDER_NAME"]);
  3173.             }
  3174.         }
  3175.         $data['SCRIPT'.= "source_type = elt('source-type');".
  3176.             "source_type.onchange = switchSourceType;".
  3177.             "switchSourceType()";
  3178.         return $data;
  3179.     }
  3180.  
  3181.     /**
  3182.      * Responsible for handling admin request related to the configure activity
  3183.      *
  3184.      * The configure activity allows a user to set the work directory for
  3185.      * storing data local to this SeekQuarry/Yioop instance. It also allows one
  3186.      * to set the default language of the installation, dbms info, robot info,
  3187.      * test info, as well as which machine acts as the queue server.
  3188.      *
  3189.      * @return array $data fields for available language, dbms, etc as well as
  3190.      *       results of processing sub activity if any
  3191.      */
  3192.     function configure()
  3193.     {
  3194.         $data array();
  3195.         $profile array();
  3196.  
  3197.         $data['SYSTEM_CHECK'$this->systemCheck();
  3198.         $languages $this->localeModel->getLocaleList();
  3199.         foreach($languages as $language{
  3200.             $data['LANGUAGES'][$language['LOCALE_TAG']] =
  3201.                 $language['LOCALE_NAME'];
  3202.         }
  3203.         if(isset($_REQUEST['lang']&& $_REQUEST['lang']{
  3204.             $data['lang'$this->clean($_REQUEST['lang']"string");
  3205.             $profile['DEFAULT_LOCALE'$data['lang'];
  3206.             setLocaleObject($data['lang']);
  3207.         }
  3208.  
  3209.         $data["ELEMENT""configureElement";
  3210.         $data['SCRIPT'"";
  3211.  
  3212.         $data['PROFILE'false;
  3213.         if(isset($_REQUEST['WORK_DIRECTORY']|| (defined('WORK_DIRECTORY'&&
  3214.             defined('