diff --git a/index.php b/index.php index 88bf4f3d8..42f488338 100644 --- a/index.php +++ b/index.php @@ -55,7 +55,8 @@ function passthruYioopRequest() return true; } /** - * + * Used to process the command line arguments to yioop when run in CLI + * mode as its own web server */ function processCommandLine() { @@ -205,7 +206,7 @@ if ($web_site->isCli()) { }); if (file_exists(C\LOG_DIR)) { // don't use logging until after log directory built on first run - $web_site->useMiddleware(function () use ($web_site) + $web_site->middleware(function () use ($web_site) { $lock_file = L\CrawlDaemon::getLockFileName("index", ''); /* in first run case, lock file won't be created by outer diff --git a/src/configs/Config.php b/src/configs/Config.php index f6a2eb0cc..8a5b80daa 100755 --- a/src/configs/Config.php +++ b/src/configs/Config.php @@ -1069,7 +1069,5 @@ nsconddefine('AD_DATE_FORMAT','Y-m-d'); nsconddefine('AD_LOGO','resources/adv-logo.png'); /** sentence compression enabled or not*/ nsconddefine('SENTENCE_COMPRESSION_ENABLED', false); -/** Define cipher to be used in AES */ -nsconddefine('AES_256_CBC', 'aes-256-cbc'); /** The number of rows to be used in bulk insert from Lexicon */ nsconddefine('NUM_LEX_BULK_INSERTS',100000); diff --git a/src/controllers/Controller.php b/src/controllers/Controller.php index 6a7c3892e..5870f97ab 100755 --- a/src/controllers/Controller.php +++ b/src/controllers/Controller.php @@ -77,7 +77,10 @@ function e($text) abstract class Controller { /** - * @var object + * Stores a reference to the web server when Yioop runs in CLI mode, + * it acts as request router in non-CLI mode. + * In CLI, mode it is useful for caching files in RAM as they are read + * @var seekquarry\yioop\library\WebSite */ public $web_site; /** @@ -119,7 +122,10 @@ abstract class Controller /** * Sets up component activities, instance array, and plugins. * - * @param array $web_site + * @param seekquarry\yioop\library\WebSite $web_site is the web server + * when Yioop runs in CLI mode, it acts as request router in non-CLI + * mode. In CLI, mode it is useful for caching files in RAM as they + * are read */ public function __construct($web_site = null) { diff --git a/src/controllers/FetchController.php b/src/controllers/FetchController.php index 0846e18b0..e1f51d7b1 100755 --- a/src/controllers/FetchController.php +++ b/src/controllers/FetchController.php @@ -277,8 +277,7 @@ class FetchController extends Controller implements CrawlConstants we only set crawl time. Other data such as allowed sites should come from index. */ - $this->model("crawl")->sendStartCrawlMessage($crawl_params, - null, null); + $this->model("crawl")->sendStartCrawlMessage($crawl_params); } } } diff --git a/src/controllers/RegisterController.php b/src/controllers/RegisterController.php index ad233756c..ed80ab015 100755 --- a/src/controllers/RegisterController.php +++ b/src/controllers/RegisterController.php @@ -118,6 +118,11 @@ class RegisterController extends Controller implements CrawlConstants /** * Besides invoking the base controller, sets up in field variables * the captcha and recovery question and possible answers. + * + * @param seekquarry\yioop\library\WebSite $web_site is the web server + * when Yioop runs in CLI mode, it acts as request router in non-CLI + * mode. In CLI, mode it is useful for caching files in RAM as they + * are read */ public function __construct($web_site = null) { diff --git a/src/controllers/SearchController.php b/src/controllers/SearchController.php index e64726e22..89f718ad8 100755 --- a/src/controllers/SearchController.php +++ b/src/controllers/SearchController.php @@ -69,6 +69,11 @@ class SearchController extends Controller implements CrawlConstants * In addition to calling the base class' constructor, set up * FileCache or Memcache objects if we're configured to do query * caching + * + * @param seekquarry\yioop\library\WebSite $web_site is the web server + * when Yioop runs in CLI mode, it acts as request router in non-CLI + * mode. In CLI, mode it is useful for caching files in RAM as they + * are read */ public function __construct($web_site = null) { diff --git a/src/controllers/components/ChatbotComponent.php b/src/controllers/components/ChatbotComponent.php index 06e868e2b..1255cd5a3 100644 --- a/src/controllers/components/ChatbotComponent.php +++ b/src/controllers/components/ChatbotComponent.php @@ -44,6 +44,9 @@ use seekquarry\yioop\library\PhraseParser; */ class ChatbotComponent extends Component { + /** + * @return array $data + */ public function botStory() { $parent = $this->parent; @@ -78,7 +81,6 @@ class ChatbotComponent extends Component $data['SCRIPT'] .= "\nswitchTab('createexpressionstab',". "'createintenttab', 'createentitytab');\n"; } - $data['MESSAGE'] = ""; $data['INTENT_LISTS']=""; $data["CURRENT_STORY"] = [ diff --git a/src/controllers/components/CrawlComponent.php b/src/controllers/components/CrawlComponent.php index 4f26da9a2..248b496e7 100644 --- a/src/controllers/components/CrawlComponent.php +++ b/src/controllers/components/CrawlComponent.php @@ -275,6 +275,8 @@ class CrawlComponent extends Component implements CrawlConstants * @param array& $data an array of info to supply to AdminView * @param array $machine_urls string urls of machines managed by this * Yioop name server on which to perform the crawl + * @param array $request_fields if start crawl fails this is a list of + * request fields to preserve in the redirect message * @param array $seed_info allowed, disallowed, seed urls, etc to use in * crawl */ @@ -1587,7 +1589,7 @@ class CrawlComponent extends Component implements CrawlConstants $locale_tag = $data['CURRENT_LOCALE_TAG']; $data['CURRENT_LOCALE_NAME'] = $locale_model->getLocaleName($locale_tag); - $data['ENTITIES'] = $entity_model->viewall($locale_tag); + $data['ENTITIES'] = $entity_model->viewAll($locale_tag); $data['NUM_ENTITIES_SHOW'] = 100; $data['TOTAL_ENTITIES'] = count($data['ENTITIES']); $data['LIMIT'] = (isset($_REQUEST['limit'])) ? @@ -1601,7 +1603,7 @@ class CrawlComponent extends Component implements CrawlConstants $data['entity_default'] = [tl('pageoptions_element_actions')]; return $data; } - /**$data['SCRIPT'] .= 'entities = ' . $data['ENTITIES'] + /** * Handles admin request related to the Scrapers activity * * This activity allows a user to specify the configuration for the @@ -2139,7 +2141,10 @@ class CrawlComponent extends Component implements CrawlConstants "switchSourceType()"; return $data; } - + /** + * + * @param array $data + */ public function addNamedEntites($data) { $parent = $this->parent; @@ -2148,7 +2153,7 @@ class CrawlComponent extends Component implements CrawlConstants if (isset($_REQUEST['ENTITY_NAME'])) { $entity = $_REQUEST['ENTITY_NAME']; $locale = $_REQUEST['LOCALE_TAG']; - $entity_model->addentity($entity, $locale); + $entity_model->addEntity($entity, $locale); } else if (isset($_FILES['ENTITY_FILE'])) { $file_name = $_FILES['ENTITY_FILE']['tmp_name']; if (!empty($file_name)) { @@ -2163,7 +2168,7 @@ class CrawlComponent extends Component implements CrawlConstants isset($_REQUEST['LOCALE_TAG'])) { $entity = $_REQUEST['value']; $locale = $_REQUEST['LOCALE_TAG']; - $entity_model->deleteentity($entity, $locale); + $entity_model->deleteEntity($entity, $locale); } break; } diff --git a/src/controllers/components/SocialComponent.php b/src/controllers/components/SocialComponent.php index 84b947170..84cdf46d7 100644 --- a/src/controllers/components/SocialComponent.php +++ b/src/controllers/components/SocialComponent.php @@ -121,8 +121,8 @@ class SocialComponent extends Component implements CrawlConstants C\ONE_MONTH => tl('social_component_one_month'), ]; $data['ENCRYPTION_CODES'] = [ - true => tl('social_component_encryption_enable'), - false => tl('social_component_encryption_disable'), + 1 => tl('social_component_encryption_enable'), + 0 => tl('social_component_encryption_disable'), ]; $search_array = []; $default_group = ["name" => "","id" => "", "owner" =>"", @@ -359,7 +359,8 @@ class SocialComponent extends Component implements CrawlConstants "register" => ["REGISTER_CODES", C\REQUEST_JOIN], "vote_access" => ["VOTING_CODES", C\NON_VOTING_GROUP], - "post_lifetime" => ["VOTING_CODES", C\FOREVER] + "post_lifetime" => ["POST_LIFETIMES", C\FOREVER], + "encryption" => ["ENCRYPTION_CODES", 0] ]; foreach ($group_fields as $field => $info) { if (!isset($_REQUEST[$field]) || @@ -835,8 +836,8 @@ class SocialComponent extends Component implements CrawlConstants $group_model->isGroupEncrypted($group_id)) { // Decrypt thread's title $key = $group_model->getGroupKey($group_id); - $decrypted_item_name = $group_model->cipher( - $item_name, $key, 1); + $decrypted_item_name = $group_model->decrypt( + $item_name, $key); $data['STATISTICS'][$field][$period][ $decrypted_item_name] = $item_data; unset($data['STATISTICS'][$field][$period][ diff --git a/src/error.php b/src/error.php index eddb95711..dd67c7915 100755 --- a/src/error.php +++ b/src/error.php @@ -35,7 +35,9 @@ namespace seekquarry\yioop; use seekquarry\yioop\library as L; use seekquarry\yioop\controllers\StaticController; - +/** + * Used to handle rquest errors in non-cli, non-webserver redirect case + */ function webError() { if (!isset($_REQUEST['p']) || diff --git a/src/library/Cipher.php b/src/library/Cipher.php deleted file mode 100644 index 3e6d1e557..000000000 --- a/src/library/Cipher.php +++ /dev/null @@ -1,150 +0,0 @@ -<?php -namespace seekquarry\yioop\library; - -use seekquarry\yioop\configs as C; -/** - * Encryption class for adding encryption and - * decryption methods. This class uses AES as - * a symmetric encryption algorithm. - */ -class Cipher -{ - protected $key; - protected $data; - protected $mode; - protected $IV; - /** - * Sets up data with it's key and mode of operation used by - * the algorithm - * @param string $data data to encrypt - * @param string $key key used for encryption and decryption - * @param string $mode mode of operation used by the algorithm - */ - function __construct($data = null, $mode = null, $key = null) - { - $this->setData($data); - $this->setMode($mode); - $this->setKey($key); - $this->setIV(""); - } - /** - * Sets the data. Data is the value that is encrypted/decrypted. - * @param string $data data to set - */ - public function setData($data) - { - $this->data = $data; - } - /** - * Returns the data that is being encrypted/decrypted - * @return string $data - */ - protected function getData() - { - return $this->data; - } - /** - * Sets the key. Key is unique for each data value. - * @param string $key set key for encryption/decryption - */ - public function setKey($key) - { - $this->key = $key; - } - /** - * Returns the key that is needed to encrypt/decrypt - * @return string $key - */ - protected function getKey() - { - return $this->key; - } - /** - * Sets the mode. Mode means mode of operation used by algorithm - * @param string $mode mode to set - */ - public function setMode($mode) - { - $this->mode = $mode; - } - /** - * Returns the mode that is used by the algorithm - * @return string $mode - */ - protected function getMode() - { - return $this->mode; - } - /** - * checks if parameters are valid - * @return boolean - */ - public function validateParams() - { - if ($this->data != null && - $this->key != null) { - return true; - } else { - return false; - } - } - /** - * Sets initialization vector - * @return string $IV initialization vector - */ - public function setIV($IV) - { - $this->IV = $IV; - } - /** - * Returns the initialization vector - * @return string $IV initialization vector - */ - protected function getIV() - { - if ($this->IV == "") { - $this->IV = openssl_random_pseudo_bytes( - openssl_cipher_iv_length(C\AES_256_CBC)); - } - return $this->IV; - } - /** - * Encrypts the data using the associated key and some random data - * @return string $data encrypted data - * @throws Exception if the parameters are not valid - */ - public function encrypt() - { - if ($this->validateParams()) { - $encrypted = openssl_encrypt($this->getData(), $this->getMode(), - $this->getKey(), 0, $this->getIV()); - // append value of $IV to the encrypted data with a separator - // so this can be used during decryption - $encrypted = $this->getIV() . str_repeat("0", 10) . $encrypted; - return $encrypted; - } else { - throw new Exception('Invalid params!'); - } - } - /** - * Decrypts the data using the associated key - * @return string $data decrypted data - * @throws Exception if the parameters are not valid - */ - public function decrypt() - { - if ($this->validateParams()) { - $pos = strpos($this->getData(), str_repeat("0", 10)); - if ($pos !== false) { - $data_parts = explode(str_repeat("0", 10), $this->getData()); - } else { - throw new Exception('Could not split keys'); - } - $decrypted = openssl_decrypt($data_parts[1], $this->getMode(), - $this->getKey(), 0, $data_parts[0]); - return $decrypted; - } else { - throw new Exception('Invalid params!'); - } - } -} diff --git a/src/library/CrawlDaemon.php b/src/library/CrawlDaemon.php index ebd65c44b..137b7bea8 100644 --- a/src/library/CrawlDaemon.php +++ b/src/library/CrawlDaemon.php @@ -142,7 +142,7 @@ class CrawlDaemon implements CrawlConstants * both queue server processes (Indexer and Scheduler) to use the * same lock file. If exit is >=3 or <= -3 then doesn't check lock * to see if already running before starting - * @param string $use_message echo'd if incorrect parameters sent + * @param string $use_message echo'd if incorrect parameters sent */ public static function init($argv, $name, $exit_type = 1, $use_message = "") diff --git a/src/library/Utility.php b/src/library/Utility.php index 2856fa443..a814a047f 100755 --- a/src/library/Utility.php +++ b/src/library/Utility.php @@ -2066,8 +2066,7 @@ function arraytoCsv($arr) fclose($fp); return rtrim($data, "\n"); } -/* - * +/** * Computes a Unix-style diff of two strings. That is it only * outputs lines which disagree between the two strings. It outputs +line * if a line occurs in the second but not first string and -line if a @@ -2076,6 +2075,7 @@ function arraytoCsv($arr) * @param string $data1 first string to compare * @param string $data2 second string to compare * @param bool $html whether to output html highlighting + * @return string respresenting info about where $data1 and $data2 don't match */ function diff($data1, $data2, $html = false) { diff --git a/src/library/VersionManager.php b/src/library/VersionManager.php index 4fcbf4752..dad6b49e0 100644 --- a/src/library/VersionManager.php +++ b/src/library/VersionManager.php @@ -539,7 +539,7 @@ class VersionManager /** * Gets the most recent version timestamp of a version in the repository * that is less than or equal to the searched for timestamp. - * @param int $timestamp want to find the version in the repository + * @param int $search_timestamp want to find the version in the repository * closest to, but not exceeding this value. * @param bool $lock whether or not a lock should be obtained before * carrying out the operation @@ -634,7 +634,7 @@ class VersionManager * * @param int $start_time look for timestamps in repository above or equal * this value - * @param int $end_timelook for timestamps in repository below or equal + * @param int $end_time look for timestamps in repository below or equal * this value * @param bool $lock whether or not a lock should be obtained before * carrying out the operation diff --git a/src/library/WebSite.php b/src/library/WebSite.php index 8a50f4178..5baa8e0f9 100644 --- a/src/library/WebSite.php +++ b/src/library/WebSite.php @@ -49,36 +49,131 @@ use seekquarry\yioop\configs as C; */ class WebSite { + /* + Field component name constants used in input or output stream array + */ const CONNECTION = 0; const DATA = 1; const MODIFIED_TIME = 2; const CONTEXT = 3; - const IS_FILE = 4; - const REQUEST_HEAD = 5; + const REQUEST_HEAD = 4; + /* + CONNECT OPTIONS and the longest named HTTP method. It has length 7 + */ const LEN_LONGEST_HTTP_METHOD = 7; + /** + * Keys of stream that won't disappear (for example the server socket) + * @var array + */ public $immortal_stream_keys = []; + /** + * Used to determine portion of path to ignore + * when checking if a route matches against the current request. + * @var string + */ public $base_path; + /** + * Default values to write into $_SERVER before processing a connection + * request (in CLI mode) + * @var array + */ protected $default_server_globals; + /** + * Associative array http_method => array of how to handle paths for + * that method + * @var array + */ protected $routes = ["CONNECT" => [], "DELETE" => [], "ERROR" => [], "GET" => [], "HEAD" => [], "OPTIONS" => [], "POST" => [], "PUT" => [], "TRACE" => []]; + /** + * Default values used to set up session variables + * @var array + */ protected $session_configs = [ 'cookie_lifetime' => '0', 'cookie_path' => '/', 'cookie_domain' => '', 'cookie_secure' => '', 'cookie_httponly' => '', 'name' => 'PHPSESSID', 'save_path' => '']; + /** + * Array of all connection streams for which we are receiving data from + * client together with associated stream info + * @var array + */ protected $in_streams = []; + /** + * Array of all connection streams for which we are writing data to a + * client together with associated stream info + * @var array + */ protected $out_streams = []; + /** + * Middle ware callbacks to run when processing a request + * @var array + */ protected $middle_wares = []; + /** + * Filename of currently requested script + * @var array + */ protected $request_script; + /** + * Used to store session data in ram for all currently active sessions + * @var array + */ protected $sessions = []; + /** + * Keeps an ordering of currently actively sessions so if run out of + * memory know who to evict + * @var array + */ protected $session_queue = []; + /** + * Timer function callbacks that have been declared + * @var array + */ protected $timers = []; + /** + * Priority queue of which timers are about to go off + * @var \SplMinHeap + */ protected $timer_alarms; + /** + * Cookie name value of the currently active request's session + * @var string + */ protected $current_session = ""; + /** + * List of HTTP methods for which routes have been declared + * @var array + */ protected $http_methods; + /** + * Holds the header portion so far of the HTTP response + * @var string + */ protected $header_data; + /** + * Whether the current response already has declared a Content-Type. + * If not, WebSite will default to adding a text/html header + * @var bool + */ protected $content_type; + /** + * Used to cache in RAM files which have been read or written by the + * fileGetContents or filePutContents + * @var array + */ protected $file_cache = ['MARKED' => [], 'UNMARKED' => [], 'PATH' => []]; + /** + * Whether this object is being run from teh command line with a listen() + * call or if it is being run under a web server and so only used for + * routing + * @var bool + */ protected $is_cli; + /** + * Whether https is being used + * @var bool + */ protected $is_secure = false; /** * Sets the base path used for determining request routes. Sets @@ -128,6 +223,8 @@ class WebSite * are a two element array with a route and a callback function and add * the appropriate route to a routing table. * + * @param string $method HTTP command to add $route_callack for in routing + * table * @param array $route_callback a two element array consisting of a routing * pattern and a callback function. * In the route pattern @@ -203,7 +300,7 @@ class WebSite * @param callable $callback function to be called before processing of * request */ - public function useMiddleware(callable $callback) + public function middleware(callable $callback) { $this->middle_wares[] = $callback; } @@ -458,7 +555,7 @@ class WebSite if ($this->isCli()) { if (!empty($this->file_cache['PATH'][$filename])) { /* - we are caching realpath which already has its own cache. + We are caching realpath which already has its own cache. realpath's cache though is based on time, ours is based on the marking algorithm. */ @@ -519,6 +616,7 @@ class WebSite * * @param string $filename name of file to write to persistent storages * @param string $data string of data to store in file + * @return int number of bytes written */ public function filePutContents($filename, $data) { @@ -712,6 +810,7 @@ class WebSite "MAX_REQUEST_LEN" => 10000000, "PATH" => $path, "SERVER_ADMIN" => "you@example.com", "SERVER_NAME" => "localhost", "SERVER_SIGNATURE" => "", + "USER" => empty($_SERVER['USER']) ? "" : $_SERVER['USER'], "SERVER_SOFTWARE" => "ATTO WEBSITE SERVER", ]; $original_address = $address; @@ -777,12 +876,33 @@ class WebSite stream_set_blocking($server, 0); $this->default_server_globals = array_merge($_SERVER, $default_server_globals, $server_globals); + $as_user = ""; + if (function_exists("posix_getuid") && + function_exists("posix_getpwuid") && + function_exists("posix_getpwnam") && + !empty($this->default_server_globals['USER'])) { + $uid = posix_getuid(); + $active_user_info = posix_getpwuid($uid); + if ($active_user_info['name'] != + $this->default_server_globals['USER']) { + $user_info = + posix_getpwnam($this->default_server_globals['USER']); + if (!empty($user_info['uid'])) { + posix_setuid($user_info['uid']); + $uid = posix_getuid(); + if ($uid == $user_info['uid']) { + $as_user = " running as user " . + $this->default_server_globals['USER']; + } + } + } + } $this->immortal_stream_keys[] = (int)$server; $this->in_streams = [self::CONNECTION => [(int)$server => $server], self::DATA => [""]]; $this->out_streams = [self::CONNECTION => [], self::DATA => []]; $excepts = null; - echo "SERVER listening at $address\n"; + echo "SERVER listening at $address $as_user\n"; $num_selected = 1; while (!$this->stop || $num_selected > 0) { if ($this->stop) { @@ -795,10 +915,10 @@ class WebSite $in_streams_with_data = $this->in_streams[self::CONNECTION]; } $out_streams_with_data = $this->out_streams[self::CONNECTION]; - if ($this->timer_alarms->isEmpty()) { - $timeout = null; - $micro_timeout = 0; - } else { + if ($this->timer_alarms->isEmpty()) { + $timeout = null; + $micro_timeout = 0; + } else { $next_alarm = $this->timer_alarms->top(); $pre_timeout = max(0, $next_alarm[0] - microtime(true)); $timeout = floor($pre_timeout); diff --git a/src/locale/hi/resources/Tokenizer.php b/src/locale/hi/resources/Tokenizer.php index f96ffb3cd..25aa2603f 100755 --- a/src/locale/hi/resources/Tokenizer.php +++ b/src/locale/hi/resources/Tokenizer.php @@ -45,35 +45,36 @@ class Tokenizer { /** * List of verb-like parts of speech that might appear in lexicon - * @array + * @var array */ public static $verb_phrases = ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "RB"]; /** * List of noun-like parts of speech that might appear in lexicon - * @array + * @var array */ public static $noun_phrases = ["NN", "NNS", "NNP", "NNPS", "DT"]; /** * List of adjective-like parts of speech that might appear in lexicon - * @array + * @var array */ public static $adjective_phrases = ["JJ", "JJR", "JJS"]; /** * List of postpositional-like parts of speech that might appear in lexicon - * @array + * @var array */ public static $postpositional_phrases = ["IN", "inj", "PREP", "proNN", "CONJ", "INT", "particle", "case", "PSP", "direct_DT", "PRP"]; - /* + /** * List of questions in Hindi + * @var array */ public static $questions = ["क्या", "कब", "कहा", "क्यों", "कौन", "जिसे", "जिसका", "कहाँ", "कहां"]; /** * Any unique identifier corresponding to the component of a triplet which * can be answered using a question answer list - * @string + * @var string */ public static $question_marker = "qqq"; /** @@ -175,11 +176,16 @@ class Tokenizer return self::tagUnknownWords($result); } /** - * This method tags the remaining words from the text. + * This method tags the remaining words in a partially tagged text array. + * + * @param array $partially_tagged_text term array representing a text + * passage. Each element in array is in turnan associative array + * [token => token_value, tag => tag_value (may be empty)] + * @return array text passage array where all empty tags now have values */ - public static function tagUnknownWords($partiallyTaggedText) + public static function tagUnknownWords($partially_tagged_text) { - $result = $partiallyTaggedText; + $result = $partially_tagged_text; $verbs = ["VBZ","VBD","VBN"]; $length = count($result); $previous = $result[0]; @@ -241,12 +247,10 @@ class Tokenizer $current["tag"] = "JJ"; $result[$i] = $current; } - if ($current["tag"] == "UNKNOWN") { $current["tag"] = "NN"; $result[$i] = $current; } - if ($previous["tag"] == "UNKNOWN"){ $previous["tag"] = "NN"; $result[$i-1] = $previous; @@ -330,6 +334,7 @@ class Tokenizer * "tag"=> part_of_speech_tag_for_term) * @param array $tree that consists of ["cur_node" => * current parse position in $tagged_phrase] + * @param int $index position in array to start from * @return array has fields * "cur_node" index of how far we parsed $tagged_phrase */ @@ -374,7 +379,7 @@ class Tokenizer } $tree_pp["cur_node"] = $cur_node; $tree_next = self::extractPostpositionPhrase($tagged_phrase, - $tree_pp, $index+1); + $tree_pp, $index + 1); $tree_pp = array_merge ($tree_pp, $tree_next); } $tree["cur_node"] = $tree_pp["cur_node"]; diff --git a/src/models/BotModel.php b/src/models/BotModel.php index 917ec1f5b..af9e46877 100644 --- a/src/models/BotModel.php +++ b/src/models/BotModel.php @@ -37,8 +37,7 @@ use seekquarry\yioop\library\Processors\ImageProcessor; /** For getLocaleTag*/ require_once __DIR__.'/../library/LocaleFunctions.php'; /** - * This class is used to handle - * database statements related to Bot User stories + * This class is used to handle database statements related to Bot User stories * * @author Harika Nukala */ @@ -62,8 +61,8 @@ class BotModel extends Model /** * Add an expression to the database using provided string * - * @param string $expression, the expression to be added - * @param int $intent_id, the provided intent + * @param string $expression the expression to be added + * @param int $intent_id the provided intent */ public function addExpression($expression, $intent_id) { @@ -74,13 +73,13 @@ class BotModel extends Model /** * Add an entity value to the database using provided string * - * @param int $expression_id, id of particular expression + * @param int $expression_id id of particular expression * @param int $intent_id id of intent under which expression belongs * @param int $entity_id id of entity name * @param string $entity_value value of entity */ public function addEntityValue($expression_id, $intent_id, - $entity_id, $entity_value){ + $entity_id, $entity_value){ $db = $this->db; $sql = "INSERT INTO INTENT_EXPRESSION_ENTITY(INTENT_ID, EXPRESSION_ID, ENTITY_ID, ENTITY_VALUE) VALUES (?, ?, ?, ?)"; @@ -143,10 +142,11 @@ class BotModel extends Model return $intent_id; } /** - * Get entity id associated with entityname and userid + * Get entity id associated with $entity_name and $user_id * - * @param string $entity_name to use to look up a entity_id - * @return int entity_id corresponding to the entityname. + * @param string $entity_name to use to look up an $entity_id + * @param string $user_id id of bot user + * @return int $entity_id corresponding to the $entity_name. */ public function getEntityId($entity_name, $user_id) { @@ -185,7 +185,8 @@ class BotModel extends Model * Get entity associated with entity_id * * @param int $entity_id to use to look up an entity - * @return array entity corresponding to the entityid. + * @return array associative array containing information about + * the entity that was looked up */ public function getEntity($entity_id) { @@ -276,36 +277,36 @@ class BotModel extends Model /** * Update an intent * - * @param array $obj, intent object to update + * @param array $intent intent object to update */ - public function updateIntent($obj) + public function updateIntent($intent) { - $intent_id = $obj['INTENT_ID']; + $intent_id = $intent['INTENT_ID']; $sql = "UPDATE INTENT SET "; $comma = ""; $params = []; - foreach ($obj as $field => $value) { - $sql .= "$comma $field=? "; + foreach ($intent as $field => $value) { + $sql .= "$comma $field = ? "; $comma = ","; $params[] = $value; } - $sql .= " WHERE INTENT_ID=?"; + $sql .= " WHERE INTENT_ID = ?"; $params[] = $intent_id; $this->db->execute($sql, $params); } /** * Update an entity * - * @param array $obj, entity object to update + * @param array $entity object to update */ - public function updateEntity($obj) + public function updateEntity($entity) { $db = $this->db; - $entity_id = $obj['ENTITY_ID']; + $entity_id = $entity['ENTITY_ID']; $sql = "UPDATE ENTITY SET "; - $comma =""; + $comma = ""; $params = []; - foreach ($obj as $field => $value) { + foreach ($entity as $field => $value) { $sql .= "$comma $field=? "; $comma = ","; $params[] = $value; @@ -313,13 +314,13 @@ class BotModel extends Model $sql .= " WHERE ENTITY_ID=?"; $params[] = $entity_id; $db->execute($sql, $params); - if($obj['ENTITY_VALUE'] != ""){ + if($entity['ENTITY_VALUE'] != ""){ $sql = "SELECT * FROM INTENT_EXPRESSION_ENTITY - WHERE ENTITY_ID = (?)". $db->limitOffset(1); + WHERE ENTITY_ID = (?) ". $db->limitOffset(1); $result = $db->execute($sql, [$entity_id]); $row = $db->fetchArray($result); - $this->addEntityValue($row['EXPRESSION_ID'], $row['INTENT_ID'] - , $entity_id , $obj['ENTITY_VALUE']); + $this->addEntityValue($row['EXPRESSION_ID'], $row['INTENT_ID'], + $entity_id , $entity['ENTITY_VALUE']); } } /** @@ -335,16 +336,16 @@ class BotModel extends Model /** * Update an expression * - * @param array $obj, expression object to update + * @param array $expression object to update */ - public function updateExpression($obj) + public function updateExpression($expression) { - $expression_id = $obj['EXPRESSION_ID']; + $expression_id = $expression['EXPRESSION_ID']; $sql = "UPDATE EXPRESSION SET "; $comma = ""; $params = []; - foreach ($obj as $field => $value) { - $sql .= "$comma $field=? "; + foreach ($expression as $field => $value) { + $sql .= "$comma $field = ?"; $comma = ","; $params[] = $value; } @@ -357,7 +358,8 @@ class BotModel extends Model * * @param int $expression_id to use to look up expression */ - public function deleteExpression($expression_id){ + public function deleteExpression($expression_id) + { $sql = "DELETE FROM EXPRESSION WHERE EXPRESSION_ID=?"; $this->db->execute($sql, [$expression_id]); } @@ -427,26 +429,26 @@ class BotModel extends Model */ public function whereCallback($args = null) { - $where=""; + $where = ""; list($query, $conditions, ) = $args; - if ($query=="INTENTS" || $query=="ENTITIES") { - $where = "USER_ID=". $_SESSION['USER_ID']; - } - else if($query=="EXPRESSIONS") { - $where = "I.INTENT_ID = E.INTENT_ID AND USER_ID=" - .$_SESSION['USER_ID']; - } - else if ($query=="ENTITY_VALUES"&&$conditions[0]) { - $where = "ENTITY_ID =".$conditions[0]; + if ($query == "INTENTS" || $query == "ENTITIES") { + $where = "USER_ID=" . $_SESSION['USER_ID']; + } else if ($query == "EXPRESSIONS") { + $where = "I.INTENT_ID = E.INTENT_ID AND USER_ID=" . + $_SESSION['USER_ID']; + } else if ($query == "ENTITY_VALUES" && $conditions[0]) { + $where = "ENTITY_ID =" . $conditions[0]; } return $where; } /** - * Add an intent to the database using provided string + * Add the frequency count of the number of occurences of a term + * in an intent to the BOT_TERM_FREQUENCY table * - * @param string $intent the intent to be added + * @param string $term to compute count for + * @param string $intent to count occurrences of term within */ - public function addTermFrequency($term,$intent) + public function addTermFrequency($term, $intent) { $db = $this->db; $sql = "SELECT FREQUENCY AS NUM FROM " . @@ -481,7 +483,8 @@ class BotModel extends Model * @param string $user_id to use to look up list of intents * @return array intents corresponding to the user_id. */ - public function getIntents($user_id){ + public function getIntents($user_id) + { $db = $this->db; $intents = []; $sql = "SELECT INTENT_ID,INTENT FROM INTENT WHERE USER_ID = (?) "; @@ -492,12 +495,15 @@ class BotModel extends Model return $intents; } /** - * Get list of intents having the term + * Gets count of the number of intents having the term * * @param string $term to use to look up intents - * @return int intent_count. + * @param string $intents ids of intents to look up to see if have term + * @return int the count of number intent within the listed intents that + * have the term */ - public function getIntentsWithTerm($term, $intents){ + public function getIntentsWithTerm($term, $intents) + { $db = $this->db; $sql = "SELECT COUNT(*) AS INTENT_COUNT FROM BOT_TERM_FREQUENCY WHERE TERM = (?) AND INTENT_ID IN (?)"; @@ -506,16 +512,17 @@ class BotModel extends Model return $row['INTENT_COUNT']; } /** - * Get frequency associated with term and intent + * Gets the number of occurences of a term within a given intent * - * @param string $term to use to look up frequency - * @param int $intent to use to look up frequency + * @param string $term to count occurences of + * @param int $intent to count within * @return int frequency corresponding to the term and intent. */ - public function getTermFrequency($term, $intent){ + public function getTermFrequency($term, $intent) + { $db = $this->db; $sql = "SELECT FREQUENCY FROM BOT_TERM_FREQUENCY - WHERE TERM = ? AND INTENT_ID = (?) "; + WHERE TERM = ? AND INTENT_ID = (?) "; $result = $db->execute($sql, [$term, $intent]); $row = $db->fetchArray($result); return $row['FREQUENCY']; @@ -525,7 +532,8 @@ class BotModel extends Model * * @param int $id to use to look up entity value */ - public function deleteEntityValue($id){ + public function deleteEntityValue($id) + { $sql = "DELETE FROM INTENT_EXPRESSION_ENTITY WHERE ID = ?"; $this->db->execute($sql, [$id]); } diff --git a/src/models/CrawlModel.php b/src/models/CrawlModel.php index 4bc2c0083..295c1ae8b 100755 --- a/src/models/CrawlModel.php +++ b/src/models/CrawlModel.php @@ -914,6 +914,11 @@ EOT; * @param array $seed_info what urls to crawl, etc as from the crawl.ini * file * @param array $machine_urls an array of urls of yioop queue servers + * @param int $has_queue_server if the value is greater than 0 the machine + * has a queue server on it. + * @param int $num_fetchers number of fetchers on machine to start. + * This parameter and $has_queue_server are used to start the daemons + * running on the machines if they aren't already running */ public function sendStartCrawlMessage($crawl_params, $seed_info = null, $machine_urls = null, $has_queue_server = 0, $num_fetchers = 0) diff --git a/src/models/EntityModel.php b/src/models/EntityModel.php index fe333cce2..070971961 100644 --- a/src/models/EntityModel.php +++ b/src/models/EntityModel.php @@ -110,7 +110,7 @@ class EntityModel extends Model * * @param string $locale locale to which the entities belong */ - public function viewall($locale) + public function viewAll($locale) { $entities = []; $db = $this->db; @@ -128,7 +128,7 @@ class EntityModel extends Model * @param string $entity entity to delete * @param string $locale locale to which the entities belong */ - public function deleteentity($entity, $locale) + public function deleteEntity($entity, $locale) { $db = $this->db; $sql = "DELETE FROM LEXICON WHERE TERM = ? AND LOCALE = ?"; diff --git a/src/models/GroupModel.php b/src/models/GroupModel.php index 6f4403e5e..2169f9835 100644 --- a/src/models/GroupModel.php +++ b/src/models/GroupModel.php @@ -33,7 +33,6 @@ namespace seekquarry\yioop\models; use seekquarry\yioop as B; use seekquarry\yioop\configs as C; use seekquarry\yioop\library as L; -use seekquarry\yioop\library\Cipher; use seekquarry\yioop\library\MediaConstants; use seekquarry\yioop\library\VersionManager; use seekquarry\yioop\library\WikiParser; @@ -242,6 +241,9 @@ class GroupModel extends Model implements MediaConstants * posts: NON_VOTING_GROUP, UP_VOTING_GROUP, UP_DOWN_VOTING_GROUP * @param int $post_lifetime specifies the time in seconds that posts should * live before they expire and are deleted + * @param int $encryption 0 means don't encrypt group, 1 means encrypt + * group + * @return int id of group added */ public function addGroup($group_name, $user_id, $register = C\REQUEST_JOIN, $member = C\GROUP_READ, $voting = C\NON_VOTING_GROUP, @@ -271,7 +273,7 @@ class GroupModel extends Model implements MediaConstants $db); ImpressionModel::initWithDb(C\PUBLIC_GROUP_ID, $last_id, C\GROUP_IMPRESSION, $db); - if ($this->isGroupEncrypted($last_id)) { + if ($encryption != 0) { $sql = "INSERT INTO TYPE_KEYS (TYPE_ID, KEY_NAME) VALUES (?, ?)"; $encrypt_key = openssl_random_pseudo_bytes(32); $private_db->execute($sql, [$last_id, $encrypt_key]); @@ -388,18 +390,17 @@ class GroupModel extends Model implements MediaConstants * Check whether group's encryption is enabled or not * * @param int $group_id to check for encryption value - * @return boolean isEncryptionEnabled value of encryption + * @return boolean whether thte group is encrypted or not */ public function isGroupEncrypted($group_id) { $db = $this->db; - $sql = "SELECT G.ENCRYPTION AS ENCRYPTION FROM ". - "GROUPS G WHERE G.GROUP_ID = ? "; + $sql = "SELECT ENCRYPTION FROM GROUPS WHERE GROUP_ID = ? "; $result = $db->execute($sql, [$group_id]); if (!$row = $db->fetchArray($result)) { return false; } - return $row['ENCRYPTION']; + return ($row['ENCRYPTION'] != 0); } /** * Delete a group from the database and any associated data in @@ -706,8 +707,8 @@ class GroupModel extends Model implements MediaConstants if ($this->isGroupEncrypted($row['GROUP_ID'])) { // Decrypt group's title and description $key = $this->getGroupKey($row['GROUP_ID']); - $row['TITLE'] = $this->cipher($row['TITLE'], $key, 1); - $row['DESCRIPTION'] = $this->cipher($row['DESCRIPTION'], $key, 1); + $row['TITLE'] = $this->decrypt($row['TITLE'], $key); + $row['DESCRIPTION'] = $this->decrypt($row['DESCRIPTION'], $key); } return $row; } @@ -786,8 +787,8 @@ class GroupModel extends Model implements MediaConstants } if ($this->isGroupEncrypted($group_id)) { $key = $this->getGroupKey($group_id); - $title = $this->cipher($title, $key); - $description = $this->cipher($description, $key); + $title = $this->encrypt($title, $key); + $description = $this->encrypt($description, $key); } $sql = "INSERT INTO GROUP_ITEM (PARENT_ID, GROUP_ID, USER_ID, TITLE, DESCRIPTION, PUBDATE, EDIT_DATE, TYPE) VALUES @@ -804,22 +805,42 @@ class GroupModel extends Model implements MediaConstants return $id; } /** - * Encrypts or decrypts data based on provided key. - * This is a wrapper function which in turn calls Cipher class. + * Encrypts data based on provided key. * - * @param string $data what data to encrypt/decrypt - * @param string $key what key to use to encrypt/decrypt - * @param int $type either encrypt(0) or decrypt(1) data + * @param string $data what data to encrypt + * @param string $key what key to use to encrypt + * @param string $cipher_method a cipher encrypt/decrypt method + * supported by OpenSSL + * @return string $out_data the encrypted string */ - public function cipher($data, $key, $type = 0) + public function encrypt($data, $key, $cipher_method = 'aes-256-cbc') { - $cipher = new Cipher($data, C\AES_256_CBC, $key); - if ($type == 0) { - $data = $cipher->encrypt(); - } else { - $data = $cipher->decrypt(); + $iv = openssl_random_pseudo_bytes( + openssl_cipher_iv_length($cipher_method )); + $encrypted = openssl_encrypt($data, $cipher_method, $key, 0, $iv); + // append value of $IV to the encrypted data with a separator + // so this can be used during decryption + return $iv . str_repeat("0", 10) . $encrypted; + } + /** + * Decrypts data based on provided key. + * + * @param string $data what data to decrypt + * @param string $key what key to use to decrypt + * @param string $cipher_method a cipher encrypt/decrypt method + * supported by OpenSSL + * @return string $out_data the decrypted string + */ + public function decrypt($data, $key, $cipher_method = 'aes-256-cbc') + { + $out_data = ""; + $pos = strpos($data, str_repeat("0", 10)); + if ($pos !== false) { + $data_parts = explode(str_repeat("0", 10), $data); + $out_data= openssl_decrypt($data_parts[1], $cipher_method, + $key, 0, $data_parts[0]); } - return $data; + return $out_data; } /** * Updates a group feed item's title and description. This assumes @@ -836,8 +857,8 @@ class GroupModel extends Model implements MediaConstants $group_item = $this->getGroupItem($id); if ($this->isGroupEncrypted($group_item['GROUP_ID'])) { $key = $this->getGroupKey($group_item['GROUP_ID']); - $title = $this->cipher($title, $key); - $description = $this->cipher($description, $key); + $title = $this->encrypt($title, $key); + $description = $this->encrypt($description, $key); } $sql = "UPDATE GROUP_ITEM SET TITLE=?, DESCRIPTION=?, EDIT_DATE=? WHERE ID=?"; @@ -994,10 +1015,10 @@ class GroupModel extends Model implements MediaConstants foreach ($groups as $group_key => $group_value) { if ($this->isGroupEncrypted($group_value['GROUP_ID'])) { $keys = $this->getGroupKey($group_value['GROUP_ID']); - $groups[$i]['TITLE'] = $this->cipher( - $group_value['TITLE'], $keys, 1); - $groups[$i]['DESCRIPTION'] = $this->cipher( - $group_value['DESCRIPTION'], $keys, 1); + $groups[$i]['TITLE'] = $this->decrypt( + $group_value['TITLE'], $keys); + $groups[$i]['DESCRIPTION'] = $this->decrypt( + $group_value['DESCRIPTION'], $keys); } $i++; } diff --git a/src/models/ImpressionModel.php b/src/models/ImpressionModel.php index d31555ceb..b7e64d486 100644 --- a/src/models/ImpressionModel.php +++ b/src/models/ImpressionModel.php @@ -345,7 +345,6 @@ class ImpressionModel extends Model * differs from the NUM_VIEWS COLUMN will this method need to be called. * @param int $fuzzy_num_views number of views after epsilon privacy * fuzzification applied. - * @param int $time_period time period for which to update the view */ public function updatePrivacyViews($item_id, $num_views, $fuzzy_num_views) { diff --git a/src/models/datasources/DatasourceManager.php b/src/models/datasources/DatasourceManager.php index c95578789..7d3aebe12 100755 --- a/src/models/datasources/DatasourceManager.php +++ b/src/models/datasources/DatasourceManager.php @@ -59,7 +59,9 @@ abstract class DatasourceManager * @var int */ public $total_time; - /** Sets up the query_log for query statistics */ + /** + * Sets up the query_log for query statistics + */ public function __construct() { $this->query_log = []; @@ -327,6 +329,7 @@ abstract class DatasourceManager * Used to convert a sql string that does an insert statement * into a sql string where a duplicate insert * is ignored + * @param string ssql original SQL insert statement * @param array $dbinfo containing fields for the current DBMS * (pdo for Postgres) and DB_HOST (DSN connection string to database) * @return string what to use for insert ignore expression diff --git a/src/views/TestsView.php b/src/views/TestsView.php index c52cf50b6..1dad09aaa 100644 --- a/src/views/TestsView.php +++ b/src/views/TestsView.php @@ -42,15 +42,17 @@ use seekquarry\yioop\configs as C; */ class TestsView extends View { - /** This view is drawn on a web layout + /** + * This view is drawn on a web layout * @var string */ public $layout = "web"; /** - * sDraws the web page on which users can control their search settings. + * Draws the web page on which users can control their search settings. * - * @param array $data contains anti CSRF token as well - * the language info and the current and possible per page settings + * @param array $data passed from controller. It fields contain info + * about the possibble tests that can be run, which test activity to + * carry out, etc. */ public function renderView($data) { @@ -89,6 +91,8 @@ class TestsView extends View /** * This function is responsible for listing out HTML links to the available * unit tests a user can run + * @param array $data passed from controller. Its $data['TEST_NAMES'] + * field should contain an array of all the tests that can be run */ function renderTestsList($data) { @@ -109,7 +113,9 @@ class TestsView extends View /** * Uses draw the results of all avaliable unit tests in Yioop * - * @param string $name the name of a unit test file in the current directory + * @param string $data passed from controller. Its $data['ALL_RESULTS'] + * field should contain an array of all the test results for unit + * tests that were carried out */ function renderAllTests($data) { @@ -124,7 +130,12 @@ class TestsView extends View * Uses draw the results of unit test class, run the tests in it and display * the results * - * @param string $name the name of a unit test file in the current directory + * @param string $data passed from controller. Its $data['TEST_NAME'] + * field should contain an array test case results for a particular + * unit test. It's $data['NO_ALL_LINK'] should say whether all tests + * are currently being run or just one. $data['PHANTOMJS_REQUIRED'] + * indicates some tests for Javascript that were run require + * Phantomjs to work */ function renderTest($data) { diff --git a/src/views/elements/BotstoryElement.php b/src/views/elements/BotstoryElement.php index ab159d0a8..17e2499b9 100644 --- a/src/views/elements/BotstoryElement.php +++ b/src/views/elements/BotstoryElement.php @@ -41,6 +41,12 @@ use seekquarry\yioop\configs as C; */ class BotstoryElement extends Element { + /** + * Used to draw the Bot Story action forms which are used to configure + * new Group Chat Bots + * @param array $data associative array of info from the ChatBotComponent + * needed to draw the form + */ public function render($data) { ?> diff --git a/tests/VersionManagerTest.php b/tests/VersionManagerTest.php index e5959f26a..00cfdd744 100644 --- a/tests/VersionManagerTest.php +++ b/tests/VersionManagerTest.php @@ -47,6 +47,7 @@ class VersionManagerTest extends UnitTest */ public $db; /** + * Folder to use for test repository * @var string */ public $version_test_folder = C\TEST_DIR .