_binary From eb5e88329adf0ee7b7e8c3039786d50c5d1f4608 Mon Sep 17 00:00:00 2001
From: Akshat Kukreti <akshatkukreti@gmail.com>
Date: Tue, 23 Oct 2012 01:39:05 -0700
Subject: [PATCH] cache control code modified

---
 controllers/search_controller.php | 68 +++++++++++++++++++++++++++++++++++----
 models/parallel_model.php         | 23 ++++++++++++-
 2 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/controllers/search_controller.php b/controllers/search_controller.php
index bdf080f..f8b588f 100755
--- a/controllers/search_controller.php
+++ b/controllers/search_controller.php
@@ -171,7 +171,7 @@ class SearchController extends Controller implements CrawlConstants
 
         if(isset($_SESSION['USER_ID'])) {
             $user = $_SESSION['USER_ID'];
-            $token_okay = $this->checkCSRFToken(CSRF_TOKEN, $user);
+            $token_okay = $this->checkCSRFToken('YIOOP_TOKEN', $user);
             if($token_okay === false) {
                 unset($_SESSION['USER_ID']);
                 $user = $_SERVER['REMOTE_ADDR'];
@@ -358,7 +358,7 @@ class SearchController extends Controller implements CrawlConstants
             $data["SUBSEARCH"] = $this->subsearch_name;
         }
         $data["HAS_STATISTICS"] = file_exists($stats_file);
-        $data[CSRF_TOKEN] = $this->generateCSRFToken($user);
+        $data['YIOOP_TOKEN'] = $this->generateCSRFToken($user);
         if($view == "search" && $raw == 0 && isset($data['PAGES'])) {
             $data['PAGES'] = $this->makeMediaGroups($data['PAGES']);
         }
@@ -592,6 +592,17 @@ class SearchController extends Controller implements CrawlConstants
      *  @param string $query original query string
      *  @param bool $raw the $_REQUEST['raw'] value
      *  @param bool if the current index name is that of a crawl mix
+
+
+
+
+
+
+
+
+
+
+
      *
      *  @return array ($query, $raw, $use_network, 
      *      $use_cache_if_possible, $guess_semantics)
@@ -808,6 +819,8 @@ class SearchController extends Controller implements CrawlConstants
         for($k = 0; $node->childNodes->length; $k++)  {
             if(!$node->childNodes->item($k)) { break; }
 
+
+
             $clone = $node->childNodes->item($k)->cloneNode(true);
 
             if($clone->nodeType == XML_TEXT_NODE) { 
@@ -849,9 +862,11 @@ class SearchController extends Controller implements CrawlConstants
      */
     function canonicalizeLinks($node, $url)
     {
+
         if(!isset($node->childNodes->length)) {
             return $node;
         }
+
         for($k = 0; $node->childNodes->length; $k++) {
             if(!$node->childNodes->item($k)) { break; }
 
@@ -973,6 +988,9 @@ class SearchController extends Controller implements CrawlConstants
     {
         global $CACHE, $IMAGE_TYPES;
 
+        $flag=0;
+        $crawl_item=null;
+
         $hash_key = crawlHash(
             $terms.$url.serialize($highlight).serialize($crawl_time));
         if(USE_CACHE) {
@@ -985,17 +1003,50 @@ class SearchController extends Controller implements CrawlConstants
         if($crawl_time == 0) {
             $crawl_time = $this->crawlModel->getCurrentIndexDatabaseName();
         }
-        $this->phraseModel->index_name = $crawl_time;
-        $this->crawlModel->index_name = $crawl_time;
 
-        $data = array();
+        //Get all crawl times
+        $crawl_times = array();
+        $all_crawl_details = $this->crawlModel->getCrawlList();
+        foreach($all_crawl_details as $crawl_details){
+            if($crawl_details['CRAWL_TIME'] !== null){
+                array_push($crawl_times,$crawl_details['CRAWL_TIME']);
+            }
+        }
+        for($i=0; $i < count($crawl_times); $i++){
+            $crawl_times[$i] = intval($crawl_times[$i]);
+        }
+        asort($crawl_times);
+
+        //Get int value of current crawl time for comparing
+        $crawl_time_int = intval($crawl_time);
+
+        foreach($crawl_times as $time){
+            if($time >= $crawl_time_int){
+
+                $crawl_time = (string)$time;
 
-        $crawl_item = $this->crawlModel->getCrawlItem($url, $queue_servers);
+                $this->phraseModel->index_name = $crawl_time;
+                $this->crawlModel->index_name = $crawl_time;
 
-        if(!$crawl_item ) {
+                $crawl_item = $this->crawlModel->
+                    getCrawlItem($url, $queue_servers);
+
+                if($crawl_item !== false){
+                    $flag = 1;
+                    break;
+                } else {
+                    continue;
+                }
+            }
+        }
+
+        $data = array();
+
+        if(!$flag) {
             $this->displayView("nocache", $data);
             return;
         }
+
         $in_url = "";
         $image_flag = false;
         if(isset($crawl_item[self::THUMB])) {
@@ -1135,6 +1186,7 @@ class SearchController extends Controller implements CrawlConstants
             $head = $this->canonicalizeLinks($head, $url);
         }
         $body =  $dom->getElementsByTagName('body')->item(0);
+
         if($body == false) {
             $body_tags = "<frameset><frame><noscript><img><span><b><i><em>".
                 "<strong><h1><h2><h3><h4><h5><h6><p><div>".
@@ -1146,6 +1198,7 @@ class SearchController extends Controller implements CrawlConstants
             @$dom->loadHTML($cache_file);
             $body =  $dom->getElementsByTagName('body')->item(0);
         }
+
         //make tags in body absolute
         $body = $this->canonicalizeLinks($body, $url);
         $first_child = $body->firstChild;
@@ -1160,6 +1213,7 @@ class SearchController extends Controller implements CrawlConstants
         $summaryNode->setAttributeNS("","id", "summary-page-id");
 
 
+
         if(isset($cache_item[self::HEADER])) {
             $summary_string = $cache_item[self::HEADER]."\n". $summary_string;
         }
diff --git a/models/parallel_model.php b/models/parallel_model.php
index 0e47e90..f50ac60 100755
--- a/models/parallel_model.php
+++ b/models/parallel_model.php
@@ -106,8 +106,10 @@ class ParallelModel extends Model implements CrawlConstants
         if($index_name == "") {
             $index_name = $this->index_name;
         }
+
         $results = $this->getCrawlItems(
             array($hash_url =>array($url, $index_name)), $machine_urls);
+
         if(isset($results[$hash_url])) {
             return $results[$hash_url];
         }
@@ -227,15 +229,26 @@ class ParallelModel extends Model implements CrawlConstants
      */
     function nonNetworkGetCrawlItems($lookups)
     {
+        $summary_offset = null;
+        $generation = null;
         $summaries = array();
         $db = $this->db;
         $db->selectDB(DB_NAME);
+
         foreach($lookups as $lookup => $lookup_info) {
             if(count($lookup_info) == 2 && $lookup_info[0][0] === 'h') {
                 list($url, $index_name) = $lookup_info;
+
                 $index_archive = IndexManager::getIndex($index_name);
-                list($summary_offset, $generation) = 
+
+                $offset_gen_arr = 
                     $this->lookupSummaryOffsetGeneration($url, $index_name);
+
+                if($offset_gen_arr !== false){
+                    list($summary_offset, $generation) = $offset_gen_arr;
+                } else {
+                    return false;
+                }
                 $summary = 
                     $index_archive->getPage($summary_offset, $generation);
             } else {
@@ -342,22 +355,30 @@ class ParallelModel extends Model implements CrawlConstants
     function lookupSummaryOffsetGeneration($url_or_key, $index_name = "", 
         $is_key = false)
     {
+
         if($index_name == "") {
             $index_name = $this->index_name;
         }
         $index_archive = IndexManager::getIndex($index_name);
+
+        //var_dump($index_archive);
+
         $num_retrieved = 0;
         $pages = array();
         $summary_offset = NULL;
         $num_generations = $index_archive->generation_info['ACTIVE'];
+
         $hash_key = ($is_key) ? crawlHash($url_or_key) : 
             crawlHash("info:$url_or_key");
+
         $word_iterator = new WordIterator($hash_key, $index_name);
 
         if(is_array($next_docs = $word_iterator->nextDocsWithWord())) {
+
              foreach($next_docs as $doc_key => $doc_info) {
                  $summary_offset =
                     $doc_info[CrawlConstants::SUMMARY_OFFSET];
+
                  $generation = $doc_info[CrawlConstants::GENERATION];
                  $index_archive->setCurrentShard($generation, true);
                  $page = @$index_archive->getPage($summary_offset);
-- 
1.7.11.msysgit.1

