_binary From 5920e28e8665171a61f964641598df4657a73c99 Mon Sep 17 00:00:00 2001
From: unknown <Tanmayee@Tanmayee-PC.(none)>
Date: Sat, 22 Oct 2011 12:21:00 -0500
Subject: [PATCH 263/263] Tanmayee_commit

---
 bin/fetcher.php         |    6 ++++++
 lib/crawl_constants.php |    3 ++-
 lib/fetch_url.php       |    4 ++++
 3 files changed, 12 insertions(+), 1 deletions(-)

diff --git a/bin/fetcher.php b/bin/fetcher.php
index d48d24b..4700919 100755
--- a/bin/fetcher.php
+++ b/bin/fetcher.php
@@ -871,6 +871,12 @@ class Fetcher implements CrawlConstants
 
         foreach($site_pages as $site) {
             $response_code = $site[self::HTTP_CODE]; 
+            
+	    //deals with short URLs and directs them to the original link
+	    if(isset($site[self::LOCATION]))
+	    {
+	    	$site[self::URL]=$site[self::LOCATION];
+	    }	
 
             //process robot.txt files separately
             if(isset($site[self::ROBOT_PATHS])) {
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
index f0fb00e..4001c9c 100644
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -165,7 +165,8 @@ interface CrawlConstants
     const DOMAIN_WEIGHTS = 'bm';
     const POSITION_LIST = 'bn';
     const PROXIMITY = 'bo';
-    
+    const LOCATION = 'bp';
+
     const NEEDS_OFFSET_FLAG = 0x7FFFFFFF;
 
 } 
diff --git a/lib/fetch_url.php b/lib/fetch_url.php
index 5cc7a37..3bb840f 100755
--- a/lib/fetch_url.php
+++ b/lib/fetch_url.php
@@ -281,6 +281,10 @@ class FetchUrl implements CrawlConstants
                 $site[CrawlConstants::MODIFIED] = 
                     strtotime(@trim($line_parts[1]));
             }
+  	    if(stristr($line,'Location:')){
+	    	$line_parts=explode("Location:",line);
+		$site[CrawlConstants::LOCATION]=@trim(line_parts[1]);
+	    }	
 
         }
         if(!isset($site[CrawlConstants::ENCODING]) ) {
-- 
1.7.7.msysgit.1

