_binary From 7548d5ba754edd044afe7807b387ed13f2f6034c Mon Sep 17 00:00:00 2001
From: Snigdha Parvatneni <snigdha.parvatneni@gmail.com>
Date: Sun, 15 Dec 2013 14:43:00 -0800
Subject: [PATCH] source code search

---
 bin/fetcher.php                     | 172 ++++++++-
 controllers/search_controller.php   |  23 +-
 lib/crawl_constants.php             |   4 +-
 lib/fetch_git_repository_urls.php   | 670 ++++++++++++++++++++++++++++++++++++
 lib/locale_functions.php            |  31 ++
 lib/phrase_parser.php               | 295 +++++++++++++++-
 lib/processors/java_processor.php   | 208 +++++++++++
 lib/processors/python_processor.php | 209 +++++++++++
 models/phrase_model.php             |  23 +-
 views/search_view.php               |  23 +-
 10 files changed, 1622 insertions(+), 36 deletions(-)
 mode change 100644 => 100755 lib/crawl_constants.php
 create mode 100755 lib/fetch_git_repository_urls.php
 mode change 100644 => 100755 lib/locale_functions.php
 create mode 100755 lib/processors/java_processor.php
 create mode 100755 lib/processors/python_processor.php

diff --git a/bin/fetcher.php b/bin/fetcher.php
index 580a3bb..f04c483 100755
--- a/bin/fetcher.php
+++ b/bin/fetcher.php
@@ -113,6 +113,8 @@ require_once BASE_DIR."/lib/fetch_url.php";
 require_once BASE_DIR."/lib/crawl_constants.php";
 /** used to build mini-inverted index*/
 require_once BASE_DIR."/lib/index_shard.php";
+/** Used to extract Git internal urls from given Git url*/
+require_once BASE_DIR."/lib/fetch_git_repository_urls.php";
 
 /*
  *  We'll set up multi-byte string handling to use UTF-8
@@ -416,12 +418,54 @@ class Fetcher implements CrawlConstants
     var $active_rankers;
 
     /**
+     * To keep track of total number of Git internal urls
+     *
+     * @var int
+     */
+    var $total_git_urls;
+
+    /**
+     * To store all the internal git urls fetched
+     *
+     * @var array
+     */
+    var $all_git_urls;
+
+    /**
+     * To map programming languages with their extensions
+     *
+     * @var array
+     */
+    var $programming_language_extension;
+
+    /**
      * Before receiving any data from a queue server's web app this is
      * the default assumed post_max_size in bytes
      */
     const DEFAULT_POST_MAX_SIZE = 2000000;
 
     /**
+     * constant indicating Git repository
+     */
+    const REPOSITORY_GIT = 'git';
+
+    /**
+     * constant indicating Git repository
+     */
+    const GIT_URL_CONTINUE = '@@@@';
+
+    /**
+     * An indicator to tell no actions to be taken
+     */
+    const INDICATOR_NONE = 'none';
+
+    /**
+     * A indicator to represent next position after the access code in Git
+     * tree object
+     */
+     const HEX_NULL_CHARACTER = "\x00";
+
+    /**
      * Sets up the field variables so that crawling can begin
      */
     function __construct()
@@ -475,6 +519,11 @@ class Fetcher implements CrawlConstants
         $this->active_classifiers = array();
         $this->active_rankers = array();
 
+        $this->total_git_urls = 0;
+        $this->all_git_urls = array();
+        $this->programming_language_extension = array('java' => 'java',
+            'py' => 'py');
+
         //we will get the correct crawl order from a queue_server
         $this->crawl_order = self::PAGE_IMPORTANCE;
     }
@@ -707,10 +756,23 @@ class Fetcher implements CrawlConstants
         $site_pages = array_merge($site_pages,
             FetchUrl::getPages($filtered_sites, true,
                 $this->page_range_request, $tmp_dir));
-
+        for($j = 0; $j < count($site_pages); $j++) {
+            if(isset($site_pages[$j][self::REPOSITORY_TYPE])) {
+                $git_repository_url = $site_pages[$j][self::URL];
+                $git_compressed_content = FetchGitRepositoryUrls::getGitdata(
+                    $git_repository_url);
+                $git_uncompressed_content = gzuncompress(
+                    $git_compressed_content);
+                $length = strlen ($git_uncompressed_content);
+                $git_hash_end = strpos($git_uncompressed_content,
+                    self::HEX_NULL_CHARACTER);
+                $git_uncompressed_content = substr($git_uncompressed_content,
+                    $git_hash_end + 1, $length);
+                $site_pages[$j][self::PAGE] = $git_uncompressed_content;
+            }
+        }
         list($downloaded_pages, $schedule_again_pages) =
             $this->reschedulePages($site_pages);
-
         if($can_schedule_again == true) {
             //only schedule to crawl again on fail sites without crawl-delay
             crawlLog("  Scheduling again..");
@@ -1379,9 +1441,26 @@ class Fetcher implements CrawlConstants
                 // only download if host doesn't seem congested
                 if(!isset($this->hosts_with_errors[$host]) ||
                     $this->hosts_with_errors[$host] < DOWNLOAD_ERROR_THRESHOLD){
-                    $seeds[$i][self::URL] = $site_pair['value'][0];
-                    $seeds[$i][self::WEIGHT] = $site_pair['value'][1];
-                    $seeds[$i][self::CRAWL_DELAY] = $site_pair['value'][2];
+                    $url_to_check = $site_pair['value'][0];
+                    $extension = UrlParser::getDocumentType($url_to_check);
+                    $repository_indicator = FetchGitRepositoryUrls::
+                        checkForRepository($extension);
+                    if($repository_indicator == self::REPOSITORY_GIT) {
+                        $git_internal_urls = FetchGitRepositoryUrls::
+                            setGitRepositoryUrl($url_to_check, $i, $seeds,
+                                $repository_indicator, $site_pair,
+                                    $this->total_git_urls, $this->all_git_urls);
+                        $i = $git_internal_urls['position'];
+                        $git_url_index = $git_internal_urls['index'];
+                        $seeds = $git_internal_urls['seeds'];
+                        $repository_indicator = $git_internal_urls['indicator'];
+                        $this->total_git_urls = $git_internal_urls['count'];
+                        $this->all_git_urls = $git_internal_urls['all'];
+                    } else {
+                        $seeds[$i][self::URL] = $site_pair['value'][0];
+                        $seeds[$i][self::WEIGHT] = $site_pair['value'][1];
+                        $seeds[$i][self::CRAWL_DELAY] = $site_pair['value'][2];
+                    }
                     /*
                       Crawl delay is only used in scheduling on the queue_server
                       on the fetcher, we only use crawl-delay to determine
@@ -1403,11 +1482,29 @@ class Fetcher implements CrawlConstants
         } //end while
 
         foreach($delete_indices as $delete_index) {
+            $extension = UrlParser::getDocumentType(
+                $this->to_crawl[$delete_index][0]);
+            $repository_type = FetchGitRepositoryUrls::checKForRepository(
+                $extension);
+            $git_set = false;
             if($to_crawl_flag == true) {
-                unset($this->to_crawl[$delete_index]);
+                if($repository_type != self::REPOSITORY_GIT) {
+                    unset($this->to_crawl[$delete_index]);
+                }
             } else {
                 unset($this->to_crawl_again[$delete_index]);
             }
+            if($repository_type == self::REPOSITORY_GIT) {
+                if(!$git_set) {
+                    $next_url_start = $url_to_check . self::GIT_URL_CONTINUE.
+                        $git_url_index;
+                    $git_set = true;
+                    $this->to_crawl[$delete_index][0] = $next_url_start;
+                }
+                if($repository_indicator == self::INDICATOR_NONE) {
+                    unset($this->to_crawl[$delete_index]);
+                }
+            }
         }
 
         crawlLog("Fetch url list to download time ".
@@ -1501,7 +1598,19 @@ class Fetcher implements CrawlConstants
             } else {
                 $type = $site[self::TYPE];
             }
-
+            if(isset($site[self::FILE_NAME])) {
+                $extension = UrlParser::getDocumentType($site[self::FILE_NAME]);
+                if($extension == $this->programming_language_extension['java']){
+                    $type = "text/java";
+                } else if($extension == $this->programming_language_extension[
+                    'py']){
+                    $type = "text/py";
+                } else {
+                    $type = $site[self::TYPE];
+                }
+            } else {
+                $type = $site[self::TYPE];
+            }
             $handled = false;
             /*deals with short URLs and directs them to the original link
               for robots.txt don't want to introduce stuff that can be
@@ -1582,8 +1691,17 @@ class Fetcher implements CrawlConstants
                     }
                 }
                 crawlLog("  Using Processor...".$page_processor);
+                if(isset($site[self::REPOSITORY_TYPE]) &&
+                    $site[self::REPOSITORY_TYPE] == self::REPOSITORY_GIT) {
+                    $tmp_url_store = $site[self::URL];
+                    $site[self::URL] = $site[self::FILE_NAME];
+                }
                 $doc_info = $processor->handle($site[self::PAGE],
                     $site[self::URL]);
+                if(isset($site[self::REPOSITORY_TYPE]) &&
+                    $site[self::REPOSITORY_TYPE] == self::REPOSITORY_GIT) {
+                    $site[self::URL] = $tmp_url_store;
+                }
                 if(!$doc_info) {
                     crawlLog("  Processing Yielded No Data For: ".
                         $site[self::URL]);
@@ -1659,11 +1777,22 @@ class Fetcher implements CrawlConstants
 
                 $summarized_site_pages[$i][self::URL] =
                     strip_tags($site[self::URL]);
-                $summarized_site_pages[$i][self::TITLE] = strip_tags(
-                    $site[self::DOC_INFO][self::TITLE]);
+                if(isset($site[self::REPOSITORY_TYPE]) &&
+                    $site[self::REPOSITORY_TYPE] == self::REPOSITORY_GIT) {
+                     $summarized_site_pages[$i][self::TITLE] =
+                         $site[self::FILE_NAME];
+                } else {
+                     $summarized_site_pages[$i][self::TITLE] = strip_tags(
+                         $site[self::DOC_INFO][self::TITLE]);
                     // stripping html to be on the safe side
-                $summarized_site_pages[$i][self::DESCRIPTION] =
-                    strip_tags($site[self::DOC_INFO][self::DESCRIPTION]);
+                }
+                if(!isset($site[self::REPOSITORY_TYPE])) {
+                    $summarized_site_pages[$i][self::DESCRIPTION] =
+                        strip_tags($site[self::DOC_INFO][self::DESCRIPTION]);
+                } else {
+                    $summarized_site_pages[$i][self::DESCRIPTION] =
+                        $site[self::DOC_INFO][self::DESCRIPTION];
+                }
                 if(isset($site[self::DOC_INFO][self::JUST_METAS]) ||
                     isset($site[self::ROBOT_PATHS])) {
                     $summarized_site_pages[$i][self::JUST_METAS] = true;
@@ -2510,8 +2639,19 @@ class Fetcher implements CrawlConstants
                 if($is_link) {
                     $phrase_string = $site[self::DESCRIPTION];
                 } else {
-                    $phrase_string = $host_words." ".$site[self::TITLE] .
+                    if(isset($site[self::LANG])) {
+                        if(isset($this->programming_language_extension[$site[
+                            self::LANG]])) {
+                            $phrase_string = $site[self::DESCRIPTION];
+                        } else {
+                            $phrase_string = $host_words." ".$site[self::TITLE]
+                                . " ". $path_words . " ". $site[self::
+                                    DESCRIPTION];
+                        }
+                    } else {
+                        $phrase_string = $host_words." ".$site[self::TITLE] .
                             " ". $path_words . " ". $site[self::DESCRIPTION];
+                    }
                 }
                 if(isset($site[self::LANG])) {
                     $lang = guessLocaleFromString(
@@ -2522,8 +2662,9 @@ class Fetcher implements CrawlConstants
                     PhraseParser::extractPhrasesInLists($phrase_string,
                         $lang);
                 $len = strlen($phrase_string);
-                if(PhraseParser::computeSafeSearchScore($word_lists, $len) <
-                    0.012) {
+                if(isset($this->programming_language_extension[$lang]) ||
+                    PhraseParser::computeSafeSearchScore($word_lists, $len) <
+                        0.012) {
                     $meta_ids[] = "safe:true";
                     $safe = true;
                 } else {
@@ -2569,7 +2710,8 @@ class Fetcher implements CrawlConstants
                 In this case link  info is not particularly useful for indexing
                 and can greatly slow building inverted index.
              */
-            if(!$this->no_process_links && !isset($site[self::JUST_METAS])) {
+            if(!$this->no_process_links && !isset($site[self::JUST_METAS]) &&
+                !isset($this->programming_language_extension[$lang])) {
                 foreach($site[self::LINKS] as $url => $link_text) {
                     /* this mysterious check means won't index links from
                       robots.txt. Sitemap will still be in TO_CRAWL, but that's
diff --git a/controllers/search_controller.php b/controllers/search_controller.php
index dd0470d..a8a4625 100755
--- a/controllers/search_controller.php
+++ b/controllers/search_controller.php
@@ -94,6 +94,11 @@ class SearchController extends Controller implements CrawlConstants
     var $subsearch_identifier = "";
 
     /**
+     * constant indicating Git repository
+     */
+    const REPOSITORY_GIT = 'git';
+
+    /**
      * This is the main entry point for handling a search request.
      *
      * ProcessRequest determines the type of search request (normal request ,
@@ -128,13 +133,17 @@ class SearchController extends Controller implements CrawlConstants
                     $save_timestamp);
                     // calculate the results of a search if there is one
             } else {
-                $ui_array = array("highlight", "yioop_nav", "history",
-                    "summaries", "version");
-                if(isset($_REQUEST['from_cache'])) {
-                    $ui_array[] = "cache_link_referrer";
-                }
-                if(isset($_REQUEST['hist_open'])) {
-                    $ui_array[] = "hist_ui_open";
+                if($_REQUEST['repository'] == self::REPOSITORY_GIT) {
+                    $ui_array = array();
+                } else {
+                    $ui_array = array("highlight", "yioop_nav", "history",
+                        "summaries", "version");
+                    if(isset($_REQUEST['from_cache'])) {
+                        $ui_array[] = "cache_link_referrer";
+                    }
+                    if(isset($_REQUEST['hist_open'])) {
+                        $ui_array[] = "hist_ui_open";
+                    }
                 }
                 $this->cacheRequestAndOutput($arg, $ui_array, $query,
                     $index_timestamp);
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
old mode 100644
new mode 100755
index d80d7c9..039605b
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -240,7 +240,9 @@ interface CrawlConstants
     const ACTIVE_RANKERS = 'db';
     const USER_RANKS = "dc";
     const INDEXING_PLUGINS_DATA = "dd";
-
+    const REPOSITORY_TYPE = 'de';
+    const FILE_NAME = 'df';
+    const SHA_HASH = 'dg';
     const NEEDS_OFFSET_FLAG = 0x7FFFFFFF;
 }
 ?>
diff --git a/lib/fetch_git_repository_urls.php b/lib/fetch_git_repository_urls.php
new file mode 100755
index 0000000..ac15065
--- /dev/null
+++ b/lib/fetch_git_repository_urls.php
@@ -0,0 +1,670 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2013  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Snigdha Rao Parvatneni
+ * @package seek_quarry
+ * @subpackage library
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2013
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** Load in global configuration settings */
+require_once BASE_DIR.'/configs/config.php';
+
+/**
+ * Library of functions used to fetch Git internal urls
+ *
+ * @author Chris Pollett
+ * @package seek_quarry
+ * @subpackage library
+ */
+class FetchGitRepositoryUrls implements CrawlConstants
+{
+    /**
+     * An indicator to tell no actions to be taken
+     */
+    const INDICATOR_NONE = 'none';
+
+    /**
+     * An indicator to indicate git repository
+     */
+    const INDICATOR_GIT = 'git';
+
+    /**
+     * An indicator to tell more git urls need to be fetched
+     */
+    const GIT_URL_CONTINUE = '@@@@';
+
+    /**
+     * An indicator to tell starting position of Git url to be used
+     */
+    const GIT_BASE_URL_START = 0;
+
+    /**
+     * An indicator to tell ending position of Git url to be used
+     */
+    const GIT_BASE_URL_END = '###';
+
+    /**
+     * A fixed component to be used with Git base url to form Git first url
+     */
+    const GIT_URL_EXTENSION = 'info/refs?service=git-upload-pack';
+
+    /**
+     * A fixed component to be used with Git urls to get next Git urls
+     */
+    const GIT_URL_OBJECT = 'objects/';
+
+    /**
+     * A fixed indicator used to get last letter of git base url
+     */
+    const GIT_BASE_URL_END_POSITION = -1;
+
+    /**
+     * A fixed indicator used to get last letter of git base url
+     */
+    const GIT_BASE_END_LETTER = 1;
+
+    /**
+     * A fixed position used to indicate starting point to fetch next Git url
+     * from the master file
+     */
+    const GIT_NEXT_URL_START = 0;
+
+    /**
+     * A fixed position used to indicate ending position to fetch next Git url
+     * from the master file
+     */
+    const GIT_NEXT_URL_END = 40;
+
+    /**
+     * A fixed indicator used to make desired Git folder structure from SHA hash
+     */
+    const GIT_URL_SPLIT = '/';
+
+    /**
+     * A fixed indicator used to mark starting position of SHA hash of Git
+     * master tree
+     */
+    const GIT_MASTER_TREE_HASH_START = 16;
+
+    /**
+     * A fixed indicator used to mark ending position of SHA hash of Git
+     * master tree
+     */
+    const GIT_MASTER_TREE_HASH_END = 41;
+
+    /**
+     * A fixed indicator used to mark starting position of SHA hash used to
+     * indicate Git object folder
+     */
+    const GIT_FOLDER_NAME_START = 0;
+
+    /**
+     * A fixed indicator used to mark ending position of SHA hash used to
+     * indicate Git object folder
+     */
+    const GIT_FOLDER_NAME_END = 2;
+
+    /**
+     * A fixed indicator used to mark starting position of SHA hash used to
+     * indicate Git object file
+     */
+    const GIT_FILE_NAME_START = 2;
+
+    /**
+     * A fixed indicator used to mark ending position of SHA hash used to
+     * indicate Git object file
+     */
+    const GIT_FILE_NAME_END = 38;
+
+    /**
+     * A fixed indicator used to indicate Git blob object
+     */
+     const GIT_BLOB_OBJECT = "blob";
+
+    /**
+     * A fixed indicator used to indicate Git tree object
+     */
+     const GIT_TREE_OBJECT = "tree";
+
+    /**
+     * A cURL time out parameter
+     */
+     const CURL_TIMEOUT = 5;
+
+    /**
+     * A cURL transfer parameter
+     */
+     const CURL_TRANSFER = 1;
+
+    /**
+     * Git blob access code starting position
+     */
+     const BLOB_ACCESS_CODE_START = 0;
+
+    /**
+     * Git blob access code ending position
+     */
+     const BLOB_ACCESS_CODE_END = 6;
+
+    /**
+     * Git tree access code starting position
+     */
+     const TREE_ACCESS_CODE_START = 0;
+
+    /**
+     * Git tree access code ending position
+     */
+     const TREE_ACCESS_CODE_END = 5;
+
+    /**
+     * Git SHA hash binary starting position
+     */
+     const SHA_HASH_BINARY_START = 0;
+
+    /**
+     * Git SHA hash binary ending position
+     */
+     const SHA_HASH_BINARY_END = 20;
+
+    /**
+     * A indicator for starting of Git file or folder name
+     */
+     const GIT_NAME_START = 0;
+
+    /**
+     * A indicator to represent next position after the access code in Git
+     * blob object
+     */
+     const GIT_BLOB_NEXT = 7;
+
+    /**
+     * A indicator to represent next position after the access code in Git
+     * tree object
+     */
+     const GIT_TREE_NEXT = 6;
+
+    /**
+     * A indicator to represent next position after the access code in Git
+     * tree object
+     */
+     const HEX_NULL_CHARACTER = "\x00";
+
+    /**
+     * A indicator to represent that a git file is a blob file
+     */
+    const GIT_BLOB_INDICATOR = '100';
+
+    /**
+     * A indicator to represent that a git file is a tree file
+     */
+    const GIT_TREE_INDICATOR = '400';
+
+    /**
+     * A list of meta words that might be extracted from a query
+     * @var array
+     */
+    static $repository_types = array('git' => 'git', 'svn' => 'svn',
+        'cvs' => 'cvs', 'vss' => 'vss', 'mercurial' => 'mercurial',
+        'monotone' => 'monotone', 'bazaar' => 'bazaar', 'darcs' => 'darcs',
+        'arch' => 'arch');
+
+    /**
+     * An array used to store all the Git internal urls
+     * @var array
+     */
+    static $all_git_urls;
+
+    /**
+     * Checks repository type based on extension
+     *
+     * @param string $extension to check
+     * @return string $repository_type repository type based on the
+     * extension of urls
+     */
+    static function checkForRepository($extension)
+    {
+        $repository_type;
+        if(isset(self::$repository_types[$extension])) {
+            $repository_type = self::$repository_types[$extension];
+        } else {
+            $repository_type = self::INDICATOR_NONE;
+        }
+        return $repository_type;
+    }
+
+    /**
+     * Checks repository type based on extension
+     *
+     * @param string $url_to_check url needs to be processed
+     * @param int $counter to keep track of number of urls processed
+     * @param array $seeds store sites which are ready to be downloaded
+     * @param array $repository_indicator indicates the type of the repository
+     * @param array $site_pair contains original Git url crawled
+     * @return array $git_internal_urls containing all the internal Git urls
+     * fetched from the parent Git url
+     */
+    static function setGitRepositoryUrl($url_to_check, $counter, $seeds,
+        $repository_indicator, $site_pair, $total_git_urls, $all_git_urls)
+    {
+        $git_internal_urls = array();
+        if(!strpos($url_to_check, self::GIT_URL_CONTINUE)) {
+            $git_next_urls = self::fetchGitRepositoryUrl($url_to_check);
+            $all_git_urls = $git_next_urls;
+            $total_git_urls = count($all_git_urls);
+            $count_all_git_urls = $total_git_urls;
+            if(intval(NUM_MULTI_CURL_PAGES) - $counter < $total_git_urls) {
+                $total_git_urls = intval(NUM_MULTI_CURL_PAGES) - $counter;
+            }
+            for($j = 0; $j < $total_git_urls; $j++) {
+                $seeds[$counter][self::URL] = $git_next_urls[$j][2];
+                $seeds[$counter][self::WEIGHT] = $site_pair['value'][1];
+                $seeds[$counter][self::CRAWL_DELAY] = $site_pair['value'][2];
+                $seeds[$counter][self::REPOSITORY_TYPE] = $repository_indicator;
+                $seeds[$counter][self::FILE_NAME] = $git_next_urls[$j][0];
+                $seeds[$counter][self::SHA_HASH] = $git_next_urls[$j][1];
+                $counter++;
+                $git_url_index = $j + 1;
+                if($git_url_index >= $count_all_git_urls) {
+                    $repository_indicator = self::INDICATOR_NONE;
+                } else {
+                    $repository_indicator = self::INDICATOR_GIT;
+                }
+            }
+            $counter--;
+        } else {
+            $position = strpos($url_to_check, self::GIT_URL_CONTINUE);
+            $extension_string = substr($url_to_check, $position,
+                strlen($url_to_check));
+            $extension_count = explode(self::GIT_URL_CONTINUE,
+                $extension_string);
+            $git_index = intval(array_sum($extension_count));
+            $url_to_check = substr($url_to_check, self::GIT_NEXT_URL_START,
+                $position);
+            $count_all_git_urls = $total_git_urls;
+            if(intval(NUM_MULTI_CURL_PAGES) - $counter < $total_git_urls -
+                $git_index) {
+                $total_git_urls = intval(NUM_MULTI_CURL_PAGES) - $counter;
+            } else {
+                $total_git_urls = $total_git_urls - $git_index;
+            }
+            for($j = 0; $j < $total_git_urls; $j++) {
+                $seeds[$counter][self::URL] = $all_git_urls[$git_index][2];
+                $seeds[$counter][self::WEIGHT] = $site_pair['value'][1];
+                $seeds[$counter][self::CRAWL_DELAY] = $site_pair['value'][2];
+                $seeds[$counter][self::REPOSITORY_TYPE] = $repository_indicator;
+                $seeds[$counter][self::FILE_NAME] = $all_git_urls[$git_index]
+                    [0];
+                $seeds[$counter][self::SHA_HASH] = $all_git_urls[$git_index][1];
+                $counter++;
+                $git_index++;
+                $git_url_index = $j + 1;
+                if($git_index >= $count_all_git_urls) {
+                    $repository_indicator = self::INDICATOR_NONE;
+                } else {
+                    $repository_indicator = self::INDICATOR_GIT;
+                }
+            }
+            $counter--;
+        }
+        $git_internal_urls['position'] = $counter;
+        $git_internal_urls['index'] = $git_url_index;
+        $git_internal_urls['seeds'] = $seeds;
+        $git_internal_urls['indicator'] = $repository_indicator;
+        $git_internal_urls['count'] = $count_all_git_urls;
+        $git_internal_urls['all'] = $all_git_urls;
+        return $git_internal_urls;
+    }
+
+    /**
+     * Get the Git internal urls from the parent Git url
+     *
+     * @param string $url_to_check url needs to be processed
+     * @return an array $git_next_urls consists of list of Git
+     * internal urls wich are called during the git clone
+     */
+    static function fetchGitRepositoryUrl($url_to_check)
+    {
+        $compression_indicator = false;
+        $position = strpos($url_to_check, self::GIT_BASE_URL_END);
+        $git_base_url = substr($url_to_check, self::GIT_BASE_URL_START,
+            $position);
+        $base_url_last_letter = substr($git_base_url, self::
+            GIT_BASE_URL_END_POSITION, self::GIT_BASE_END_LETTER);
+        if($base_url_last_letter != self::GIT_URL_SPLIT) {
+            $git_base_url = $git_base_url . self::GIT_URL_SPLIT;
+        }
+        $git_first_url =  $git_base_url.self::GIT_URL_EXTENSION;
+        $git_first_url_content = self::getNextGitUrl($git_first_url,
+            $compression_indicator);
+        $compression_indicator = true;
+        $git_second_url = self::getGitMasterFile($git_first_url_content,
+            $git_base_url);
+        $git_second_url_content = self::getNextGitUrl($git_second_url,
+            $compression_indicator);
+        $git_third_url = self::getGitMasterTree($git_second_url_content,
+            $git_base_url);
+        $git_third_url_content = self::getNextGitUrl($git_third_url,
+            $compression_indicator);
+        $git_next_urls = self::getObjects($git_third_url_content,
+            $git_base_url);
+        return $git_next_urls;
+    }
+
+    /**
+     * Get the Git second url which points to Git master tree structure
+     *
+     * @param string $git_first_url_content contents of Git first url
+     * @param string $git_base_url common portion of Git urls
+     * @return string $git_next_url consists of second internal Git url
+     */
+    static function getGitMasterFile($git_first_url_content, $git_base_url)
+    {
+        $git_extended_url = substr($git_first_url_content,
+            self::GIT_NEXT_URL_START, self::GIT_NEXT_URL_END);
+        $first_split_git_extended_url = substr($git_extended_url,
+            self::GIT_FOLDER_NAME_START, self::GIT_FOLDER_NAME_END);
+        $second_split_git_extended_url = substr($git_extended_url,
+            self::GIT_FILE_NAME_START, self::GIT_FILE_NAME_END);
+        $git_url_connector = $first_split_git_extended_url .
+            self::GIT_URL_SPLIT . $second_split_git_extended_url;
+        $git_next_url = $git_base_url . self::GIT_URL_OBJECT .
+            $git_url_connector;
+        return $git_next_url;
+    }
+
+    /**
+     * Get the Git third url which contains the information about the
+     *     organization of entire git repository
+     *
+     * @param string $git_second_url_content contents of Git second url
+     * @param string $git_base_url common portion of git urls
+     * @return string $git_next_url consists of third internal git url
+     */
+    static function getGitMasterTree($git_second_url_content, $git_base_url)
+    {
+        $git_master_tree_hash = substr($git_second_url_content,
+            self::GIT_MASTER_TREE_HASH_START, self::GIT_MASTER_TREE_HASH_END);
+        $git_object_folder_name = substr($git_master_tree_hash,
+            self::GIT_FOLDER_NAME_START, self::GIT_FOLDER_NAME_END);
+        $git_object_file_name = substr($git_master_tree_hash,
+            self::GIT_FILE_NAME_START, self::GIT_FILE_NAME_END);
+        $git_object_path = $git_object_folder_name . self::GIT_URL_SPLIT .
+            $git_object_file_name;
+        $git_next_url = $git_base_url . self::GIT_URL_OBJECT . $git_object_path;
+        return $git_next_url;
+    }
+
+    /**
+     * Get the Git content from url which will be used to get the
+     *     next git url
+     *
+     * @param string $git_url git url to extract contents from it
+     * @param string $compression_indicator indicator for compress and
+     * uncompress contents
+     * @return string $git_object_content consists contents extracted from the
+     * url
+     */
+    static function getNextGitUrl($git_url, $compression_indicator)
+    {
+        if(!$compression_indicator) {
+            $git_object_compress_content = self::getGitdata($git_url);
+            $git_object_content = $git_object_compress_content;
+        } else {
+            $git_object_compress_content = self::getGitdata($git_url);
+            $git_object_uncompress_content = gzuncompress(
+                $git_object_compress_content);
+            $git_object_content = $git_object_uncompress_content;
+        }
+        return $git_object_content;
+    }
+
+    /**
+     * Get the Git blob and tree objects
+     *
+     * @param string $git_object_content compressed content of git master tree
+     *     file
+     * @param string $git_base_url common content of git url
+     * @return array $blob_url contains information and url for git blob objects
+     */
+    static function getObjects($git_object_content, $git_base_url)
+    {
+        $blob_url = array();
+        $temp_git_object_content['content'] = $git_object_content;
+        for($i = 0; $i < strlen($git_object_content); $i++) {
+            $blob_position = strpos($temp_git_object_content['content'],
+                self::GIT_BLOB_INDICATOR);
+            $tree_position = strpos($temp_git_object_content['content'],
+                self::GIT_TREE_INDICATOR);
+            $git_object_positions = self::checkPosition($blob_position,
+                $tree_position, $git_object_content);
+            $blob_position = $git_object_positions['blob'];
+            $tree_position = $git_object_positions['tree'];
+            if($blob_position < $tree_position) {
+                $temp_git_object_content = self::readBlobSha(
+                    $temp_git_object_content['content'], $blob_position,
+                        strlen($temp_git_object_content['content']),
+                            $git_base_url);
+            }
+            else if($tree_position < $blob_position) {
+                $temp_git_object_content = self::readTreeSha(
+                    $temp_git_object_content['content'], $tree_position,
+                        strlen($temp_git_object_content['content']),
+                            $git_base_url);
+            }
+            $i = strlen($temp_git_object_content['content']);
+            $i = strlen($git_object_content) - $i;
+            if($temp_git_object_content['value']['indicator'] !=
+                self::GIT_TREE_OBJECT) {
+                $blob_details[0] = $temp_git_object_content['value']['name'];
+                $blob_details[1] = $temp_git_object_content['value']['hash'];
+                $blob_details[2] = $temp_git_object_content['value']['url'];
+                $blob_url[] = $blob_details;
+            }
+            if($temp_git_object_content['indicator'] != self::GIT_BLOB_OBJECT) {
+                for($k = 0; $k < count($temp_git_object_content['indicator']);
+                    $k++) {
+                    $blob_details[0] = $temp_git_object_content['indicator'][$k]
+                        [0];
+                    $blob_details[1] = $temp_git_object_content['indicator'][$k]
+                        [1];
+                    $blob_details[2] = $temp_git_object_content['indicator'][$k]
+                        [2];
+                    $blob_url[] = $temp_git_object_content['indicator'][$k];
+                }
+            }
+        }
+        return $blob_url;
+    }
+
+    /**
+     * checks the position of access code for null values
+     *
+     * @param string $git_blob_position first occuence of git blob access code
+     * @param string $git_tree_position first occuence of git tree access code
+     * @param string $git_object_content compressed content of git master tree
+     * @return array $git_object_positions length of the compressed content
+     *     afterthe access code
+     */
+    static function checkPosition($git_blob_position, $git_tree_position,
+        $git_object_content)
+    {
+        $git_object_positions = array();
+        if(is_bool($git_blob_position) === true) {
+            $git_blob_position = strlen($git_object_content);
+        }
+        if(is_bool($git_tree_position) === true) {
+            $git_tree_position = strlen($git_object_content);
+        }
+        $git_object_positions['blob'] = $git_blob_position;
+        $git_object_positions['tree'] = $git_tree_position;
+        return $git_object_positions;
+    }
+
+    /**
+     * Get the details of the blob file i.e blob file name, sha hash and content
+     *
+     * @param string $git_object_content compressed content of git master tree
+     * @param string $blob_position first occuence of git blob access code
+     *     in $content
+     * @param string $length length of the compressed content of git master tree
+     * @param string $git_base_url common portion of git url
+     * @return array $git_blob_content contains details of git blob object
+     */
+    static function readBlobSha($git_object_content, $blob_position, $length,
+        $git_base_url)
+    {
+        $git_blob_content = array();
+        $blob_values = array();
+        $temp_git_content = substr($git_object_content, $blob_position,
+            $length);
+        $access_code = substr($temp_git_content, self::BLOB_ACCESS_CODE_START,
+            self::BLOB_ACCESS_CODE_END);
+        $blob_values['code'] = $access_code;
+        $temp_git_content = substr($temp_git_content, self::GIT_BLOB_NEXT,
+            $length);
+        $temp_position = strpos($temp_git_content, self::HEX_NULL_CHARACTER);
+        $file_name = substr($temp_git_content, self::GIT_NAME_START,
+            $temp_position);
+        $blob_values['name'] = $file_name;
+        $temp_git_content = substr($temp_git_content, $temp_position + 1,
+            $length);
+        $sha_binary = substr($temp_git_content, self::SHA_HASH_BINARY_START,
+            self::SHA_HASH_BINARY_END);
+        $sha_hash = bin2hex($sha_binary);
+        $blob_values['hash'] = $sha_hash;
+        $temp_git_content = substr($temp_git_content, self::SHA_HASH_BINARY_END,
+            $length);
+        $blob_url = self::urlMaker($sha_hash, $git_base_url);
+        $blob_values['url'] = $blob_url;
+        $blob_values['indicator'] = self::GIT_BLOB_OBJECT;
+        $git_blob_content['value'] = $blob_values;
+        $git_blob_content['content'] = $temp_git_content;
+        $git_blob_content['indicator'] = self::GIT_BLOB_OBJECT;
+        return $git_blob_content;
+    }
+
+    /**
+     * Get the details of the tree file i.e folder name, sha hash and
+     * blob url inside the tree
+     *
+     * @param string $git_object_content compressed content of git master tree
+     * @param string $tree_position first occuence of git tree access code in
+     *  the $content
+     * @param string $length length of the compressed content of git master tree
+     * @param string $git_base_url common portion of git url
+     * @return array $git_tree_content contains details of git blob object
+     */
+    static function readTreeSha($git_object_content, $tree_position, $length,
+        $git_base_url)
+    {
+        $git_tree_content = array();
+        $tree_values = array();
+        $temp_git_content = substr($git_object_content, $tree_position,
+            $length);
+        $access_code = substr($temp_git_content, self::TREE_ACCESS_CODE_START,
+            self::TREE_ACCESS_CODE_END);
+        $tree_values['code'] = $access_code;
+        $temp_git_content = substr($temp_git_content, self::GIT_TREE_NEXT,
+            $length);
+        $temp_position = strpos($temp_git_content, self::HEX_NULL_CHARACTER);
+        $folder_name = substr($temp_git_content, self::GIT_NAME_START,
+            $temp_position);
+        $tree_values['name'] = $folder_name;
+        $temp_git_content = substr($temp_git_content, $temp_position + 1,
+            $length);
+        $sha_binary = substr($temp_git_content, self::SHA_HASH_BINARY_START,
+            self::SHA_HASH_BINARY_END);
+        $sha_hash = bin2hex($sha_binary);
+        $tree_values['hash'] = $sha_hash;
+        $tree_values['indicator'] = self::GIT_TREE_OBJECT;
+        $temp_git_content = substr($temp_git_content, self::SHA_HASH_BINARY_END,
+            $length);
+        $blob_url = self::checkNestedStructure($sha_hash, $git_base_url);
+        $git_tree_content['value'] = $tree_values;
+        $git_tree_content['content'] = $temp_git_content;
+        $git_tree_content['indicator'] = $blob_url;
+        return $git_tree_content;
+    }
+
+    /**
+     * Checks the nested structure inside git tree object
+     *
+     * @param string $sha_hash sha of the git tree object
+     * @param string $git_base_url common portion of the parent git url
+     * @return string $blob_url contains url of the blob file inside the folder
+     */
+    static function checkNestedStructure($sha_hash, $git_base_url)
+    {
+        $url = self::urlMaker($sha_hash, $git_base_url);
+        $git_compressed_content = self::getGitData($url);
+        $git_uncompressed_content = gzuncompress($git_compressed_content);
+        $blob_url = self::getObjects($git_uncompressed_content, $git_base_url);
+        return $blob_url;
+    }
+
+    /**
+     * Makes the git clone internal url for blob objects
+     *
+     * @param string $sha_hash of the git blob object
+     * @param string $git_base_url common portion of git url
+     * @return string $git_object_url contains the complete url of the blob file
+     */
+    static function urlMaker($sha_hash, $git_base_url)
+    {
+        $git_object_folder = substr($sha_hash, self::GIT_FOLDER_NAME_START,
+            self::GIT_FOLDER_NAME_END);
+        $git_object_file = substr($sha_hash, self::GIT_FILE_NAME_START,
+            self::GIT_FILE_NAME_END);
+        $git_object_path = $git_object_folder . self::GIT_URL_SPLIT .
+            $git_object_file;
+        $git_object_url = $git_base_url . self::GIT_URL_OBJECT .
+            $git_object_path;
+        return $git_object_url;
+    }
+
+    /**
+     * Makes the cURL call to get the contents
+     *
+     * @param string $git_url url to dowmload the contents
+     * @return string $git_content actual content of the git url
+     */
+    static function getGitData($git_url)
+{
+        $ch = curl_init();
+        $timeout = self::CURL_TIMEOUT;
+        curl_setopt($ch, CURLOPT_URL, $git_url);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, self::CURL_TRANSFER);
+        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
+        $git_content = curl_exec($ch);
+        curl_close($ch);
+        return $git_content;
+    }
+}
+?>
diff --git a/lib/locale_functions.php b/lib/locale_functions.php
old mode 100644
new mode 100755
index 4557ee0..85f15f4
--- a/lib/locale_functions.php
+++ b/lib/locale_functions.php
@@ -106,6 +106,7 @@ function guessLocale()
  */
 function guessLocaleFromString($phrase_string, $locale_tag = NULL)
 {
+    $query_string = $phrase_string;
     $locale_tag = ($locale_tag == NULL) ? getLocaleTag() : $locale_tag;
     $phrase_string = mb_ereg_replace(PUNCT."|[0-9]|\s", "", $phrase_string);
     $phrase_string = mb_convert_encoding($phrase_string, "UTF-32", "UTF-8");
@@ -150,8 +151,38 @@ function guessLocaleFromString($phrase_string, $locale_tag = NULL)
             }
         }
     }
+    if($locale_tag == 'en-US') {
+        $locale_tag = checkQuery($query_string);
+    }
     return $locale_tag;
 }
+
+/**
+ * Tries to find wether query belongs to a programming language
+ *
+ *  @param string $query query entered by user
+ *
+ *  @return string $lang programming language for the the query provided
+ */
+function checkQuery($query)
+{
+    $programming_language_map = array('java:' => 'java', 'python:' => 'py');
+    $control_word = "/^(java:|python:)/";
+    $position = preg_match($control_word, trim($query),
+        $matches, PREG_OFFSET_CAPTURE);
+    if(isset($matches[0][0])) {
+        $matched_word = $matches[0][0];
+        if(isset($programming_language_map[$matched_word])) {
+            $lang = $programming_language_map[$matched_word];
+        } else {
+            $lang = 'en-US';
+        }
+    } else {
+        $lang = 'en-US';
+    }
+    return $lang;
+}
+
 /**
  * Tries to guess at a language tag based on the name of a character
  * encoding
diff --git a/lib/phrase_parser.php b/lib/phrase_parser.php
index bf204a8..1c7e3e6 100755
--- a/lib/phrase_parser.php
+++ b/lib/phrase_parser.php
@@ -100,6 +100,28 @@ class PhraseParser
     static $materialized_metas = array("media:", "safe:", "class:");
 
     /**
+     * A list of meta words that might be extracted from a query
+     * @var array
+     */
+    static $programming_language_map = array('java' => 'java',
+            'py' => 'python');
+
+    /**
+     * Constant storing the string
+     */
+    const TOKENIZER = 'Tokenizer';
+
+    /**
+     * Indicates the control word for programming languages
+     */
+    const CONTROL_WORD_INDICATOR = ':';
+
+    /**
+     * Indicates the control word for programming languages
+     */
+    const REGEX_INITIAL_POSITION = 1;
+
+    /**
      * Converts a summary of a web page into a string of space separated words
      *
      * @param array $page associative array of page summary data. Contains
@@ -141,9 +163,14 @@ class PhraseParser
     static function extractPhrases($string, $lang = NULL, $index_name = NULL,
         $exact_match = false, $threshold = 10)
     {
-        self::canonicalizePunctuatedTerms($string, $lang);
+        if(isset(self::$programming_language_map[$lang])) {
+            $control_word = self::$programming_language_map[$lang] .
+                self::CONTROL_WORD_INDICATOR;
+            $string = trim(substr($string, strlen($control_word) + 1));
+        } else {
+            self::canonicalizePunctuatedTerms($string, $lang);
+        }
         $terms = self::stemCharGramSegment($string, $lang);
-
         $num = count($terms);
         if($index_name == NULL || $num <= 1) {
             return $terms;
@@ -233,7 +260,9 @@ class PhraseParser
      */
     static function extractPhrasesInLists($string, $lang = NULL)
     {
-        self::canonicalizePunctuatedTerms($string, $lang);
+        if(!isset(self::$programming_language_map[$lang])) {
+            self::canonicalizePunctuatedTerms($string, $lang);
+        }
         return self::extractMaximalTermsAndFilterPhrases($string, $lang);
     }
 
@@ -341,16 +370,264 @@ class PhraseParser
      */
     static function stemCharGramSegment($string, $lang)
     {
-        mb_internal_encoding("UTF-8");
-        $string = mb_strtolower($string);
-        $string = mb_ereg_replace("\s+|".PUNCT, " ", $string);
-        $terms = self::segmentSegment($string, $lang);
-        $terms = self::charGramTerms($terms, $lang);
-        $terms = self::stemTerms($terms, $lang);
+        if(isset(self::$programming_language_map[$lang])) {
+            mb_internal_encoding("UTF-8");
+            $tokenizer_name = self::$programming_language_map[$lang] .
+                self::TOKENIZER;
+            $terms = self::$tokenizer_name($string, $lang);
+        } else {
+            mb_internal_encoding("UTF-8");
+            $string = mb_strtolower($string);
+            $string = mb_ereg_replace("\s+|".PUNCT, " ", $string);
+            $terms = self::segmentSegment($string, $lang);
+            $terms = self::charGramTerms($terms, $lang);
+            $terms = self::stemTerms($terms, $lang);
+        }
         return $terms;
     }
 
     /**
+     * Given a string tokenizes into Java tokens
+     *
+     * @param string $string what to extract terms from
+     * @param string $lang indicates programming language
+     *
+     * @return array the terms computed from the string
+     */
+    static function javaTokenizer($string, $lang)
+    {
+        //Comments
+        $single_line_comments = "(\/\/).*?(\n)";
+        $multiline_comments = "\\/\\*[^(\\/\\*)]*?\\*\\/";
+        $javadoc_comments = "\/\*([^*]|[\r\n]|(\*+([^*\/]|[\r\n])))*\*+\/";
+        $multiple_line_comments = "$javadoc_comments|$multiline_comments";
+        $comments = "($multiple_line_comments|$single_line_comments)";
+        //Identifiers
+        $alphabetic = "[A-Za-z]";
+        $id_start = "($alphabetic)|\\".'$'."|\_";
+        $numeric = "[0-9]";
+        $repeat = "$id_start|$numeric";
+        $identifiers = "($id_start)($repeat)*";
+        //Keywords
+        $keywords_part1 = "abstract|assert|boolean|break|byte|case|catch|char";
+        $keywords_part2 = "class|const|continue|default|do|double|else|extends";
+        $keywords_part3 = "final|finally|float|for|goto|if|implements|import";
+        $keywords_part4 = "instanceof|int|interface|long|native|new|package";
+        $keywords_part5 = "private|protected|public|return|short|static";
+        $keywords_part6 = "strictfp|super|synchronized|switch|this|throw";
+        $keywords_part7 = "throws|transient|try|void|volatile|while";
+        $keywords_string1 = "$keywords_part1|$keywords_part2|$keywords_part3";
+        $keywords_string2 = "$keywords_part4|$keywords_part5|$keywords_part6";
+        $keywords_string3 = "$keywords_part7";
+        $keywords = "($keywords_string1|$keywords_string2|$keywords_string3)";
+        //Seperators
+        $seperators = "(;|,|\.|\(|\)|\{|\}|\[|\])";
+        //Operators
+        $operators_part1 = "\+|\-|\*|\/|&|\||\^|%|<<|>>|=|>|<|!|~|\?|:";
+        $operators_part2 = "\-\-|\+\+|>>>|==|<=|>=|!=|&&|\|\|";
+        $operators_part3 = "\+=|\-=|\*=|\/=|&=|\|=|\^=|%=|<<=|>>=|>>>=";
+        $operators = "($operators_part3|$operators_part2|$operators_part1)";
+        //Null Literal
+        $null_literal = "null";
+        //Boolean Literal
+        $boolean_literal = "true|false";
+        //Floating point Literal
+        $non_zero_digit = "1|2|3|4|5|6|7|8|9";
+        $digit = "0|$non_zero_digit";
+        $digits = "($digit)($digit)*";
+        $exponent_part = "(e|E)([\+|\-])($digits)";
+        $float_part1 = "($digits)($exponent_part)";
+        $float_part2 = "($digits)(\.)($digits)?($exponent_part)?";
+        $float_part3 = "(\.$digits)($exponent_part)?";
+        $floating_point_numeral = "$float_part1|$float_part2|$float_part3";
+        //Integer Literal
+        $decimal_numeral = "0|($non_zero_digit)($digits){0,1}";
+        $hex_numeral = "0[x|X][0-9A-Fa-f]+";
+        $octal_numeral = "0[0-7]+";
+        $integer_numeral = "($hex_numeral|$octal_numeral|$decimal_numeral)";
+        //Character Literal
+        $special_part1 = "\!|%|\^|&|\*|\(|\)|\-|\+|\=|\{|\}|\||~|\[|\]|\\|;";
+        $special_part2 = "'|\:|\<|\>|\?|,|\.|\/|#|@|`|_";
+        $special = "$special_part1|$special_part2";
+        $alphanumeric = "[A-Za-z0-9]";
+        $graphic = "$alphanumeric|$special";
+        $escape = "\\n|\\t|\\v|\\a|\\b|\\r|\\f|\\\\|\\'|\\\"";
+        $char_literal = "(\'($graphic)\'|\'\s\'|\'($escape)\')";
+        //String Literal
+        $string_literal = "(\"($graphic|\s|$escape)*?[^\\\]\")";
+        //Literals
+        $literals_part1 = "$string_literal|$floating_point_numeral";
+        $literals_part2 = "$integer_numeral|$char_literal|$boolean_literal";
+        $literals_part3 = "$null_literal";
+        $literals = "($literals_part1|$literals_part2|$literals_part3)";
+        //Java Tokens
+        $tokens_part1 = "$comments|$literals|$operators";
+        $tokens_part2 = "$seperators|$keywords|$identifiers";
+        $tokens = "($tokens_part1|$tokens_part2)";
+        $length = strlen($string);
+        $current_length = $length;
+        $position = self::REGEX_INITIAL_POSITION;
+        $results = array();
+        while($position == 1 && $current_length > 0) {
+            $temp_results = array();
+            $position = preg_match("/$tokens/", $string, $matches,
+                PREG_OFFSET_CAPTURE);
+            if(isset($matches[0][0])) {
+                $text = preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/", "\n",
+                    $matches[0][0]);
+                $lines = explode("\n", trim($text));
+                $line = implode(' ', $lines);
+                $data = preg_replace("/[\t\s]+/", ' ', trim($line));
+                $temp_results = explode(" ", trim($data));
+                foreach($temp_results as $result) {
+                    if(!empty($result)) {
+                        $results[] = self::$programming_language_map[$lang] .
+                            self::CONTROL_WORD_INDICATOR . trim($result);
+                    }
+                }
+                $current_length = (strlen($matches[0][0]));
+                $string = trim(substr($string, $current_length, $length));
+            }
+        }
+        return $results;
+    }
+
+    /**
+     * Given a string tokenizes into Python tokens
+     *
+     * @param string $string what to extract terms from
+     * @param string $lang indicates programming language
+     *
+     * @return array the terms computed from the string
+     */
+    static function pythonTokenizer($string, $lang)
+    {
+        //Comments
+        $ordinary_part1 = "_|\(|\)|\[|\]|\{|\}|\+|\-|\*|\/|%";
+        $ordinary_part2 = "\!|&|\||\^|~|\<|\=|\>|,|\.|\:|;|$|\?|#|\@";
+        $ordinary = "$ordinary_part1|$ordinary_part2";
+        $lower = "a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z";
+        $upper = "A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z";
+        $digit = "0|1|2|3|4|5|6|7|8|9";
+        $graphic = "$lower|$upper|$digit|$ordinary";
+        $text_chars = "$graphic|\s|\"|\'";
+        $comments = "#($text_chars|\\\)*?(\n)";
+        //Identifiers
+        $id_start = "$lower|$upper|\_";
+        $repeat = "$id_start|$digit";
+        $identifiers = "($id_start)($repeat)*";
+        //Keywords
+        $keywords_part1 = "False|None|True|and|as|assert|break|class|continue";
+        $keywords_part2 = "finally|for|from|global|elif|import|in|def|del|if";
+        $keywords_part3 = "is|lambda|nonlocal|not|or|pass|raise|else|except";
+        $keywords_part4 = "return|try|while|with|yield";
+        $keywords_string1 = "$keywords_part1|$keywords_part2";
+        $keywords_string2 = "$keywords_part3|$keywords_part4";
+        $keywords = "($keywords_string1|$keywords_string2)";
+        //Operators
+        $operators_part1 = "is|in|or|not|and|\+|\-|\*|\/|%|<|>|&";
+        $operators_part2 = "\*\*|==|!=|<=|>=|\/\/";
+        $operators_part3 = "<<|>>|\^|~|\|";
+        $operators = "($operators_part3|$operators_part2|$operators_part1)";
+        //Delimiters
+        $delimiters_part1 = "\.|,|:|;|@|=|\(|\)|\{|\}|\[|\]";
+        $delimiters_part2 = "\+=|\-=|\*=|\/=|\/\/=|%=|\*\*=";
+        $delimiters_part3 = "&=|\|=|\^=|<<=|>>=";
+        $delimiters = "$delimiters_part3|$delimiters_part2|$delimiters_part1";
+        //Floating point Literal
+        $digits = "($digit)($digit)*";
+        $mantissa = "($digits)\.($digit)*|\.($digits)";
+        $exponent = "e[\+|\-]$digits|E[\+|\-]$digits";
+        $float_literal = "($mantissa)($exponent)*|($digits)($exponent)";
+        //Integer Literal
+        $non_zero_digit = "1|2|3|4|5|6|7|8|9";
+        $binary_digit = "0|1";
+        $octal_digit = "0|1|2|3|4|5|6|7";
+        $hex_digit = "$digit|a|b|c|d|e|f|A|B|C|D|E|F";
+        $decimal_literal = "0+|($non_zero_digit)($digit)*";
+        $binary_literal_part1 = "0b($binary_digit)($binary_digit)*";
+        $binary_literal_part2 = "0B($binary_digit)($binary_digit)*";
+        $binary_literal = "$binary_literal_part1|$binary_literal_part2";
+        $octal_literal_part1 = "0O($octal_digit)($octal_digit)*";
+        $octal_literal_part2 = "0o($octal_digit)($octal_digit)*";
+        $octal_literal = "$octal_literal_part1|$octal_literal_part2";
+        $hex_literal_part1 ="0X($hex_digit)($hex_digit)*";
+        $hex_literal_part2 ="0x($hex_digit)($hex_digit)*";
+        $hex_literal = "$hex_literal_part1|$hex_literal_part2";
+        $integer_literal_part1 = "($binary_literal)|($octal_literal)";
+        $integer_literal_part2 = "($hex_literal)|($decimal_literal)";
+        $integer_literal = "$integer_literal_part1|$integer_literal_part2";
+        //Boolean Literal
+        $boolean_literal = "True|False";
+        //None Type Literal
+        $none_literal = "None";
+        //String Literal
+        $esc_a = "\\\o[$octal_digit]{3}|\\\h[$hex_digit]{2}|\\\[$text_chars]";
+        $unicode = "[^\\x00-\\x80]+";
+        $esc_u = "$esc_a|\\\n$unicode|\\\u[$hex_digit]{4}|\\\U[$hex_digit]{8}";
+        $raw_opt = "r|R";
+        $bytes_opt = "b|B";
+        $single_quoted_element1 = "($graphic|$esc_u|\\s|\\t|\')*";
+        $single_quoted_element2 = "($graphic|$esc_u|\\s|\\t|\")*";
+        $single_quoted_string1 = "(\"$single_quoted_element1\")";
+        $single_quoted_string2 = "(\'$single_quoted_element2\')";
+        $single_quoted_string = "$single_quoted_string1|$single_quoted_string2";
+        $triple_quoted_element = "$text_chars|$esc_u";
+        $triple_quoted_string1 = "(\"\"\"($triple_quoted_element)*?\"\"\")";
+        $triple_quoted_string2 = "(\'\'\'($triple_quoted_element)*?\'\'\')";
+        $triple_quoted_string = "$triple_quoted_string1|$triple_quoted_string2";
+        $string_literal_part1 = "($raw_opt)?($triple_quoted_string)";
+        $string_literal_part2 = "($raw_opt)?($single_quoted_string)";
+        $string_literal = "$string_literal_part1|$string_literal_part2";
+        //Byte Literal
+        $single_quoted_element3 = "($graphic|$esc_a|\\s|\\t|\')*";
+        $single_quoted_element4 = "($graphic|$esc_a|\\s|\\t|\")*";
+        $single_quoted_byte1 = "(\"$single_quoted_element3\")";
+        $single_quoted_byte2 = "(\'$single_quoted_element4\')";
+        $single_quoted_byte = "$single_quoted_byte1|$single_quoted_byte2";
+        $triple_quoted_byte1 = "(\"\"\"($triple_quoted_element)*?\"\"\")";
+        $triple_quoted_byte2 = "(\'\'\'($triple_quoted_element)*?\'\'\')";
+        $triple_quoted_byte = "$triple_quoted_byte1|$triple_quoted_byte2";
+        $bytes_literal_part1 = "($bytes_opt)($raw_opt)?($triple_quoted_byte)";
+        $bytes_literal_part2 = "($bytes_opt)($raw_opt)?($single_quoted_byte)";
+        $bytes_literal = "$bytes_literal_part1|$bytes_literal_part2";
+        //Literals
+        $literals_part1 = "$string_literal|$bytes_literal|$float_literal";
+        $literals_part2 = "$integer_literal|$boolean_literal|$none_literal";
+        $literals = "($literals_part1|$literals_part2)";
+        //Python Tokens
+        $tokens_part1 = "$comments|$literals|$delimiters";
+        $tokens_part2 = "$operators|$keywords|$identifiers";
+        $tokens = "($tokens_part1|$tokens_part2)";
+        $length = strlen($string);
+        $current_length = $length;
+        $position = self::REGEX_INITIAL_POSITION;
+        $results = array();
+        while($position == 1 && $current_length > 0) {
+            $temp_results = array();
+            $position = preg_match("/$tokens/", $string, $matches,
+                PREG_OFFSET_CAPTURE);
+            if(isset($matches[0][0])) {
+                $text = preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/", "\n",
+                    $matches[0][0]);
+                $lines = explode("\n", trim($text));
+                $line = implode(' ', $lines);
+                $data = preg_replace("/[\t\s]+/", ' ', trim($line));
+                $temp_results = explode(" ", trim($data));
+                foreach($temp_results as $result) {
+                    if(!empty($result)) {
+                        $results[] = self::$programming_language_map[$lang] .
+                            self::CONTROL_WORD_INDICATOR . trim($result);
+                    }
+                }
+                $current_length = (strlen($matches[0][0]));
+                $string = trim(substr($string, $current_length, $length));
+            }
+        }
+        return $results;
+    }
+
+    /**
      * Given an array of pre_terms returns the characters n-grams for the 
      * given terms where n is the length Yioop uses for the language in 
      * question. If a stemmer is used for language then n-gramming is not 
diff --git a/lib/processors/java_processor.php b/lib/processors/java_processor.php
new file mode 100755
index 0000000..6845db7
--- /dev/null
+++ b/lib/processors/java_processor.php
@@ -0,0 +1,208 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2013  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Snigdha Rao Parvatneni
+ * @package seek_quarry
+ * @subpackage processor
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2013
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** Register File Types We Handle*/
+$add_extensions = array("java");
+$INDEXED_FILE_TYPES = array_merge($INDEXED_FILE_TYPES, $add_extensions);
+$add_types = array(
+    "text/java" => "JavaProcessor"
+);
+$PAGE_PROCESSORS =  array_merge($PAGE_PROCESSORS, $add_types);
+
+/**
+ * Load the base class
+ */
+require_once BASE_DIR."/lib/processors/page_processor.php";
+
+/**
+ * So can extract parts of the URL if need to guess lang
+ */
+require_once BASE_DIR."/lib/url_parser.php";
+
+
+/**
+ * Parent class common to all processors used to create crawl summary
+ * information  that involves basically text data
+ *
+ * @author Chris Pollett
+ * @package seek_quarry
+ * @subpackage processor
+ */
+class JavaProcessor extends TextProcessor
+{
+    /**
+     * Computes a summary based on a text string of a document
+     *
+     * @param string $page text string of a document
+     * @param string $url location the document came from, not used by
+     *      JavaProcessor at this point. Some of its subclasses override
+     *      this method and use url to produce complete links for
+     *      relative links within a document
+     *
+     * @return array a summary of (title, description,links, and content) of
+     *      the information in $page
+     */
+    function process($page, $url)
+    {
+        $summary = NULL;
+
+        if(is_string($page)) {
+            $summary[self::TITLE] = "";
+            $summary[self::DESCRIPTION] = $page;
+            $summary[self::LANG] = self::calculateLang(
+                $summary[self::DESCRIPTION], $url);
+            $summary[self::LINKS] = self::extractHttpHttpsUrls($page);
+            $summary[self::PAGE] = "<html><body><div><pre>".
+                strip_tags($page)."</pre></div></body></html>";
+        }
+        return $summary;
+    }
+
+    /**
+     *  Tries to determine the language of the document by looking at the
+     *  $sample_text and $url provided
+     *  the language
+     *  @param string $sample_text sample text to try guess the language from
+     *  @param string $url url of web-page as a fallback look at the country
+     *      to figure out language
+     *
+     *  @return string language tag for guessed language
+     */
+    static function calculateLang($sample_text = NULL, $url = NULL)
+    {
+        if($url != NULL) {
+            $lang = UrlParser::getDocumentType($url);
+        }
+        return $lang;
+    }
+
+    /**
+     * Gets the text between two tags in a document starting at the current
+     * position.
+     *
+     * @param string $string document to extract text from
+     * @param int $cur_pos current location to look if can extract text
+     * @param string $start_tag starting tag that we want to extract after
+     * @param string $end_tag ending tag that we want to extract until
+     * @return array pair consisting of when in the document we are after
+     *      the end tag, together with the data between the two tags
+     */
+    static function getBetweenTags($string, $cur_pos, $start_tag, $end_tag)
+    {
+        $len = strlen($string);
+        if(($between_start = strpos($string, $start_tag, $cur_pos)) ===
+            false ) {
+            return array($len, "");
+        }
+
+        $between_start  += strlen($start_tag);
+        if(($between_end = strpos($string, $end_tag, $between_start)) ===
+            false ) {
+            $between_end = $len;
+        }
+
+        $cur_pos = $between_end + strlen($end_tag);
+
+        $between_string = substr($string, $between_start,
+            $between_end - $between_start);
+        return array($cur_pos, $between_string);
+
+    }
+
+    /**
+     * Tries to extract http or https links from a string of text.
+     * Does this by a very approximate regular expression.
+     *
+     * @param string $page text string of a document
+     * @return array a set of http or https links that were extracted from
+     *      the document
+     */
+    static function extractHttpHttpsUrls($page)
+    {
+        $pattern =
+            '@((http|https)://([^ \t\r\n\v\f\'\"\;\,<>\{\}])*)@i';
+        $sites = array();
+        preg_match_all($pattern, $page, $matches);
+        $i = 0;
+        foreach($matches[0] as $url) {
+            if(!isset($sites[$url]) && strlen($url) < MAX_URL_LENGTH &&
+                strlen($url) > 4) {
+                $sites[$url] = preg_replace("/\s+/", " ", strip_tags($url));
+                $i++;
+                if($i >= MAX_LINKS_TO_EXTRACT) {break;}
+            }
+        }
+        return $sites;
+    }
+
+    /**
+     * If an end of file is reached before closed tags are seen, this methods
+     * closes these tags in the correct order.
+     *
+     * @param string &$page a reference to an xml or html document
+     */
+    static function closeDanglingTags(&$page)
+    {
+        $l_pos = strrpos($page, "<");
+        $g_pos = strrpos($page, ">");
+        if($g_pos && $l_pos > $g_pos) {
+            $page = substr($page, 0, $l_pos);
+        }
+        // put all opened tags into an array
+        preg_match_all("#<([a-z]+)( .*)?(?!/)>#iU", $page, $result);
+        $openedtags = $result[1];
+
+        // put all closed tags into an array
+        preg_match_all("#</([a-z]+)>#iU", $page, $result);
+        $closedtags=$result[1];
+        $len_opened = count($openedtags);
+        // all tags are closed
+        if(count($closedtags) == $len_opened){
+            return;
+        }
+
+        $openedtags = array_reverse($openedtags);
+        // close tags
+        for($i=0;$i < $len_opened;$i++) {
+            if (!in_array($openedtags[$i],$closedtags)){
+              $page .= '</'.$openedtags[$i].'>';
+            } else {
+              unset($closedtags[array_search($openedtags[$i],$closedtags)]);
+            }
+        }
+    }
+}
+
+?>
diff --git a/lib/processors/python_processor.php b/lib/processors/python_processor.php
new file mode 100755
index 0000000..3e8ff5e
--- /dev/null
+++ b/lib/processors/python_processor.php
@@ -0,0 +1,209 @@
+<?php
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2013  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Snigdha Rao Parvatneni
+ * @package seek_quarry
+ * @subpackage processor
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2013
+ * @filesource
+ */
+
+if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+/** Register File Types We Handle*/
+$add_extensions = array("py");
+$INDEXED_FILE_TYPES = array_merge($INDEXED_FILE_TYPES, $add_extensions);
+$add_types = array(
+    "text/py" => "PythonProcessor"
+);
+$PAGE_PROCESSORS =  array_merge($PAGE_PROCESSORS, $add_types);
+
+/**
+ * Load the base class
+ */
+require_once BASE_DIR."/lib/processors/page_processor.php";
+
+/**
+ * So can extract parts of the URL if need to guess lang
+ */
+require_once BASE_DIR."/lib/url_parser.php";
+
+
+/**
+ * Parent class common to all processors used to create crawl summary
+ * information  that involves basically text data
+ *
+ * @author Chris Pollett
+ * @package seek_quarry
+ * @subpackage processor
+ */
+class PythonProcessor extends TextProcessor
+{
+    /**
+     * Computes a summary based on a text string of a document
+     *
+     * @param string $page text string of a document
+     * @param string $url location the document came from, not used by
+     *      PythonProcessor at this point. Some of its subclasses override
+     *      this method and use url to produce complete links for
+     *      relative links within a document
+     *
+     * @return array a summary of (title, description,links, and content) of
+     *      the information in $page
+     */
+    function process($page, $url)
+    {
+        $summary = NULL;
+
+        if(is_string($page)) {
+            $summary[self::TITLE] = "";
+            $summary[self::DESCRIPTION] = $page;
+            $summary[self::LANG] = self::calculateLang(
+                $summary[self::DESCRIPTION], $url);
+            $summary[self::LINKS] = self::extractHttpHttpsUrls($page);
+            $summary[self::PAGE] = "<html><body><div><pre>".
+                strip_tags($page)."</pre></div></body></html>";
+        }
+        return $summary;
+    }
+
+
+    /**
+     *  Tries to determine the language of the document by looking at the
+     *  $sample_text and $url provided
+     *  the language
+     *  @param string $sample_text sample text to try guess the language from
+     *  @param string $url url of web-page as a fallback look at the country
+     *      to figure out language
+     *
+     *  @return string language tag for guessed language
+     */
+    static function calculateLang($sample_text = NULL, $url = NULL)
+    {
+        if($url != NULL) {
+            $lang = UrlParser::getDocumentType($url);
+        }
+        return $lang;
+    }
+
+    /**
+     * Gets the text between two tags in a document starting at the current
+     * position.
+     *
+     * @param string $string document to extract text from
+     * @param int $cur_pos current location to look if can extract text
+     * @param string $start_tag starting tag that we want to extract after
+     * @param string $end_tag ending tag that we want to extract until
+     * @return array pair consisting of when in the document we are after
+     *      the end tag, together with the data between the two tags
+     */
+    static function getBetweenTags($string, $cur_pos, $start_tag, $end_tag)
+    {
+        $len = strlen($string);
+        if(($between_start = strpos($string, $start_tag, $cur_pos)) ===
+            false ) {
+            return array($len, "");
+        }
+
+        $between_start  += strlen($start_tag);
+        if(($between_end = strpos($string, $end_tag, $between_start)) ===
+            false ) {
+            $between_end = $len;
+        }
+
+        $cur_pos = $between_end + strlen($end_tag);
+
+        $between_string = substr($string, $between_start,
+            $between_end - $between_start);
+        return array($cur_pos, $between_string);
+
+    }
+
+    /**
+     * Tries to extract http or https links from a string of text.
+     * Does this by a very approximate regular expression.
+     *
+     * @param string $page text string of a document
+     * @return array a set of http or https links that were extracted from
+     *      the document
+     */
+    static function extractHttpHttpsUrls($page)
+    {
+        $pattern =
+            '@((http|https)://([^ \t\r\n\v\f\'\"\;\,<>\{\}])*)@i';
+        $sites = array();
+        preg_match_all($pattern, $page, $matches);
+        $i = 0;
+        foreach($matches[0] as $url) {
+            if(!isset($sites[$url]) && strlen($url) < MAX_URL_LENGTH &&
+                strlen($url) > 4) {
+                $sites[$url] = preg_replace("/\s+/", " ", strip_tags($url));
+                $i++;
+                if($i >= MAX_LINKS_TO_EXTRACT) {break;}
+            }
+        }
+        return $sites;
+    }
+
+    /**
+     * If an end of file is reached before closed tags are seen, this methods
+     * closes these tags in the correct order.
+     *
+     * @param string &$page a reference to an xml or html document
+     */
+    static function closeDanglingTags(&$page)
+    {
+        $l_pos = strrpos($page, "<");
+        $g_pos = strrpos($page, ">");
+        if($g_pos && $l_pos > $g_pos) {
+            $page = substr($page, 0, $l_pos);
+        }
+        // put all opened tags into an array
+        preg_match_all("#<([a-z]+)( .*)?(?!/)>#iU", $page, $result);
+        $openedtags = $result[1];
+
+        // put all closed tags into an array
+        preg_match_all("#</([a-z]+)>#iU", $page, $result);
+        $closedtags=$result[1];
+        $len_opened = count($openedtags);
+        // all tags are closed
+        if(count($closedtags) == $len_opened){
+            return;
+        }
+
+        $openedtags = array_reverse($openedtags);
+        // close tags
+        for($i=0;$i < $len_opened;$i++) {
+            if (!in_array($openedtags[$i],$closedtags)){
+              $page .= '</'.$openedtags[$i].'>';
+            } else {
+              unset($closedtags[array_search($openedtags[$i],$closedtags)]);
+            }
+        }
+    }
+}
+
+?>
diff --git a/models/phrase_model.php b/models/phrase_model.php
index 8acf0ee..982aded 100755
--- a/models/phrase_model.php
+++ b/models/phrase_model.php
@@ -85,6 +85,18 @@ class PhraseModel extends ParallelModel
     var $query_info;
 
     /**
+     * Used to hold extension of programming language which is used the language
+     * @var string
+     */
+    var $programming_language_map;
+
+   /**
+     * A indicator to indicate source code files
+     * @var string
+     */
+    var $program_indicator;
+
+    /**
      * Number of pages to cache in one go in memcache or filecache
      * Size chosen based on 1MB max object size for memcache or filecache
      */
@@ -95,6 +107,8 @@ class PhraseModel extends ParallelModel
     function __construct($db_name = DB_NAME)
     {
         parent::__construct($db_name);
+        $this->programming_language_map = array('java' => 'java', 'py' => 'py');
+        $this->program_indicator = false;
     }
 
     /**
@@ -476,6 +490,7 @@ class PhraseModel extends ParallelModel
      */
     function parseWordStructConjunctiveQuery(&$phrase, $guess_semantics = true)
     {
+        $query = $phrase;
         $indent= "&nbsp;&nbsp;";
         $in2 = $indent . $indent;
         $in3 = $in2 . $indent;
@@ -496,7 +511,10 @@ class PhraseModel extends ParallelModel
             snippets in the results by bolding either
          */
         $query_words = explode(" ", $query_string); //not stemmed
-
+        if($this->program_indicator) {
+            $query_string = $query;
+            $this->program_indicator = false;
+        }
         $locale_tag = guessLocaleFromString($query_string);
 
         $quote_state = false;
@@ -825,6 +843,9 @@ class PhraseModel extends ParallelModel
         }
 
         $tag = guessLocaleFromString($phrase);
+        if(isset($this->programming_language_map[$tag])) {
+            $this->program_indicator = true;
+        }
         $main_tag = substr($tag, 0, 2);
         if($len == 1) {
             $letter = "";
diff --git a/views/search_view.php b/views/search_view.php
index af65a3f..1523343 100755
--- a/views/search_view.php
+++ b/views/search_view.php
@@ -64,6 +64,11 @@ class SearchView extends View implements CrawlConstants
     var $layout = "web";
 
     /**
+     * Represent extension of Git urls
+     */
+    const GIT_EXTENSION = ".git";
+
+    /**
      *  Draws the main landing pages as well as search result pages
      *
      *  @param array $data  PAGES contains all the summaries of web pages
@@ -215,9 +220,21 @@ class SearchView extends View implements CrawlConstants
                 ?>
 
                 <h2>
-                <a href="<?php  e(htmlentities($link_url));  ?>" rel="nofollow"
-                 <?php if($data["OPEN_IN_TABS"]) {
-                    ?> target="_blank" <?php }?> ><?php
+                <?php
+                    if(strpos($link_url, self::GIT_EXTENSION)) { ?>
+                    <a href="?<?php e(CSRF_TOKEN."=".$data[CSRF_TOKEN]);
+                        ?>&amp;c=search&amp;a=cache&amp;q=<?php
+                        e($data['QUERY']); ?>&amp;arg=<?php
+                        e(urlencode($url));
+                        ?>&amp;its=<?php e($page[self::CRAWL_TIME]);?>
+                        &amp;repository=git"
+                        rel='nofollow'>
+                <?php } else { ?>
+                    <a href="<?php  e(htmlentities($link_url));  ?>"
+                        rel="nofollow"
+                        <?php if($data["OPEN_IN_TABS"]) {
+                        ?> target="_blank" <?php }?> >
+                <?php }
                  if(isset($page[self::THUMB]) && $page[self::THUMB] != 'NULL') {
                     ?><img src="<?php e($page[self::THUMB]); ?>" alt="<?php
                         e($title); ?>"  /> <?php
-- 
1.8.2

