_binary From 735d5540b442c448d3589e8e05440b70c2c1026a Mon Sep 17 00:00:00 2001
From: Akshat Kukreti <akshatkukreti@gmail.com>
Date: Wed, 22 May 2013 20:03:06 -0700
Subject: [PATCH 1/5] crawl constants for cache page validators

---
 lib/btree.php           | 756 ++++++++++++++++++++++++++++++++++++++++++++++++
 lib/btree_node.php      |  87 ++++++
 lib/crawl_constants.php |   3 +
 tests/btree_test.php    | 225 ++++++++++++++
 4 files changed, 1071 insertions(+)
 create mode 100644 lib/btree.php
 create mode 100644 lib/btree_node.php
 create mode 100644 tests/btree_test.php

diff --git a/lib/btree.php b/lib/btree.php
new file mode 100644
index 0000000..b2442e3
--- /dev/null
+++ b/lib/btree.php
@@ -0,0 +1,756 @@
+<?php
+
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2013  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @package seek_quarry
+ * @subpackage bin
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2013
+ * @filesource
+ */
+
+/**
+ * Load Node class
+ */
+require_once(BASE_DIR.'/lib/btree_node.php');
+
+/**
+ * Default value of minimum degree. The minimum degree determines the minimum 
+ * and maximum number of keys and child nodes, for nodes 
+ * other than root node
+ */
+define('MIN_DEGREE', 501);
+
+/**
+ * This class implements the B-Tree data structure for storing int key based 
+ * key-value pairs based on the algorithms in Introduction To Algorithms,
+ * by T.H. Cormen, C.E. Leiserson, R.L. Rivest, and C. Stein. Second 
+ * Edition, 2001, The MIT Press
+ *
+ *  @author Akshat Kukreti
+ *  @package seek_quarry
+ */
+class BTree
+{
+    /**
+     * Minimum degree of the B-Tree. Used in determining the minimum/maximum
+     * keys and links a B-Tree node may have.
+     * minimum_keys = minimum_degree - 1
+     * minimum_links = minimum_keys + 1
+     * maximum_keys = 2 * minimum_degree - 1
+     * maximum_links = maximum_keys + 1
+     * @var int
+     */
+    var $min_degree;
+
+    /**
+     * Storage for root node of the B-Tree
+     * @var object
+     */
+    var $root;
+
+    /**
+     * Counter for node Ids
+     * @var int
+     */
+    var $id_count;
+
+    /**
+     * Directory for storing the B-Tree files
+     * @var string
+     */
+    var $dir;
+
+    /**
+     * Creates/Loads B-Tree having specified directory and minimum_degree. The
+     * default minimum_degree is 501.
+     * @param string $dir is the directory for storing the B-Tree files
+     */
+    function __construct($dir, $min_degree = MIN_DEGREE)
+    {
+        $this->dir = $dir;
+        $this->min_degree = $min_degree;
+        if(!is_dir($this->dir)) {
+            mkdir($this->dir);
+            chmod($this->dir, 0777);
+        }
+        $root_file = $this->dir."/root.txt";
+        if(file_exists($root_file)) {
+            $this->root = unserialize(file_get_contents($root_file));
+            $this->id_count = unserialize(file_get_contents($this->dir.
+                "/count.txt"));
+        } else {
+            $this->root = new Node();
+            $this->root->id = "root";
+            $this->id_count = 1;
+        }
+    }
+
+    /**
+     * Reads node from file saved on disk
+     * @param int $id is the Id of the node to be read
+     * @return object $node is the node
+     */
+    function readNode($id)
+    {
+        $node_file = $this->dir."/$id.txt";
+        if(file_exists($node_file)) {
+            $node = unserialize(file_get_contents($node_file));
+            return $node;
+        } else {
+            exit("Could not read node $id from disk");
+        }
+    }
+
+    /**
+     * Writes node to disk
+     * @param object $node is the node to be written to disk
+     */
+    function writeNode($node)
+    {
+        $node_file = $this->dir."/$node->id.txt";
+        $contents = serialize($node);
+        file_put_contents($node_file, $contents);
+        chmod($node_file, 0777);
+    }
+
+    /**
+     * Deletes file associated with given node from disk
+     * @param int $id is the id of the node whose file is to be deleted
+     */
+    function deleteNodeFile($id)
+    {
+        $node_file = $this->dir."/$id.txt";
+        if(file_exists($node_file)) {
+            unlink($node_file);
+        } else exit("Could not delete node $id from disk");
+    }
+
+    /**
+     * Saves value of node id counter
+     * @param int $count is the id counter
+     */
+    function saveNodeCount()
+    {
+        $count_file = $this->dir."/count.txt";
+        $node_count = serialize($this->id_count);
+        file_put_contents($count_file, $node_count);
+    }
+
+    /**
+     * Deletes the node id count file
+     */
+    function deleteCount()
+    {
+        unlink($this->dir."/count.txt");
+    }
+
+    /**
+     * Returns key-value pair in the B-Tree based on key
+     * @param int $key is the key for whicht the key-value pair is to be 
+     * found
+     * @return array key-value pair associated with $key or null if the 
+     * key-value pair is not found in the tree.
+     */
+    function findValue($key)
+    {
+        list($node, $flag, $pos) = $this->search($this->root, $key);
+        if($pos !== null) {
+            if($flag == 1)
+                return $node->keys[$pos];
+            else return null;
+        } else return null;
+    }
+
+    /**
+     * Searches for key-value pair for a given key in a node. If key value pair 
+     * is not found in the node, recursively searches in the root node of the 
+     * sub-tree till the pair is found. Search stops at leaf nodes.
+     * @param object $node is the B-Tree node from where the search starts
+     * @param int $key is the key for which the key-value pair is to be 
+     * searched
+     */
+    function search($node, $key)
+    {
+        $flag = -1;
+        if(empty($node->keys)) {
+            return array($node, $flag, null);
+        } else {
+            list($flag, $pos) = $this->binarySearch($node->keys, $key);
+            if($flag == 1) {
+                return array($node, $flag, $pos);
+            }
+            if($node->is_leaf == true) {
+                return array($node, $flag, $pos);
+            } else {
+                $next_id = $node->links[$pos];
+                $next_node = $this->readNode($next_id);
+                return $this->search($next_node, $key);
+            }
+        }
+    }
+
+    /**
+     * Inserts a key-value pair in the B-Tree
+     * @param array $pair is the key-value pair to be inserted
+     */
+    function insert($pair)
+    {
+        $node = $this->root;
+        if(empty($node->keys)) {
+            $node->keys = array($pair);
+            $node->count = count($node->keys);
+            $this->writeNode($node);
+            $this->saveNodeCount();
+        } else if($node->count == 2 * $this->min_degree - 1) {
+            $temp = $this->createEmptyParentNode();
+            $this->root = $temp;
+            $this->swap($temp->id, $node->id);
+            $temp->links[0] = $node->id;
+            $this->bTreeSplitChild($temp, 0, $node);
+            $this->insertNodeNotFull($temp, $pair);
+        } else {
+            $this->insertNodeNotFull($node, $pair);
+        }
+    }
+
+    /**
+     * Inserts a key-value pair in a leaf node that is not full. Searches for
+     * the appropriate leaf node, splitting full nodes before descending 
+     * down the tree recursively.
+     * @param object $node is the node from where the search for the leaf node
+     * begins
+     * @param array $pair is the key-value pair
+     */
+    function insertNodeNotFull($node, $pair)
+    {
+        $key = $pair[0];
+        $i = $node->count - 1;
+        list($flag, $pos) = $this->binarySearch($node->keys, $key);
+        if($node->is_leaf == true) {
+            if($flag == 1) {
+                $node->keys[$pos] = $pair;
+                $this->writeNode($node);
+            } else {
+                while($i >= 0 && $node->keys[$i][0] > $key) {
+                    $node->keys[$i + 1] = $node->keys[$i];
+                    $i -= 1;
+                }
+                $node->keys[$i + 1] = $pair;
+                $node->count = count($node->keys);
+                $this->writeNode($node);
+            }
+        } else {
+            if($flag == 1) {
+                $node->keys[$pos] = $pair;
+                $this->writeNode($node);
+            } else {
+                while($i >= 0 && $node->keys[$i][0] > $key) {
+                    $i -= 1;
+                }
+                $i += 1;
+                $next_node = $this->readNode($node->links[$i]);
+                if($next_node->count == 2 * $this->min_degree - 1) {
+                    $this->bTreeSplitChild($node, $i, $next_node);
+                    if($key > $node->keys[$i][0]) {
+                        $i += 1;
+                        $next_node = $this->readNode($node->links[$i]);
+                    }
+                }
+                $this->insertNodeNotFull($next_node, $pair);
+            }
+        }
+    }
+
+    /**
+     * Splits a full node into two child node. The median key-value pair is 
+     * added to the parent node of the node being split.
+     * @param object $parent is the parent node
+     * @paran int $i is the link to child node
+     * @param object $child is the child node
+     */
+    function bTreeSplitChild($parent, $i, $child)
+    {
+        $this->id_count += 1;
+        $temp = new Node();
+        $temp->id = $this->id_count;
+        $this->saveNodeCount();
+        $temp->is_leaf = $child->is_leaf;
+        $temp->count = $this->min_degree - 1;
+        for($j = 0;$j < $this->min_degree - 1;$j++) {
+            $temp->keys[$j] = $child->keys[$this->min_degree + $j];
+        }
+        if($child->is_leaf == false) {
+            for($j = 0;$j < $this->min_degree;$j++) {
+                $temp->links[$j] = $child->links[$this->min_degree + $j];
+            }
+        }
+        for($j = $parent->count;$j > $i;$j--) {
+            $parent->links[$j + 1] = $parent->links[$j];
+        }
+        $parent->links[$j + 1] = $temp->id;
+        for($j = $parent->count - 1;$j >= $i;$j--) {
+            $parent->keys[$j + 1] = $parent->keys[$j];
+        }
+        $parent->keys[$j + 1] = $child->keys[$this->min_degree - 1];
+        $parent->count = count($parent->keys);
+        $child->keys = array_slice($child->keys, 0, $this->min_degree - 1);
+        if($child->is_leaf == false) {
+            $child->links = array_slice($child->links, 0, $this->min_degree);
+        }
+        $child->count = count($child->keys);
+        $this->writeNode($child);
+        $this->writeNode($temp);
+        $this->writeNode($parent);
+    }
+
+    /**
+     * Swaps value of two variables
+     * @param $x is the first variable
+     * @param $y is the second variable
+     */
+    function swap(&$x, &$y)
+    {
+        $temp = $x;
+        $x = $y;
+        $y = $temp;
+    }
+
+    /**
+     * Creates an empty non-leaf node
+     * @return object $node is the non-leaf node
+     */
+    function createEmptyParentNode()
+    {
+        $this->id_count += 1;
+        $temp = new Node();
+        $temp->id = $this->id_count;
+        $this->saveNodeCount();
+        $temp->is_leaf = false;
+        return $temp;
+    }
+
+    /**
+     * Performs binary search for a integer key on an array of integer key based
+     * key-value pairs
+     * @param array $keys is an array containing key-value pairs
+     * @param int $key is the key
+     * @return array containing flag indicating it the value was found or not,
+     * and the position equal to, or nearest to the position of the key being 
+     * searched
+     */
+    function binarySearch($keys, $key)
+    {
+        $low = 0;
+        $high = count($keys) - 1;
+        $flag = -1;
+        while($high >= $low) {
+            $middle = (int)floor(($high + $low) / 2);
+            if($key == $keys[$middle][0]) {
+                $flag = 1;
+                return array($flag, $middle);
+            } else if($key > $keys[$middle][0]) {
+                $low = $middle + 1;
+            } else {
+                $high = $middle - 1;
+            }
+        }
+        return array($flag, $low);
+    }
+
+    /**
+     * Removes a key-value pair from the B-Tree
+     * @param int $key associated with the key-value pair to be deleted
+     */
+    function remove($key)
+    {
+        $this->delete($this->root, $key);
+    }
+
+    /**
+     * Deletes a key-value pair from the B-Tree from a node. 
+     * Handles deletion from leaf node and internal node. If the key-value pair
+     * is not found in an internal node. The recrusion descends to the root
+     * of the sub-tree until a leaf node is encoutered that does not have the 
+     * key-value pair to be deleted.
+     * @param object $node is from where the key search starts
+     * @param int $key is the key to be deleted
+     */
+    function delete($node, $key)
+    {
+        list($flag, $pos) = $this->binarySearch($node->keys, $key);
+        if($flag == 1 && $node->is_leaf == false) {
+            $this->reArrange($node, $pos);
+        }
+        list($flag, $pos) = $this->binarySearch($node->keys, $key);
+        if($flag == 1 && $node->is_leaf == true) {
+                $this->deleteFromLeaf($node, $pos);
+        } else if($flag == 1 && $node->is_leaf == false) {
+                $this->deleteFromNonLeaf($node, $pos);
+        } else if($flag !== 1 && $node->is_leaf == false) {
+            $sub_tree_root = $this->getDescendant($node, $pos);
+            $this->delete($sub_tree_root, $key);
+        }
+    }
+
+    /**
+     * Shifts a key from a non-leaf root to it's child node using nodes 
+     * preceding and next to the key-value pair to be deleted. If the 
+     * preceding child node has atleast minimum MIN_DEGREE keys, a the last 
+     * key-value pair from the preceding node is moved to the position of the 
+     * key-value pair that is to be deleted. Otherwise the same process is done 
+     * using the first key-value pair of the child node next to the key-value 
+     * pair to be deleted.
+     * @param object $node is the internal node containing the key-value pair to
+     * be deleted
+     * @param int $pos is the position of the key-value pair within $pos.
+     */
+    function reArrange(&$node, $pos)
+    {
+        $pred_id = $node->links[$pos];
+        $pred = $this->readNode($pred_id);
+        $next_id = $node->links[$pos + 1];
+        $next = $this->readNode($next_id);
+        if($pred->count >= $this->min_degree) {
+            $this->adjustChildUsingLeftSiblingAndParent($node, $next, $pred, 
+                $pos + 1);
+        } else if($next->count >= $this->min_degree) {
+            $this->adjustChildUsingRightSiblingAndParent($node, $pred, 
+                $next, $pos);
+        }
+    }
+
+    /**
+     * Deletes key-value pair from a leaf node in a B-Tree
+     * @param object $node is the leaf node containing the key-value pair
+     * @param int $key is the key to be deleted
+     */
+    function deleteFromLeaf(&$node, $pos)
+    {
+        if($pos == $node->count - 1) {
+            array_pop($node->keys);
+            $node->count -= 1;
+            $this->writeNode($node);
+        } else {
+            for($i = $pos + 1;$i < $node->count;$i++) {
+                $node->keys[$i - 1] = $node->keys[$i];
+            }
+            $node->keys = array_slice($node->keys, 0, $node->count - 1);
+            $node->count -= 1;
+            $this->writeNode($node);
+        }
+        if($node == $this->root && $node->count == 0) {
+            $this->deleteNodeFile("root");
+            $this->deleteCount();
+        }
+    }
+
+    /**
+     * Deletes key-value pair from a non-leaf node in a B-Tree
+     * @param object $node is the non-leaf node containing the key-value pair
+     * @param int $key is the key to be deleted
+     */
+    function deleteFromNonLeaf(&$node, $pos)
+    {
+        $pred_id = $node->links[$pos];
+        $pred = $this->readNode($pred_id);
+        if($pred->count >= $this->min_degree) {
+            $pred_pair = $pred->keys[$pred->count - 1];
+            $pred_key = $pred_pair[0];
+            $this->delete($pred, $pred_key);
+            $node->keys[$pos] = $pred_pair;
+            $this->writeNode($node);
+        } else {
+            $next_id = $node->links[$pos + 1];
+            $next = $this->readNode($next_id);
+            if($next->count >= $this->min_degree) {
+                $next_pair = $next->keys[0];
+                $next_key = $next_pair[0];
+                $this->delete($next, $next_key);
+                $node->keys[$pos] = $next_pair;
+                $this->writeNode($node);
+            } else {
+                $node_pair = $node->keys[$pos];
+                $node_key = $node_pair[0];
+                $pred->keys[$pred->count] = $node_pair;
+                $pred->count += 1;
+                if($pos == $node->count - 1) {
+                    array_pop($node->keys);
+                    array_pop($node->links);
+                    $node->count -= 1;
+                } else {
+                    for($i = $pos + 1;$i < $node->count;$i++) {
+                        $node->keys[$i - 1] = $node->keys[$i];
+                    }
+                    $node->keys = array_slice($node->keys, 0, $node->count - 1);
+                    for($i = $pos + 2;$i <= $node->count;$i++) {
+                        $node->links[$i - 1] = $node->links[$i];
+                    }
+                    $node->links = array_slice($node->links, 0, $node->count);
+                    $node->count -= 1;
+                }
+                for($i = 0;$i < $next->count;$i++) {
+                    $pred->keys[$pred->count + $i] = $next->keys[$i];
+                }
+                if($next->is_leaf == false) {
+                    for($i = 0;$i <= $next->count;$i++) {
+                        $pred->links[$pred->count + $i] = $next->links[$i];
+                    }
+                }
+                $pred->count += $next->count;
+                $this->writeNode($pred);
+                $this->deleteNodeFile($next->id);
+                if($node == $this->root && $node->count == 0) {
+                    $old_id = $pred->id;
+                    $pred->id = "root";
+                    $this->root = $pred;
+                    $this->deleteNodeFile($old_id);
+                    $this->writeNode($this->root);
+                } else {
+                    $this->writeNode($node);
+                }
+                $this->delete($pred, $node_key);
+            }
+        }
+    }
+
+    /**
+     * If the key to be deleted is not found in an internal node, finds the root
+     * of the sub-tree that might contain the key to be deleted. If the node
+     * contains atleast $min_degree number of keys, the node is returned.
+     * Otherwise, the node is adjusted using one of its sibling nodes and the 
+     * parent node so that the resultant node has $min_degree keys. 
+     * @param object $parent is the parent node
+     * @param int pos is the link to the root of the sub-tree
+     * @return object $child is the child node to which the recursion will
+     * descend
+     */
+    function getDescendant($parent, $pos)
+    {
+        $child_id = $parent->links[$pos];
+        $child = $this->readNode($child_id);
+        if($child->count == $this->min_degree - 1) {
+            $siblings = $this->getSiblings($parent, $pos);
+            if($siblings[0] !== -1 && $siblings[1] !== -1) {
+                $pred_id = $siblings[0];
+                $pred = $this->readNode($pred_id);
+                if($pred->count >= $this->min_degree) {
+                    $this->adjustChildUsingLeftSiblingAndParent($parent, $child,
+                        $pred, $pos);
+                    return $child;
+                } else {
+                    $next_id = $siblings[1];
+                    $next = $this->readNode($next_id);
+                    if($next->count >= $this->min_degree) {
+                        $this->adjustChildUsingRightSiblingAndParent($parent, 
+                            $child, $next, $pos);
+                        return $child;
+                    } else {
+                        if($pred->count <= $next->count) {
+                            $this->mergeChildWithParentKeyAndRightSibling(
+                                $parent, $pred, $child, $pos - 1);
+                            return $pred;
+                        } else {
+                            $this->mergeChildWithParentKeyAndRightSibling(
+                                $parent, $child, $next, $pos);
+                            return $child;
+                        }
+                    }
+                }
+            } else if($siblings[0] !== -1) {
+                $pred_id = $siblings[0];
+                $pred = $this->readNode($pred_id);
+                if($pred->count >= $this->min_degree) {
+                    $this->adjustChildUsingLeftSiblingAndParent($parent, $child,
+                        $pred, $pos);
+                    return $child;
+                } else {
+                    $this->mergeChildWithParentKeyAndRightSibling($parent, 
+                        $pred, $child, $pos - 1);
+                    return $pred;
+                }
+            } else {
+                $next_id = $siblings[1];
+                $next = $this->readNode($next_id);
+                if($next->count >= $this->min_degree) {
+                    $this->adjustChildUsingRightSiblingAndParent($parent,
+                        $child, $next, $pos);
+                    return $child;
+                } else {
+                    $this->mergeChildWithParentKeyAndRightSibling($parent, 
+                        $child, $next, $pos);
+                    return $child;
+                }
+            }
+        } else return $child;
+    }
+
+    /**
+     * Gives a child node an extra key by moving a key from the parent to the
+     * child node, and by moving a key from the child's left sibling to the 
+     * parent node
+     * @param object $parent is the parent node
+     * @param object $child is the child node
+     * @param object $pred is the $child's left sibling node
+     * @param $pos is the link from $parent to $child
+     */
+    function adjustChildUsingLeftSiblingAndParent(&$parent, &$child, &$pred, 
+        $pos)
+    {
+        $pred_pair = array_pop($pred->keys);
+        $pred_link = -1;
+        if($pred->is_leaf == false) {
+            $pred_link = array_pop($pred->links);
+        }
+        $pred->count -= 1;
+        $this->writeNode($pred);
+        $parent_pair = $parent->keys[$pos - 1];
+        for($i = $child->count - 1;$i >= 0;$i--) {
+            $child->keys[$i + 1] = $child->keys[$i];
+        }
+        $child->keys[0] = $parent_pair;
+        if($child->is_leaf == false) {
+            for($i = $child->count;$i >= 0;$i--) {
+                $child->links[$i + 1] = $child->links[$i];
+            }
+            $child->links[0] = $pred_link;
+        }
+        $child->count += 1;
+        $this->writeNode($child);
+        $parent->keys[$pos - 1] = $pred_pair;
+        $this->writeNode($parent);
+    }
+
+    /**
+     * Gives a child node an extra key by moving a key from the parent to the
+     * child node, and by moving a key from the child's right sibling to the 
+     * parent node
+     * @param object $parent is the parent node
+     * @param object $child is the child node
+     * @param object $pred is the $child's left sibling node
+     * @param $pos is the link from $parent to $child
+     */
+    function adjustChildUsingRightSiblingAndParent(&$parent, &$child, &$next,
+        $pos)
+    {
+        $next_pair = $next->keys[0];
+        $next_link = -1;
+        for($i = 1;$i < $next->count;$i++) {
+            $next->keys[$i - 1] = $next->keys[$i];
+        }
+        $next->keys = array_slice($next->keys, 0, $next->count - 1);
+        if($next->is_leaf == false) {
+            $next_link = $next->links[0];
+            for($i = 1;$i <= $next->count;$i++) {
+                $next->links[$i - 1] = $next->links[$i];
+            }
+            $next->links = array_slice($next->links, 0, $next->count);
+        }
+        $next->count -= 1;
+        $this->writeNode($next);
+        $parent_pair = $parent->keys[$pos];
+        $child->keys[$child->count] = $parent_pair;
+        $child->count += 1;
+        if($child->is_leaf == false) {
+            $child->links[$child->count] = $next_link;
+        }
+        $this->writeNode($child);
+        $parent->keys[$pos] = $next_pair;
+        $this->writeNode($parent);
+    }
+
+    /**
+     * Merges the child node with it's right sibling. The separating key in the
+     * parent node is added as the median key to the newly formed node
+     * @param object $parent is the parent node
+     * @param object $child is the child node
+     * @param object $next is the $child's right sibling node
+     * @param $pos is the link from $parent to $child
+     */
+    function mergeChildWithParentKeyAndRightSibling(&$parent, &$child, &$next, 
+        $pos)
+    {
+        $parent_pair = $parent->keys[$pos];
+        $child->keys[$child->count] = $parent_pair;
+        $child->count += 1;
+        for($i = 0;$i < $next->count;$i++) {
+            $child->keys[$child->count + $i] = $next->keys[$i];
+        }
+        if($next->is_leaf == false) {
+            for($i = 0;$i <= $next->count;$i++) {
+                $child->links[$child->count + $i] = $next->links[$i];
+            }
+        }
+        $child->count = count($child->keys);
+        $this->writeNode($child);
+        $this->deleteNodeFile($next->id);
+        if($pos == $parent->count - 1) {
+            array_pop($parent->keys);
+            array_pop($parent->links);
+            $parent->count -= 1;
+        } else {
+            for($i = $pos + 1;$i < $parent->count;$i++) {
+                $parent->keys[$i - 1] = $parent->keys[$i];
+            }
+            $parent->keys = array_slice($parent->keys, 0, $parent->count - 1);
+            for($i = $pos + 2;$i <= $parent->count;$i++) {
+                $parent->links[$i - 1] = $parent->links[$i];
+            }
+            $parent->links = array_slice($parent->links, 0, $parent->count);
+            $parent->count -= 1;
+        }
+        if($parent == $this->root && $parent->count == 0) {
+            $old_id = $child->id;
+            $child->id = "root";
+            $this->root = $child;
+            $this->deleteNodeFile($old_id);
+            $this->writeNode($this->root);
+        } else {
+            $this->writeNode($parent);
+        }
+    }
+
+    /**
+     * Gets the siblings ids based on link in parent node
+     * @param object $parent is the parent node
+     * @param int $pos is the link for which the siblings are to be found
+     */
+    function getSiblings($parent, $pos)
+    {
+        $siblings = array();
+        if($pos > 0 && $pos < $parent->count) {
+            $siblings[] = $parent->links[$pos - 1];
+            $siblings[] = $parent->links[$pos + 1];
+        } else if($pos == 0) {
+            $siblings[] = -1;
+            $siblings[] = $parent->links[$pos + 1];
+        } else {
+            $siblings[] = $parent->links[$pos - 1];
+            $siblings[] = -1;
+        }
+        return $siblings;
+    }
+}
\ No newline at end of file
diff --git a/lib/btree_node.php b/lib/btree_node.php
new file mode 100644
index 0000000..111fdc9
--- /dev/null
+++ b/lib/btree_node.php
@@ -0,0 +1,87 @@
+<?php
+
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2013  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @package seek_quarry
+ * @subpackage bin
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2013
+ * @filesource
+ */
+
+/**
+ * This class implements B-Tree nodes used in the B-Tree implementation in 
+ * btree.php
+ * 
+ * @author Akshat Kukreti
+ * @package seek_quarry
+ */
+
+class Node
+{
+    /**
+     * Storage for id of a B-Tree node
+     * @var int
+     */
+    var $id;
+
+    /**
+     * Flag for checking if node is a leaf node or internal node
+     * @var boolean
+     */
+    var $is_leaf;
+
+    /**
+     * Storage for keeping track of node ids
+     * @var int
+     */
+    var $count;
+
+    /**
+     * Storage for key-value pairs in a B-Tree node
+     * @var array
+     */
+    var $keys;
+
+    /**
+     * Storage for links to child nodes in a B-Tree node
+     * @var array
+     */
+    var $links;
+
+    /**
+     * Creates and initializes an empty leaf node with id -1
+     * @var int
+     */
+    function __construct()
+    {
+        $this->id = -1;
+        $this->is_leaf = true;
+        $this->count = 0;
+        $this->keys = null;
+        $this->links = null;
+    }
+}
\ No newline at end of file
diff --git a/lib/crawl_constants.php b/lib/crawl_constants.php
index 18ebde8..0292c99 100644
--- a/lib/crawl_constants.php
+++ b/lib/crawl_constants.php
@@ -65,6 +65,7 @@ interface CrawlConstants
     const schedule_data_base_name = "ScheduleData";
     const schedule_name = "FetchSchedule";
     const robot_data_base_name = "RobotData";
+    const cache_page_validation_data_base_name = "CachePageValidationData";
     const index_data_base_name = "IndexData";
     const network_base_name = "Network";
     const network_crawllist_base_name = "NetworkCrawlList";
@@ -222,6 +223,8 @@ interface CrawlConstants
     const ACTIVE_CLASSIFIERS = 'cu';
     const ACTIVE_CLASSIFIERS_DATA = 'cv';
     const MAX_DESCRIPTION_LEN = 'cw';
+    const CACHE_PAGE_VALIDATORS = 'cx';
+    const CACHE_PAGE_VALIDATION_DATA = 'cy';
 
     const NEEDS_OFFSET_FLAG = 0x7FFFFFFF;
 }
diff --git a/tests/btree_test.php b/tests/btree_test.php
new file mode 100644
index 0000000..d870a19
--- /dev/null
+++ b/tests/btree_test.php
@@ -0,0 +1,225 @@
+<?php
+
+/**
+ *  SeekQuarry/Yioop --
+ *  Open Source Pure PHP Search Engine, Crawler, and Indexer
+ *
+ *  Copyright (C) 2009 - 2013  Chris Pollett chris@pollett.org
+ *
+ *  LICENSE:
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  END LICENSE
+ *
+ * @author Chris Pollett chris@pollett.org
+ * @package seek_quarry
+ * @subpackage test
+ * @license http://www.gnu.org/licenses/ GPL3
+ * @link http://www.seekquarry.com/
+ * @copyright 2009 - 2013
+ * @filesource
+ */
+
+ if(!defined('BASE_DIR')) {echo "BAD REQUEST"; exit();}
+
+ /**
+  * Load BTree
+  */
+ require_once BASE_DIR."/lib/btree.php";
+
+  /**
+  * Load crawlHash
+  */
+ require_once BASE_DIR."/lib/utility.php";
+
+ /**
+ * Used to test insert, lookup,  and deletion of key-value pairs on the B-Tree.
+ *
+ * @author Akshat Kukreti
+ * @package seek_quarry
+ * @subpackage test
+ */
+
+ class BTreeTest extends UnitTest
+ {
+    /**
+     * Minimum degree is set to 2 and the number of key-value pairs is set to 25
+     */
+    function setUp()
+    {
+        if(!defined('BTREE_TEST_DIR')) {
+            define('BTREE_TEST_DIR', WORK_DIRECTORY."/btreeTest");
+        }
+        if(!defined('DEGREE')) {
+            define('DEGREE', 2);
+        }
+        $this->test_objects['FILE1'] = new BTree(BTREE_TEST_DIR, DEGREE);
+        if(!defined('NUM_VALS')) {
+            define('NUM_VALS', 25);
+        }
+    }
+
+    /**
+     * Delete the B-Tree files created during the test
+     */
+    function tearDown()
+    {
+        $all_files = glob(BTREE_TEST_DIR."/*.txt");
+        if($all_files !== NULL) {
+            foreach($all_files as $file) {
+                unlink($file);
+            }
+        }
+    }
+
+    /** 
+     * Test to check that an empty B-Tree is not saved to disk. 
+     * A B-Tree object is created. The test directory is checked for B-Tree 
+     * files. A value is inserted, the test directory is again checked. The 
+     * inserted value is removed and the test directory is again checked for 
+     * B-Tree files.
+     */
+    function emptyBTreeNoSaveTestCase()
+    {
+        $all_files = glob(BTREE_TEST_DIR.'/*.txt');
+        $this->assertEqual(0, count($all_files), 'Empty B-Tree not saved 
+            saved to disk');
+        $this->test_objects['FILE1']->insert(array(1, 1));
+        $all_files = glob(BTREE_TEST_DIR.'/*.txt');
+        $this->assertEqual(2, count($all_files), 'Non-empty B-Tree successfully 
+            saved to disk');
+        $this->test_objects['FILE1']->remove(1);
+        $all_files = glob(BTREE_TEST_DIR.'/*.txt');
+        $this->assertEqual(0, count($all_files), 'Empty B-Tree not saved 
+            saved to disk');
+    }
+
+    /**
+     * Test to check insertion and lookup in a B-Tree
+     * Key-value pairs are inserted in the B-Tree and then looked up using keys.
+     */
+    function insertLookupTestCase()
+    {
+        //Insert values
+        $key_value_pairs = array();
+        for($i = 1;$i <= NUM_VALS;$i++) {
+            $value = crawlHash(rand(1, 1000), true);
+            $key = crawlHash($value, true);
+            $this->test_objects['FILE1']->insert(array($key, $value));
+            $key_value_pairs[] = array($key, $value);
+        }
+
+        //Lookup values
+        foreach($key_value_pairs as $key_value_pair) {
+            $key = $key_value_pair[0];
+            $value = $key_value_pair[1];
+            $lookup_entry = $this->test_objects['FILE1']->findValue($key);
+            $lookup_value = $lookup_entry[1];
+            $this->assertEqual($value, $lookup_value, 'Inserted value is equal
+                to lookup value');
+        }
+    }
+
+    /** 
+     * Test to check that a key is deleted successfully from a leaf node
+     * Key-value pairs are inserted in the B-Tree. key-value pairs present in 
+     * leaf nodes are then deleted from the B-Tree. The deleted key-value pairs
+     * are looked up to check if they were removed successfully. 
+     */
+    function deleteFromLeafNodeTestCase()
+    {
+        //Insert values
+        for($i = 1;$i <= NUM_VALS;$i++) {
+            $this->test_objects['FILE1']->insert(array($i, $i));
+        }
+
+        //Keys in leaf nodes
+        $leaf_keys = array(1,3,5,9,10);
+        foreach($leaf_keys as $key) {
+            $this->test_objects['FILE1']->remove($key);
+        }
+
+        //Lookup deleted keys
+        foreach($leaf_keys as $key) {
+            $this->assertEqual(null, 
+                $this->test_objects['FILE1']->findValue($key), 
+                'Key successfully deleted from leaf node');
+        }
+
+    }
+
+    /**
+     * Test to check that a key is deleted successfully from an internal node
+     * Key-value pairs are first added to the B-Tree. Key-value pairs are then
+     * deleted successively from the root node. The deleted key-value pairs are 
+     * then looked up to check if they were successfully
+     */
+    function deleteFromInternalNodeTestCase()
+    {
+        //Insert values
+        for($i = 1;$i <= NUM_VALS;$i++) {
+            $value = crawlHash(rand(1, 1000));
+            $key = crawlHash($value);
+            $this->test_objects['FILE1']->insert(array($key, $value));
+        }
+
+        $deleted = array();
+        for($i = 1;$i <= NUM_VALS - 10; $i++) {
+            $internal_key = $this->test_objects['FILE1']->root->keys[0][0];
+            $this->test_objects['FILE1']->remove($internal_key);
+            $this->assertEqual(null, 
+                $this->test_objects['FILE1']->findValue($internal_key), 
+                'Key deleted successfully from internal node');
+        }
+
+    }
+
+    /**
+     * Function to check that keys are successfully deleted from the B-Tree
+     * Random key-value pairs are firs inserted in the B-Tree. From the inserted
+     * key-value pairs, key-value pairs are randomly selected and deleted from
+     * the B-Tree. The deleted key-value pairs are then looked up using their 
+     * keys to check if they were successfully deleted.
+     */
+    function deleteLookupTestCase()
+    {
+        //Insert values
+        $key_value_pairs = array();
+        for($i = 1;$i <= NUM_VALS;$i++) {
+            $value = crawlHash(rand(1, 1000));
+            $key = crawlHash($value);
+            $this->test_objects['FILE1']->insert(array($key, $value));
+            $key_value_pairs[] = array($key, $key);
+        }
+
+        //Delete Values
+        $deleted = array();
+
+        for($i = 1;$i <= NUM_VALS;$i++) {
+            $index = mt_rand(0, NUM_VALS - 1);
+            $key = $key_value_pairs[$index][0];
+            $this->test_objects['FILE1']->remove($key);
+            $deleted[] = $key;
+        }
+
+        //Lookup values
+        foreach($deleted as $deleted_key) {
+            $this->assertEqual(null, 
+                $this->test_objects['FILE1']->findValue($deleted_key),
+                'Deleted Value not found');
+        }
+    }
+
+ }
\ No newline at end of file
-- 
1.7.11.msysgit.1


From 3e837000461368335a8a96e9e1f7865447ef1966 Mon Sep 17 00:00:00 2001
From: Akshat Kukreti <akshatkukreti@gmail.com>
Date: Wed, 22 May 2013 20:10:09 -0700
Subject: [PATCH 2/5] fetch_url.php modified to extract ETags from downloaded
 web pages and send ETags along with HTTP header

---
 lib/fetch_url.php | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/lib/fetch_url.php b/lib/fetch_url.php
index cceb260..9daf327 100755
--- a/lib/fetch_url.php
+++ b/lib/fetch_url.php
@@ -246,6 +246,22 @@ class FetchUrl implements CrawlConstants
     static function prepareUrlHeaders($url, $minimal = false)
     {
         $url = str_replace("&amp;", "&", $url);
+
+        /*Check if an ETag was added by the queue server. If found, create
+          If-None_Match header with the ETag and add it to the headers. Remove
+          ETag from URL
+         */
+        $if_none_match = "If-None-Match";
+        $etag = null;
+        if(stristr($url, "ETag:")) {
+            crawlLog("Adding ETag header");
+            $etag_parts = preg_split("/ETag\:/i", $url);
+            $etag_data = explode(" ", $etag_parts[1]);
+            $etag = $etag_data[1];
+            $pos = strrpos($url, "ETag:");
+            $url = substr_replace($url, "", $pos, strlen("ETag: ".$etag));
+        }
+
         /* in queue_server we added the ip (if available)
           after the url followed by ###
          */
@@ -286,6 +302,11 @@ class FetchUrl implements CrawlConstants
             $url_with_ip_if_possible = $url;
         }
         $headers[] = 'Expect:';
+        if($etag !== null) {
+            $etag_header = $if_none_match.": ".$etag;
+            $headers[] = $etag_header;
+            crawlLog("...done");
+        }
         $results = array($url, $url_with_ip_if_possible, $headers);
         return $results;
     }
@@ -335,6 +356,9 @@ class FetchUrl implements CrawlConstants
     static function parseHeaderPage(&$header_and_page,
         $value=CrawlConstants::PAGE)
     {
+        $cache_page_validators = array();
+        $cache_page_validators['etag'] = -1;
+        $cache_page_validators['expires'] = -1;
         $new_offset = 0;
         // header will include all redirect headers
         $site = array();
@@ -425,6 +449,38 @@ class FetchUrl implements CrawlConstants
                         trim($robot_meta));
                 }
             }
+            if(stristr($line, 'ETag:')) {
+                $line_parts = preg_split("/ETag\:/i", $line);
+                $etag_data = explode(" ", $line_parts[1]);
+                $etag = $etag_data[1];
+                $cache_page_validators['etag'] = $etag;
+            }
+            if(stristr($line, 'Expires:')) {
+                $line_parts = preg_split("/Expires\:/i", $line);
+                $all_dates = $line_parts[1];
+                $date_parts = explode(",", $all_dates);
+                if(count($date_parts) == 2) {
+                    $date_dt = new DateTime($date_parts[1]);
+                    $date_ts = $date_dt->getTimestamp();
+                    $cache_page_validators['expires'] = $date_ts;
+                } else if(count($date_parts) > 2) {
+                    /*Encountered some pages with more than one Expires date
+                      :O */
+                    $timestamps = array();
+                    for($i = 1;$i < count($date_parts);$i += 2) {
+                        $dt = new DateTime($date_parts[$i]);
+                        $ds = $dt->getTimestamp();
+                        $timestamps[] = $ds;
+                    }
+                    $lowest = min($timestamps);
+                    $cache_page_validators['expires'] = $lowest;
+                }
+            }
+            if(!($cache_page_validators['etag'] == -1 &&
+                $cache_page_validators['expires'] == -1)) {
+                $site[CrawlConstants::CACHE_PAGE_VALIDATORS] = 
+                    $cache_page_validators;
+            }
         }
         /*
            If the doc is HTML and it uses a http-equiv to set the encoding
-- 
1.7.11.msysgit.1


From c6485ace929598124ea4933f03e40c7a9edc6e84 Mon Sep 17 00:00:00 2001
From: Akshat Kukreti <akshatkukreti@gmail.com>
Date: Wed, 22 May 2013 20:23:44 -0700
Subject: [PATCH 3/5] fetcher.php modified to store ETag data.
 fetch_controller.php modified to write ETag data to
 disk

---
 bin/fetcher.php                  | 28 ++++++++++++++++++++++++++--
 controllers/fetch_controller.php |  9 +++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/bin/fetcher.php b/bin/fetcher.php
index 3e71e28..af903ef 100755
--- a/bin/fetcher.php
+++ b/bin/fetcher.php
@@ -432,6 +432,7 @@ class Fetcher implements CrawlConstants
         $this->to_crawl = array();
         $this->to_crawl_again = array();
         $this->found_sites = array();
+        $this->found_sites[self::CACHE_PAGE_VALIDATION_DATA] = array();
         $this->page_range_request = PAGE_RANGE_REQUEST;
         $this->max_description_len = MAX_DESCRIPTION_LEN;
         $this->fetcher_num = false;
@@ -1714,7 +1715,8 @@ class Fetcher implements CrawlConstants
             self::OPERATING_SYSTEM, self::MODIFIED, self::ROBOT_INSTANCE,
             self::LOCATION, self::SIZE, self::TOTAL_TIME, self::DNS_TIME,
             self::ROBOT_PATHS, self::GOT_ROBOT_TXT, self::CRAWL_DELAY,
-            self::AGENT_LIST, self::ROBOT_METAS, self::WARC_ID);
+            self::AGENT_LIST, self::ROBOT_METAS, self::WARC_ID
+            self::CACHE_PAGE_VALIDATORS);
 
         foreach($summary_fields as $field) {
             if(isset($site[$field])) {
@@ -1843,6 +1845,12 @@ class Fetcher implements CrawlConstants
                 }
             } //end else
 
+            //Add cache page validation data
+            if(isset($site[self::CACHE_PAGE_VALIDATORS])) {
+                $this->found_sites[self::CACHE_PAGE_VALIDATION_DATA][] = 
+                    array($site[self::URL], $site[self::CACHE_PAGE_VALIDATORS]);
+            }
+
             if(isset($this->hosts_with_errors[$host]) &&
                 $this->hosts_with_errors[$host] > DOWNLOAD_ERROR_THRESHOLD) {
                 $this->found_sites[self::ROBOT_TXT][$host][
@@ -2029,7 +2037,7 @@ class Fetcher implements CrawlConstants
             footprint.
          */
         $byte_counts = array("TOTAL" => 0, "ROBOT" => 0, "SCHEDULE" => 0,
-            "INDEX" => 0);
+            "INDEX" => 0, "CACHE_PAGE_VALIDATION" => 0);
         $post_data = array('c'=>'fetch', 'a'=>'update',
             'crawl_time' => $this->crawl_time, 'machine_uri' => WEB_URI,
             'robot_instance' => $prefix.ROBOT_INSTANCE, 'data' => '');
@@ -2046,6 +2054,22 @@ class Fetcher implements CrawlConstants
             $byte_counts["ROBOT"] = $bytes_robot;
         }
 
+        //handle cache validation data
+        if(isset($this->found_sites[self::CACHE_PAGE_VALIDATION_DATA])) {
+            $cache_page_validation_data = webencode(
+                gzcompress(serialize(
+                    $this->found_sites[self::CACHE_PAGE_VALIDATION_DATA])));
+
+            unset($this->found_sites[self::CACHE_PAGE_VALIDATION_DATA]);
+            $bytes_cache_page_validation = strlen($cache_page_validation_data);
+            $post_data['data'] .= $cache_page_validation_data;
+            crawlLog("...".$bytes_cache_page_validation.
+                " bytes of cache page validation data");
+            $byte_counts["TOTAL"] += $bytes_cache_page_validation;
+            $byte_counts["CACHE_PAGE_VALIDATION"] = 
+                $bytes_cache_page_validation;
+        }
+
         //handle schedule data
         $schedule_data = array();
         if(isset($this->found_sites[self::TO_CRAWL][$current_server])) {
diff --git a/controllers/fetch_controller.php b/controllers/fetch_controller.php
index b981de3..911ede2 100755
--- a/controllers/fetch_controller.php
+++ b/controllers/fetch_controller.php
@@ -414,6 +414,7 @@ class FetchController extends Controller implements CrawlConstants
             $byte_counts = unserialize(webdecode($_REQUEST['byte_counts']));
         }
         $robot_data = "";
+        $cache_page_validation_data = "";
         $schedule_data = "";
         $index_data = "";
         if(isset($byte_counts["TOTAL"]) &&
@@ -421,6 +422,9 @@ class FetchController extends Controller implements CrawlConstants
             $pos = 0;
             $robot_data = substr($uploaded, $pos,$byte_counts["ROBOT"]);
             $pos += $byte_counts["ROBOT"];
+            $cache_page_validation_data = substr($uploaded, $pos, 
+                $byte_counts["CACHE_PAGE_VALIDATION"]);
+            $pos += $byte_counts["CACHE_PAGE_VALIDATION"];
             $schedule_data =
                 substr($uploaded, $pos, $byte_counts["SCHEDULE"]);
             $pos += $byte_counts["SCHEDULE"];
@@ -431,6 +435,11 @@ class FetchController extends Controller implements CrawlConstants
             $this->addScheduleToScheduleDirectory(self::robot_data_base_name,
                 $robot_data);
         }
+        if(strlen($cache_page_validation_data) > 0) {
+            $this->addScheduleToScheduleDirectory(
+                self::cache_page_validation_data_base_name, 
+                $cache_page_validation_data);
+        }
         if(strlen($schedule_data) > 0) {
             $this->addScheduleToScheduleDirectory(self::schedule_data_base_name,
                 $schedule_data);
-- 
1.7.11.msysgit.1


From ee2bd4cc6585e05567d8678c0320d3fb69838328 Mon Sep 17 00:00:00 2001
From: Akshat Kukreti <akshatkukreti@gmail.com>
Date: Wed, 22 May 2013 20:25:58 -0700
Subject: [PATCH 4/5] missed a comma

---
 bin/fetcher.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/fetcher.php b/bin/fetcher.php
index af903ef..d255d54 100755
--- a/bin/fetcher.php
+++ b/bin/fetcher.php
@@ -1715,7 +1715,7 @@ class Fetcher implements CrawlConstants
             self::OPERATING_SYSTEM, self::MODIFIED, self::ROBOT_INSTANCE,
             self::LOCATION, self::SIZE, self::TOTAL_TIME, self::DNS_TIME,
             self::ROBOT_PATHS, self::GOT_ROBOT_TXT, self::CRAWL_DELAY,
-            self::AGENT_LIST, self::ROBOT_METAS, self::WARC_ID
+            self::AGENT_LIST, self::ROBOT_METAS, self::WARC_ID,
             self::CACHE_PAGE_VALIDATORS);
 
         foreach($summary_fields as $field) {
-- 
1.7.11.msysgit.1


From a3789e018c2997f530a7a11ea4c290784efdaf86 Mon Sep 17 00:00:00 2001
From: Akshat Kukreti <akshatkukreti@gmail.com>
Date: Wed, 22 May 2013 20:53:19 -0700
Subject: [PATCH 5/5] btree integrated with queue_server.php for insertion and
 lookup of cache page validation data

---
 bin/queue_server.php | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)

diff --git a/bin/queue_server.php b/bin/queue_server.php
index d1a97e5..a2270eb 100755
--- a/bin/queue_server.php
+++ b/bin/queue_server.php
@@ -86,6 +86,8 @@ require_once BASE_DIR."/lib/phrase_parser.php";
 /** Include marker interface to say we support join() method*/
 require_once BASE_DIR."/lib/join.php";
 
+/** Include B-Tree class for storing cache page validation data*/
+require_once BASE_DIR."/lib/btree.php";
 
 /** get any indexing plugins */
 foreach(glob(BASE_DIR."/lib/indexing_plugins/*_plugin.php") as $filename) {
@@ -111,6 +113,14 @@ if(USE_MEMCACHE) {
     unset($mc);
 }
 
+if(!defined("CACHE_PAGE_VALIDATORS")) {
+    define("CACHE_PAGE_VALIDATORS",CRAWL_DIR.'/cachePageValidators');
+}
+if(!is_dir(CACHE_PAGE_VALIDATORS)) {
+    mkdir(CACHE_PAGE_VALIDATORS);
+    chmod(CACHE_PAGE_VALIDATORS, 0777);
+}
+
 /**
  * Command line program responsible for managing Yioop crawls.
  *
@@ -230,6 +240,10 @@ class QueueServer implements CrawlConstants, Join
      */
     var $index_archive;
     /**
+     * Holds the B-Tree used for saving cache page validation data
+     */
+    var $btree;
+    /**
      * The timestamp of the current active crawl
      * @var int
      */
@@ -338,6 +352,7 @@ class QueueServer implements CrawlConstants, Join
         $this->indexing_plugins = array();
         $this->video_sources = array();
         $this->server_name = "IndexerAndScheduler";
+        $this->btree = false;
     }
 
     /**
@@ -477,6 +492,7 @@ class QueueServer implements CrawlConstants, Join
             case self::WEB_CRAWL:
                 if($this->isOnlyIndexer()) return;
                 $this->processRobotUrls();
+                $this->processCachePageValidators();
 
                 $count = $this->web_queue->to_crawl_queue->count;
                 $max_links = max(MAX_LINKS_PER_PAGE, MAX_LINKS_PER_SITEMAP);
@@ -800,6 +816,8 @@ class QueueServer implements CrawlConstants, Join
         } else {
             file_put_contents($close_file, "1");
         }
+        //Write B-Tree root node to disk before before exiting
+        $this->btree->writeNode($this->btree->root);
     }
 
     /**
@@ -963,6 +981,9 @@ class QueueServer implements CrawlConstants, Join
     {
         //to get here we at least have to have a crawl_time
         $this->crawl_time = $info[self::CRAWL_TIME];
+        //Initialize B-Tree for storing cache page validation data
+        $this->btree = new BTree(CACHE_PAGE_VALIDATORS.'/cachePageValidators'.
+            $this->crawl_time);
 
         $read_from_info = array(
             "crawl_order" => self::CRAWL_ORDER,
@@ -1602,6 +1623,47 @@ class QueueServer implements CrawlConstants, Join
     }
 
     /**
+     * Process cache page validation data files sent by Fetcher
+     */
+    function processCachePageValidators()
+    {
+        crawlLog("Checking for cache page validation data");
+        $cache_page_validators_dir = CRAWL_DIR."/schedules/".
+            self::cache_page_validation_data_base_name.$this->crawl_time;
+        $this->processDataFile($cache_page_validators_dir, 
+            "processCachePageValidatorsArchive");
+        crawlLog("...done");
+    }
+
+    /**
+     * Processes a cache page validation data file. Extracts key-value pairs 
+     * from the file and inserts into the B-Tree used for storing cache
+     * page validation data.
+     * @param string $file is the cache page validation data file written by
+     * Fetchers.
+     */
+    function processCachePageValidatorsArchive($file)
+    {
+        crawlLog("Processing cache page validation data in $file");
+        $start_time = microtime();
+
+        $cache_page_validation_data = 
+            unserialize(gzuncompress(webdecode(file_get_contents($file))));
+        foreach($cache_page_validation_data as $data) {
+            $link = $data[0];
+            $value = $data[1];
+            $key = crawlHash($link);
+            $entry = array($key, $value);
+            $this->btree->insert($entry);
+        }
+
+        crawlLog(" time: ".(changeInMicrotime($start_time))."\n");
+        crawlLog("Done processing cache page validation data file: $file");
+
+        unlink($file);
+    }
+
+    /**
      * Deletes all Robot informations stored by the QueueServer.
      *
      * This function is called roughly every CACHE_ROBOT_TXT_TIME.
@@ -2143,12 +2205,53 @@ class QueueServer implements CrawlConstants, Join
                 self::schedule_name.$this->crawl_time.".txt", "wb");
             fwrite($fh, $first_line);
             foreach($sites as $site) {
+                $extracted_etag = null;
                 list($url, $weight, $delay) = $site;
+
+                /*check if we have cache validation data for a URL. If both 
+                  ETag and Expires timestamp are found or only an expires
+                  timestamp is found, the timestamp is compared with the current
+                  time. If the current time is less than the expires timestamp,
+                  the URL is not added to the fetch batch. If only an ETag is 
+                  found, the ETag is appended to the URL so that it can be
+                  processed by the fetcher.
+                 */
+                $key = crawlHash($url);
+                $value = $this->btree->findValue($key);
+                if($value !== null) {
+                    $cache_validation_data = $value[1];
+                    if($cache_validation_data['etag'] !== -1 && 
+                        $cache_validation_data['expires'] !== -1) {
+                        $expires_timestamp = $cache_validation_data['expires'];
+                        $current_time = time();
+                        if($current_time < $expires_timestamp) {
+                            continue;
+                        } else {
+                            $etag = $cache_validation_data['etag'];
+                            $extracted_etag = "ETag: ".$etag;
+                        }
+                    } else if($cache_validation_data['etag'] !== -1) {
+                        $etag = $cache_validation_data['etag'];
+                        $extracted_etag = "ETag: ".$etag;
+                    } else if($cache_validation_data['expires'] !== -1){
+                        $expires_timestamp = $cache_validation_data['expires'];
+                        $current_time = time();
+                        if($current_time < $expires_timestamp) {
+                            continue;
+                        }
+                    }
+                }
+
                 $host_url = UrlParser::getHost($url);
                 $dns_lookup = $this->web_queue->dnsLookup($host_url);
                 if($dns_lookup) {
                     $url .= "###".urlencode($dns_lookup);
                 }
+
+                if($extracted_etag !== null) {
+                    $url .= $extracted_etag;
+                }
+
                 $out_string = base64_encode(
                     packFloat($weight).packInt($delay).$url)."\n";
                 fwrite($fh, $out_string);
-- 
1.7.11.msysgit.1

