<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\data_structures;
use seekquarry\yioop\library as L;
use seekquarry\yioop\configs as C;
/**
* Loads crawlLog functions if needed
*/
require_once __DIR__ . "/../Utility.php";
/**
* PackedTableTools packs records into strings and reads records back out
* of such strings, using a signature that gives each column a name and a
* type. It also saves, loads and looks up tables of such records, where
* each row is a key followed by the packed records for that key.
*
* @author Chris Pollett
*/
class PackedTableTools
{
/**
* DEFAULT_COMPRESSOR names the Compressor class used to compress rows
* when the constructor is not given one.
*/
const DEFAULT_COMPRESSOR = C\NS_COMPRESSORS . "NonCompressor";
/**
* DEFAULT_KEY_LEN is the length in bytes of the primary key when the
* signature does not give one.
*/
const DEFAULT_KEY_LEN = 16;
/**
* TYPE_SYNONYMS maps each name a signature may use for a column type to
* the type PackedTableTools stores it as, so INTEGER is stored as INT.
*/
const TYPE_SYNONYMS = [ "BOOLEAN" => "BOOL", "BOOL" => "BOOL",
"CLOB" => "TEXT", "DOUBLE" => "DOUBLE", "FLOAT" => "REAL",
"INT" => "INT", "INTEGER" => "INT", "REAL"=> "REAL",
"TEXT" => "TEXT"
];
/**
* ADD_MEM_TABLE tells add to put the row in a table held in memory as
* an array of key => packed records pairs.
*/
const ADD_MEM_TABLE = 0;
/**
* ADD_FILE_HANDLE tells add to write the row to an open file handle.
*/
const ADD_FILE_HANDLE = 1;
/**
* ADD_FILE_PATH tells add to append the row to the file at a path,
* which add opens itself.
*/
const ADD_FILE_PATH = 2;
/**
* ADD_MEM_TABLE_STRING tells add to put the row in a table held in
* memory as one string of rows, written the way save writes them, in
* place of an array of key => packed records pairs.
*/
const ADD_MEM_TABLE_STRING = 3;
/**
* REPLACE_MODE tells add and load that a row whose key is already in
* the table takes the place of the records the table has for it.
*/
const REPLACE_MODE = 0;
/**
* APPEND_MODE tells add and load that a row whose key is already in the
* table has its records added after the records the table has for it.
*/
const APPEND_MODE = 1;
/**
* AS_STRING_MODE tells load to give back the table as the string it is
* stored as, without splitting it into rows.
*/
const AS_STRING_MODE = 2;
/**
* MAX_TABLE_CACHE_BYTES is the most bytes of uncompressed table strings
* one PackedTableTools keeps in its cache, 32 megabytes. A long-running
* search server reads the same partition files again and again, and
* the cache saves reading and decompressing them each time. When
* caching a table goes past this many bytes, the tables used longest
* ago are dropped, with their arrays of row end positions, until the
* cache fits. The table just cached is kept even when it alone is
* larger, since the read in progress needs it.
*/
const MAX_TABLE_CACHE_BYTES = 33554432;
/**
* SIZE_CODES maps the number of bytes packIntColumn wrote an INT in to the
* two bit size code pack writes for the column: 0 for one byte, 1 for
* two, 2 for four and 3 for eight.
*/
const SIZE_CODES = [1 => 0, 2 => 1, 4 => 2, 8 => 3];
/**
* compressor is the Compressor that compresses a table's string when
* save writes it and decompresses it when load reads it.
* @var seekquarry\yioop\library\compressors\Compressor
*/
public $compressor;
/**
* format is the signature of the records, less the primary key: an
* array of column name => column type pairs, in the order pack writes
* the columns. Each type is one of BOOL, DOUBLE, INT, REAL and TEXT, as
* TYPE_SYNONYMS gives it.
* @var array
*/
public $format;
/**
* key_field is the name of the column used for the primary key.
* @var string
*/
public $key_field;
/**
* key_len is the number of bytes of the primary key. A value of 0 or
* less means keys vary in length, and each row gives its key's length
* in its first byte.
* @var int
*/
public $key_len;
/**
* num_bool_columns is the number of columns of a record of type BOOL.
* @var int
*/
public $num_bool_columns;
/**
* num_int_columns is the number of columns of a record of type INT.
* @var int
*/
public $num_int_columns;
/**
* num_text_columns is the number of columns of a record of type TEXT.
* @var int
*/
public $num_text_columns;
/**
* table_cache holds the uncompressed strings of tables load has read,
* under a hash of each table's path, with the table used last at the
* end. cacheTable keeps it under max_table_cache_bytes bytes.
* @var array
*/
public $table_cache;
/**
* table_cache_bytes is the sum of the lengths of the strings in
* table_cache, kept so cacheTable need not measure every string each
* time it adds one.
* @var int
*/
public $table_cache_bytes;
/**
* max_table_cache_bytes is the most bytes table_cache may hold, set to
* MAX_TABLE_CACHE_BYTES by the constructor. A test may set it lower.
* @var int
*/
public $max_table_cache_bytes;
/**
* table_entry_markers holds, under the same hash as table_cache, the
* byte positions of the \xFF bytes that end the rows of a cached table.
* cacheTable drops a table's positions when it drops the table.
* @var array
*/
public $table_entry_markers;
/**
* __construct sets up a PackedTableTools for records with the signature
* $format, whose tables are compressed with an instance of
* $compressor_type.
*
* @param array $format the signature: "PRIMARY KEY" => column name, or
* "PRIMARY KEY" => [column name, key length in bytes], and then
* column name => column type pairs, each type one of BOOL, DOUBLE,
* INT, REAL and TEXT or a name TYPE_SYNONYMS gives for one. Without a
* key length the key is DEFAULT_KEY_LEN bytes long.
* @param string $compressor_type full class name of the Compressor used
* to compress and decompress tables
* @return null when $format has a column type TYPE_SYNONYMS does not
* know or has no primary key, in which case the tool is left only
* partly set up
*/
public function __construct($format, $compressor_type =
self::DEFAULT_COMPRESSOR)
{
$this->compressor = new $compressor_type();
$this->table_cache = [];
$this->table_cache_bytes = 0;
$this->max_table_cache_bytes = self::MAX_TABLE_CACHE_BYTES;
$this->table_entry_markers = [];
$this->format = [];
$type_synonyms = self::TYPE_SYNONYMS;
$i = 0;
$this->key_field = null;
$this->key_len = self::DEFAULT_KEY_LEN;
$this->num_bool_columns = 0;
$this->num_int_columns = 0;
$this->num_text_columns = 0;
foreach ($format as $field_name => $type) {
if (strtoupper($field_name) == "PRIMARY KEY") {
if (is_array($type)) {
$this->key_field = $type[0];
$this->key_len = $type[1];
} else {
$this->key_field = $type;
}
continue;
}
$type = strtoupper($type);
if (empty($type_synonyms[$type])) {
return null;
}
$this->format[$field_name] = $type_synonyms[$type];
if ($type == "BOOL") {
$this->num_bool_columns++;
} else if ($type == "INT") {
$this->num_int_columns++;
} else if ($type == "TEXT") {
$this->num_text_columns++;
}
}
if (empty($this->key_field)) {
return null;
}
}
/**
* add puts the row $key, $table_row into $table, which may be a table in
* memory, a table string in memory, a file handle or a file path.
*
* @param mixed &$table an array of key => packed records pairs, a
* string of rows as save writes them, the path of a file of such
* rows, or a file handle at the end of such a file
* @param string $key the row's key, of the length the signature gives
* @param string $table_row records packed by pack with this tool's
* signature
* @param int $add_method which of those $table is: ADD_MEM_TABLE,
* ADD_MEM_TABLE_STRING, ADD_FILE_PATH or ADD_FILE_HANDLE
* @param int $mode for a table in memory, what happens when $key is
* already there: REPLACE_MODE puts $table_row in place of its records,
* and APPEND_MODE adds $table_row after them. A row added to a string
* or a file is always written after the rows there.
* @return bool whether the row was added
*/
public function add(&$table, $key, $table_row,
$add_method = self::ADD_MEM_TABLE, $mode = self::REPLACE_MODE)
{
$encode_key = ($this->key_len > 0) ? $key :
chr(strlen($key)) . $key;
switch ($add_method)
{
case self::ADD_FILE_PATH:
$separator = (file_exists($table) &&
filesize($table) > 0) ? "\xFF" : "";
$out = $separator . L\encode255($encode_key . $table_row);
return (file_put_contents($table, $out, FILE_APPEND) > 0);
case self::ADD_FILE_HANDLE:
$separator = (ftell($table) > 0) ? "\xFF" : "";
$out = $separator . L\encode255($encode_key . $table_row);
return (fwrite($table, $out) > 0);
case self::ADD_MEM_TABLE_STRING:
$separator = (strlen($table) > 0) ? "\xFF" : "";
$out = $separator . L\encode255($encode_key . $table_row);
$table .= $out;
return (strlen($out) > 0);
case self::ADD_MEM_TABLE:
default:
if ($mode == self::REPLACE_MODE || empty($table[$key])) {
$table[$key] = $table_row;
} else if ($mode == self::APPEND_MODE) {
$table[$key] = $this->mergeRowValues($table[$key],
$table_row);
}
}
return true;
}
/**
* mergeRowValues joins two rows of packed records, each without its key,
* into one row with the records of the first followed by those of the
* second. A row starts with its number of records, so the new row
* starts with the sum of the two counts.
*
* @param string $row_values1 first row of records, as pack makes them
* @param string $row_values2 second row of records, as pack makes them
* @return string a row of the records of $row_values1 and then those of
* $row_values2
*/
public function mergeRowValues($row_values1, $row_values2)
{
$row_values1_pos = 0;
$row_values1_count = L\vByteDecode($row_values1,
$row_values1_pos);
$row_values2_pos = 0;
$row_values2_count = L\vByteDecode($row_values2,
$row_values2_pos);
return L\vByteEncode($row_values1_count + $row_values2_count) .
substr($row_values1, $row_values1_pos) .
substr($row_values2, $row_values2_pos);
}
/**
* delete takes $key and its records out of the table $table in memory.
*
* @param array &$table array of key => packed records pairs
* @param string $key key of the row to take out
* @return bool true if $table had a row for $key, false otherwise
*/
public function delete(&$table, $key)
{
if (!empty($table[$key])) {
unset($table[$key]);
return true;
}
return false;
}
/**
* find gives the packed records the table $table in memory has for
* $key.
*
* @param array $table array of key => packed records pairs
* @param string $key key of the row to look up
* @return ?string the row's packed records, or null if $table has no
* row for $key
*/
public function find($table, $key)
{
return $table[$key] ?? null;
}
/**
* findRowFromKeyTableString looks up the packed records for $key in a
* table string, a string of rows as save writes them, by searching the
* string for the row that starts with the key. It works whatever the
* order of the rows, and reads up to the whole string.
* findRowInSortedTableString is the faster lookup for a table whose
* rows are in key order.
*
* @param string $table_string rows as save writes them: each row is
* its key and packed records, with \xFE and \xFF bytes written as two
* bytes by encode255, and rows are separated by a \xFF byte
* @param string $key key of the row to look up
* @return ?string the row's packed records, or null if no row has $key
*/
public function findRowFromKeyTableString($table_string, $key)
{
$fixed_length_key = ($this->key_len > 0);
/* A row holds its key as encode255 wrote it, after the key's length
when keys vary in length */
$encoded_key = L\encode255(($fixed_length_key) ? $key :
chr(strlen($key)) . $key);
if (strncmp($encoded_key, $table_string, strlen($encoded_key)) == 0) {
$start_row = 0;
} else {
$start_row = strpos($table_string, "\xFF" . $encoded_key);
if ($start_row === false) {
return null;
}
$start_row += 1;
}
$next_ff_pos = strpos($table_string, "\xFF", $start_row);
if ($next_ff_pos === false) {
$next_ff_pos = strlen($table_string);
}
$key_offset = $fixed_length_key ? 0 : 1;
$key_len = ($fixed_length_key) ? $this->key_len : strlen($key);
$entry = L\decode255(substr($table_string, $start_row,
$next_ff_pos - $start_row));
return substr($entry, $key_len + $key_offset);
}
/**
* findRowInSortedTableString looks up the packed records for $key in a
* table string whose rows are in key order, as save writes a table that
* was sorted by PHP's ksort (which sorts an array by its keys). It does
* a binary search on byte positions: from a middle byte it reads the
* first row that starts there or later, and keeps the half of the
* string that can still hold the key. A lookup reads about as many
* rows as the logarithm of the number of rows, where
* findRowFromKeyTableString may read the whole string.
*
* @param string $table_string rows as save writes them, in the order
* ksort gives their keys: each row is its key and packed records, with
* \xFE and \xFF bytes written as two bytes by encode255, and rows are
* separated by a \xFF byte
* @param string $key key of the row to look up
* @return ?string the row's packed records, or null if no row has $key
*/
public function findRowInSortedTableString($table_string, $key)
{
$fixed_length_key = ($this->key_len > 0);
$key_offset = $fixed_length_key ? 0 : 1;
/* encode255 writes a byte as at most two bytes, so this many bytes
of a row hold its key, and its length byte when keys vary in
length, as a byte of at most 255 */
$encoded_key_len = $fixed_length_key ? 2 * $this->key_len : 512;
$low = 0;
$high = strlen($table_string);
while ($low < $high) {
$middle = ($low + $high) >> 1;
if ($middle == 0) {
$row_start = 0;
} else {
$marker = strpos($table_string, "\xFF", $middle - 1);
$row_start = ($marker === false) ? $high : $marker + 1;
}
if ($row_start >= $high) {
$high = $middle;
continue;
}
$row_end = strpos($table_string, "\xFF", $row_start);
if ($row_end === false) {
$row_end = strlen($table_string);
}
$entry_start = L\decode255(substr($table_string, $row_start,
min($row_end - $row_start, $encoded_key_len)));
$key_len = ($fixed_length_key) ? $this->key_len :
ord($entry_start[0] ?? "\x00");
$row_key = substr($entry_start, $key_offset, $key_len);
/* ksort compares keys as <=> does, which is byte order unless
both keys look like numbers */
$order = ((string)$row_key <=> (string)$key);
if ($order == 0) {
$entry = L\decode255(substr($table_string, $row_start,
$row_end - $row_start));
return substr($entry, $key_len + $key_offset);
}
if ($order < 0) {
$low = $row_end + 1;
} else {
$high = $middle;
}
}
return null;
}
/**
* getEntryMarkers gives the byte positions of the \xFF (byte 255)
* bytes that end the rows of the table in the file $table_name, so the
* row with a given number can be read without reading the rows before
* it. The table is loaded and cached by load, and its positions are
* cached with it.
*
* @param string $table_name path of the file holding the table
* @return array of the byte positions, in order, of the \xFF bytes in
* the table's string
*/
public function getEntryMarkers($table_name)
{
$hash_name = L\crawlHash($table_name);
if (!empty($this->table_entry_markers[$hash_name])) {
return $this->table_entry_markers[$hash_name];
}
$this->table_entry_markers[$hash_name] = [];
$table_string = $this->load($table_name, self::AS_STRING_MODE, true);
$delim = "\xFF";
$this->table_entry_markers[$hash_name] = [];
if (preg_match_all("/$delim/", $table_string, $matches,
PREG_OFFSET_CAPTURE)) {
$this->table_entry_markers[$hash_name] =
array_column($matches[0], 1);
}
return $this->table_entry_markers[$hash_name];
}
/**
* findEntryAtIndexTableName gives the row numbered $index, counting from
* 0, of the table in the file $table_name, using getEntryMarkers to find
* where the row starts and ends.
*
* @param string $table_name path of the file holding the table
* @param int $index number of the row to read, counting from 0
* @return string the row, its key followed by its packed records, with
* the \xFE and \xFF bytes encode255 wrote as two bytes put back; the
* empty string if the table has no such row
*/
public function findEntryAtIndexTableName($table_name, $index)
{
$table_string = $this->load($table_name, self::AS_STRING_MODE, true);
$entry_markers = $this->getEntryMarkers($table_name);
/* a table of n rows has n - 1 markers between them */
if ($index < 0 || $index > count($entry_markers) ||
$table_string === "") {
return "";
}
$start_pos = $entry_markers[$index - 1] ?? 0;
$start_pos = ($start_pos == 0) ? 0 : $start_pos + 1;
$end_pos = $entry_markers[$index] ?? strlen($table_string);
$entry = L\decode255(substr($table_string, $start_pos,
$end_pos - $start_pos));
return $entry;
}
/**
* load reads the table save wrote to the file $table_path, either as
* the string it is stored as or as an array of key => packed records
* pairs.
*
* @param string $table_path path of the file holding the table
* @param int $mode REPLACE_MODE to make an array in which a later row
* with a key takes the place of an earlier one, APPEND_MODE to make an
* array in which the records of rows with the same key are joined by
* mergeRowValues, and AS_STRING_MODE to give back the string without
* splitting it into rows
* @param bool $cache_table whether to keep the table's string in
* table_cache, and to use the string there if it is already held
* @return mixed the table's string for AS_STRING_MODE, otherwise the
* array of key => packed records pairs, empty for an empty table;
* if there is no file, the empty string for AS_STRING_MODE and null
* otherwise
*/
public function load($table_path, $mode = self::REPLACE_MODE,
$cache_table = false)
{
if ($cache_table) {
$hash_name = L\crawlHash($table_path);
}
if ($cache_table && isset($this->table_cache[$hash_name])) {
$table = $this->table_cache[$hash_name];
unset($this->table_cache[$hash_name]);
$this->table_cache[$hash_name] = $table;
} else {
if (!file_exists($table_path)) {
return ($mode == PackedTableTools::AS_STRING_MODE) ? "" : null;
}
$table = $this->compressor->uncompressGetFile($table_path);
if ($cache_table) {
$this->cacheTable($hash_name, $table);
}
}
if ($mode == self::AS_STRING_MODE) {
return $table;
}
$rows = [];
if ($table === "") {
return $rows;
}
$fixed_length_key = ($this->key_len > 0);
$replace_mode = ($mode == self::REPLACE_MODE);
$default_key_len = $this->key_len;
$key_offset = $fixed_length_key ? 0 : 1;
$more_rows = true;
$cur_start_row = 0;
while ($more_rows) {
$next_ff_pos = strpos($table, "\xFF", $cur_start_row);
if ($next_ff_pos === false) {
$next_ff_pos = strlen($table);
$more_rows = false;
}
$entry = L\decode255(substr($table, $cur_start_row,
$next_ff_pos - $cur_start_row));
$cur_start_row = $next_ff_pos + 1;
$key_len = ($fixed_length_key) ? $default_key_len :
ord($entry[0]);
$key = substr($entry, $key_offset, $key_len);
$new_values = substr($entry, $key_len + $key_offset);
$rows[$key] = ($replace_mode || empty($rows[$key])) ? $new_values :
$this->mergeRowValues($rows[$key], $new_values);
}
return $rows;
}
/**
* cacheTable puts a table string load has just read into table_cache,
* and then drops the tables used longest ago until the cache holds at
* most max_table_cache_bytes bytes. A dropped table's row end positions
* in table_entry_markers are dropped with it. The table just added is
* kept even when it alone is larger, since the read in progress needs
* it.
*
* @param string $hash_name hash of the table's path, its name in the
* cache
* @param string $table the table's uncompressed string
*/
private function cacheTable($hash_name, $table)
{
$this->table_cache[$hash_name] = $table;
$this->table_cache_bytes += strlen($table);
while ($this->table_cache_bytes > $this->max_table_cache_bytes &&
count($this->table_cache) > 1) {
$oldest = array_key_first($this->table_cache);
$this->table_cache_bytes -=
strlen($this->table_cache[$oldest]);
unset($this->table_cache[$oldest]);
unset($this->table_entry_markers[$oldest]);
}
}
/**
* packIntColumn writes an INT column's value as pack writes it: in one,
* two, four or eight bytes, the fewest that hold its size, with the top
* bit of the first byte set when the value is negative. The number of bytes
* gives the column's two bit size code through SIZE_CODES.
*
* @param int $value the value to write
* @return string its bytes
*/
public static function packIntColumn($value)
{
$magnitude = abs($value);
$sign_bit = ($value == $magnitude) ? 0 : 128;
if ($magnitude < 128) {
return chr($magnitude + $sign_bit);
}
if ($magnitude < 32768) {
$packed_int = pack("n", $magnitude);
} else if ($magnitude < 2147483647) {
$packed_int = pack("N", $magnitude);
} else {
$packed_int = pack("J", $magnitude);
}
if ($sign_bit) {
$packed_int[0] = chr(ord($packed_int[0]) + $sign_bit);
}
return $packed_int;
}
/**
* pack writes records with this tool's signature as one string. The
* string starts with the number of records, written by vByteEncode
* (which writes a number in as few bytes as it needs). Each record
* starts with its BOOL columns as bits, eight to a byte, in signature
* order. Next comes a two bit size code for each INT column, four to a
* byte: 00 for one byte, 01 for two, 10 for four and 11 for eight. Next
* comes one byte for each TEXT column giving its length. Last come the
* DOUBLE, INT, REAL and TEXT columns in signature order: a DOUBLE in
* eight bytes, a REAL in four, an INT in the bytes its size code gives,
* with the top bit of its first byte set when it is negative, and a
* TEXT column as its bytes.
*
* @param array $items a record, an array of column name => value pairs,
* or an array of such records
* @return ?string the packed records, or null if a record lacks a
* column of the signature or has a TEXT column longer than 255 bytes
*/
public function pack($items)
{
$format = $this->format;
if (!isset($items[0])) {
$items = [$items];
}
$packed_items = L\vByteEncode(count($items));
foreach ($items as $item) {
$bool_column_data = "";
$cur_bool_int = -1;
$cur_bool_shift = 7;
$int_column_types = "";
$text_column_lengths = "";
$shift = 6;
$cur_int_char = 0;
$int_occurred = false;
$packed_data = "";
foreach ($format as $field_name => $type) {
if (!isset($item[$field_name])) {
return null;
}
switch ($type)
{
case "BOOL":
$cur_bool_int = ($cur_bool_int == -1) ? 0 :
$cur_bool_int;
$bool_value = ($item[$field_name]) ? 1 : 0;
$cur_bool_int += ($bool_value << $cur_bool_shift);
$cur_bool_shift--;
if ($cur_bool_shift < 0) {
$bool_column_data .= chr($cur_bool_int);
$cur_bool_shift = 7;
$cur_bool_int = -1;
}
break;
case "DOUBLE":
$packed_data .= pack("E", $item[$field_name]);
break;
case "INT":
$int_occurred = true;
$packed_int = self::packIntColumn($item[$field_name]);
$packed_data .= $packed_int;
$cur_int_char = max($cur_int_char, 0) +
(self::SIZE_CODES[strlen($packed_int)] << $shift);
$shift -= 2;
if ($shift < 0) {
$int_column_types .= chr($cur_int_char);
$cur_int_char = -1;
$shift = 6;
}
break;
case "REAL":
$packed_data .= pack("G", $item[$field_name]);
break;
case "TEXT":
$len = strlen($item[$field_name]);
if ($len > 255) {
return null;
}
$packed_data .= $item[$field_name];
$text_column_lengths .= chr($len);
break;
}
}
if ($cur_bool_int != -1) {
$bool_column_data .= chr($cur_bool_int);
}
if ($cur_int_char != -1 && $int_occurred) {
$int_column_types .= chr($cur_int_char);
}
$packed_items .= $bool_column_data .
$int_column_types . $text_column_lengths . $packed_data;
}
return $packed_items;
}
/**
* save writes a table to the file $table_path. Each row is its key
* followed by its packed records, with the key's length in a byte
* before the key when keys vary in length. encode255 then writes each
* \xFE byte of the row as \xFE\xFD and each \xFF byte as \xFE\xFE, so
* the rows can be joined with \xFF bytes between them. The rows are
* written in the order of the array, and the string is compressed by
* compressor.
*
* @param string $table_path path of the file to write
* @param mixed $table a table string as this method writes them, or an
* array of key => packed records pairs
* @param bool $sync whether to sync the file and its folder to the
* disk before returning, so the table survives a power loss. A
* caller that syncs the step's files itself, through
* UndoLog::finish, or whose table a crash may lose, passes false;
* the rename still leaves a killed process's table whole.
* @return bool whether the table was written
*/
public function save($table_path, $table, $sync = true)
{
if (is_string($table)) {
$out = $table;
} else {
$out = "";
if ($this->key_len > 0) {
foreach ($table as $key => $row) {
$out .= "\xFF" . L\encode255($key . $row);
}
} else {
foreach ($table as $key => $row) {
$out .= "\xFF" . L\encode255(
chr(strlen($key)) . $key . $row);
}
}
$out = substr($out, 1);
}
/* The table is written to a new name and renamed into place, so
a process killed part way leaves the old table whole rather
than a cut one whose offsets point at garbage. */
$writing = "$table_path.writing-" . getmypid();
/* An empty table, as a partition stopped before any document was
saved has, is written too, as an empty file: the partition's
build is whole only once all four of its files are there. */
if ($this->compressor->compressPutFile($writing, $out) !== false) {
if ($sync) {
UndoLog::syncPath($writing);
}
if (rename($writing, $table_path)) {
if ($sync) {
UndoLog::syncPath(dirname($table_path));
}
return true;
}
}
@unlink($writing);
return false;
}
/**
* count gives the number of records in a row of records packed by pack,
* which is the number the row starts with.
*
* @param string $table_row records packed by pack
* @return int the number of records in $table_row
*/
public function count($table_row)
{
$current_pos = 0;
return L\vByteDecode($table_row, $current_pos);
}
/**
* unpack reads records that pack wrote back into arrays of column name
* => value pairs, starting at record $offset and giving at most $limit
* records.
*
* @param string $table_row records packed by pack
* @param int $offset number of the first record to give, counting from
* 0
* @param int $limit most records to give; 0 or less for all records
* from $offset on
* @return ?array the records; an empty array if $offset is past the
* last record; null if a record does not fit the signature
*/
public function unpack($table_row, $offset = 0, $limit = -1)
{
$num_int_columns = $this->num_int_columns;
$num_text_columns = $this->num_text_columns;
$num_bool_columns = $this->num_bool_columns;
$bool_info_len = intval(ceil($this->num_bool_columns/8));
$int_info_len = intval(ceil($num_int_columns / 4));
$unpack_code = [1 => "n", 2=> "N", 3 => "J"];
$current_pos = 0;
$num_items = L\vByteDecode($table_row, $current_pos);
if ($offset >= $num_items || empty($table_row)) {
return [];
}
$limit = ($limit <= 0) ? $num_items : $limit;
$len_row = strlen($table_row);
$num_items = min($limit + $offset, $num_items);
for ($i = 0; $i < $num_items; $i++) {
$bool_info = substr($table_row, $current_pos, $bool_info_len);
$current_pos += $bool_info_len;
$int_info = substr($table_row, $current_pos, $int_info_len);
$current_pos += $int_info_len;
$text_info = substr($table_row, $current_pos, $num_text_columns);
$current_pos += $num_text_columns;
$format = $this->format;
$item = [];
$current_bool_pos = 0;
$cur_bool_shift = 7;
$current_int_pos = 0;
$current_text_pos = 0;
$shift = 6;
$bools_used = 0;
$ints_used = 0;
foreach ($format as $field_name => $type) {
switch ($type)
{
case "BOOL":
if ($bools_used >= $num_bool_columns) {
return null;
}
$item[$field_name] =
((ord($bool_info[$current_bool_pos]) &
(1 << $cur_bool_shift)) >> $cur_bool_shift) > 0;
$bools_used ++;
$cur_bool_shift --;
if ($cur_bool_shift < 0) {
$current_bool_pos ++;
$cur_bool_shift = 7;
}
break;
case "DOUBLE":
$len = min(8, $len_row - $current_pos);
$item[$field_name] = ($len == 8) ?
unpack("E", $table_row, $current_pos)[1] : 0;
$current_pos += $len;
break;
case "INT":
if ($ints_used >= $num_int_columns) {
return null;
}
if ($int_info === "") {
$item[$field_name] = 0;
break;
}
$int_code = (ord($int_info[$current_int_pos]) &
(3 << $shift)) >> $shift;
if (!isset($table_row[$current_pos])) {
return null;
}
$first_char = ord($table_row[$current_pos]);
$len = 1 << $int_code;
if ($int_code == 0) {
$value = $first_char;
if ($value > 127) {
$value = - ($value & 127);
}
$item[$field_name] = $value;
} else {
$sign = ($first_char > 127) ? -1 : 1;
if ($sign < 0) {
$table_row[$current_pos] =
chr($first_char - 128);
}
$item[$field_name] = $sign * (unpack(
$unpack_code[$int_code], $table_row,
$current_pos)[1]);
}
$current_pos += $len;
$ints_used++;
$shift -= 2;
if ($shift < 0) {
$current_int_pos++;
$shift = 6;
}
break;
case "REAL":
$len = min(4, $len_row - $current_pos);
$item[$field_name] = ($len == 4) ?
unpack("G", $table_row, $current_pos)[1] : 0;
$current_pos += $len;
break;
case "TEXT":
if ($current_text_pos >= $num_text_columns) {
return null;
}
$text_len = ord($text_info[$current_text_pos]);
$item[$field_name] = substr($table_row, $current_pos,
$text_len);
$current_pos += $text_len;
$current_text_pos++;
break;
}
}
if ($i >= $offset) {
$items[] = $item;
}
}
return $items;
}
}
- Yioop Repo - Seekquarry