<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see
* <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
/**
* Decodes a black-and-white image coded the way scanned text is coded,
* giving back one byte per pixel: 0 for black, 1 for white.
*
* This coding is defined by ITU-T Recommendation T.88, known as JBIG2. It
* is met as a PDF stream whose Filter is JBIG2Decode, which is how a
* scanner or a magazine's typesetter often carries lettering meant to be
* painted in one color.
*
* The coding contexts each pixel from the ten pixels already decoded
* around it, and writes only how wrong the guess was, using an arithmetic
* coder that spends less than one bit on a pixel it expects. Both the
* coder and the guessing are done here.
*
* What is handled and what is not:
* - A stream of segments as a PDF embeds them, of which the page
* information and an immediate generic region are read and the rest
* skipped.
* - Generic region decoding with the arithmetic coder, all four
* templates, the adaptive pixels the region names, and the
* typical-prediction shortcut where a region says a row repeats the one
* above it.
* - It does NOT handle: symbol dictionaries and text regions, which is
* how a scanner stores a page as reused letter shapes; refinement;
* halftone regions; or generic regions coded the fax way rather than
* arithmetically. Nothing comes back for those rather than a wrong
* picture.
*
* Typical use:
* $rows = JBig2::decode($bytes, $width, $height);
* // $rows[$down][$across] is the character 0 for black or 1 for white
*
* @author Chris Pollett
*/
class JBig2
{
/**
* The chance of the less likely answer at each state the arithmetic
* coder can be in, with where to move next when the guess was right,
* where to move when it was wrong, and whether being wrong turns the
* two answers around. This is the table T.88 gives.
* @var array
*/
const STATES = [
[0x5601, 1, 1, 1], [0x3401, 2, 6, 0], [0x1801, 3, 9, 0],
[0x0AC1, 4, 12, 0], [0x0521, 5, 29, 0], [0x0221, 38, 33, 0],
[0x5601, 7, 6, 1], [0x5401, 8, 14, 0], [0x4801, 9, 14, 0],
[0x3801, 10, 14, 0], [0x3001, 11, 17, 0], [0x2401, 12, 18, 0],
[0x1C01, 13, 20, 0], [0x1601, 29, 21, 0], [0x5601, 15, 14, 1],
[0x5401, 16, 14, 0], [0x5101, 17, 15, 0], [0x4801, 18, 16, 0],
[0x3801, 19, 17, 0], [0x3401, 20, 18, 0], [0x3001, 21, 19, 0],
[0x2801, 22, 19, 0], [0x2401, 23, 20, 0], [0x2201, 24, 21, 0],
[0x1C01, 25, 22, 0], [0x1801, 26, 23, 0], [0x1601, 27, 24, 0],
[0x1401, 28, 25, 0], [0x1201, 29, 26, 0], [0x1101, 30, 27, 0],
[0x0AC1, 31, 28, 0], [0x09C1, 32, 29, 0], [0x08A1, 33, 30, 0],
[0x0521, 34, 31, 0], [0x0441, 35, 32, 0], [0x02A1, 36, 33, 0],
[0x0221, 37, 34, 0], [0x0141, 38, 35, 0], [0x0111, 39, 36, 0],
[0x0085, 40, 37, 0], [0x0049, 41, 38, 0], [0x0025, 42, 39, 0],
[0x0015, 43, 40, 0], [0x0009, 44, 41, 0], [0x0005, 45, 42, 0],
[0x0001, 45, 43, 0], [0x5601, 46, 46, 0]];
/**
* How many pixels the first template looks at, which sets how many
* guesses there can be
* @var int
*/
const TEMPLATE_PIXELS = 16;
/**
* The kind number a segment carries when it holds page information
* @var int
*/
const PAGE_SEGMENT = 48;
/**
* The kind numbers a segment carries when it holds a generic region
* drawn straight onto the page
* @var array
*/
const REGION_SEGMENTS = [36, 38, 39];
/**
* The bytes being read
* @var string
*/
public $bytes;
/**
* Where in the bytes reading has got to
* @var int
*/
public $at;
/**
* The part of the range the coder is working in
* @var int
*/
public $range;
/**
* What the coder has read so far, held against the range
* @var int
*/
public $held;
/**
* How many bits of the held value are still good
* @var int
*/
public $good;
/**
* Which state each guess is in, and which answer it thinks more
* likely
* @var array
*/
public $contexts = [];
/**
* Sets up an arithmetic coder over a run of bytes.
*
* @param string $bytes the coded bytes
* @param int $at where in them to begin
*/
public function __construct($bytes, $at = 0)
{
$this->bytes = $bytes;
$this->at = $at;
$this->startArithmeticDecoder();
}
/**
* Decodes an image, giving back one byte per pixel. The bytes are a
* run of segments as a document embeds them; the first generic region
* among them is the picture.
*
* @param string $bytes the coded image
* @param int $wide how many pixels across
* @param int $high how many rows
* @return array each row as a string of 0 for black and 1 for white,
* empty where the coding is one this does not handle
*/
public static function decode($bytes, $wide, $high)
{
$found = self::genericRegionIn($bytes);
if ($found === false) {
return [];
}
$reader = new JBig2($bytes, $found["at"]);
return $reader->readGenericRegion($found["wide"], $found["high"],
$found["template"], $found["adaptive"], $found["typical"]);
}
/**
* Finds the generic region among a run of segments, and reads what it
* says about how it was coded.
*
* @param string $bytes the coded image
* @return mixed where the region's data begins and how it was coded,
* or false where there is no generic region to read
*/
public static function genericRegionIn($bytes)
{
$at = 0;
$length = strlen($bytes);
$steps = 0;
while ($at + 11 <= $length && $steps < 64) {
$steps++;
$number = unpack("N", substr($bytes, $at, 4))[1];
$flags = ord($bytes[$at + 4]);
$kind = $flags & 0x3f;
$page_wide = ($flags & 0x40) ? 4 : 1;
$refs = ord($bytes[$at + 5]) >> 5;
if ($refs == 7) {
return false;
}
$each = ($number <= 256) ? 1 : (($number <= 65536) ? 2 : 4);
$header = $at + 6 + $refs * $each + $page_wide;
if ($header + 4 > $length) {
return false;
}
$said = unpack("N", substr($bytes, $header, 4))[1];
$begins = $header + 4;
if (in_array($kind, self::REGION_SEGMENTS)) {
return self::genericRegionSettings($bytes, $begins);
}
$at = $begins + $said;
}
return false;
}
/**
* Reads what a generic region says about itself: how large it is, how
* it contexts a pixel from those around it, and which surrounding
* pixels it moved.
*
* @param string $bytes the coded image
* @param int $at where the region's data begins
* @return mixed the region's settings, or false where it is coded a
* way this does not handle
*/
public static function genericRegionSettings($bytes, $at)
{
if ($at + 18 > strlen($bytes)) {
return false;
}
$wide = unpack("N", substr($bytes, $at, 4))[1];
$high = unpack("N", substr($bytes, $at + 4, 4))[1];
$flags = ord($bytes[$at + 17]);
if ($flags & 1) {
/* Coded the fax way rather than arithmetically, which is a
different decoder. */
return false;
}
$template = ($flags >> 1) & 3;
$typical = (($flags >> 3) & 1) == 1;
$pairs = ($template == 0) ? 4 : 1;
$adaptive = [];
$reads = $at + 18;
for ($step = 0; $step < $pairs; $step++) {
$first_byte = ord($bytes[$reads]);
$second_byte = ord($bytes[$reads + 1]);
$adaptive[] = [($first_byte > 127) ? $first_byte - 256 :
$first_byte, ($second_byte > 127) ? $second_byte - 256 :
$second_byte];
$reads += 2;
}
return ["at" => $reads, "wide" => $wide, "high" => $high,
"template" => $template, "adaptive" => $adaptive,
"typical" => $typical];
}
/**
* Sets the arithmetic coder going, which reads its first bytes and
* lines them up against the range.
*/
public function startArithmeticDecoder()
{
$this->held = $this->codedByteAt($this->at) << 16;
$this->takeNextCodedByte();
$this->held <<= 7;
$this->good -= 7;
$this->range = 0x8000;
}
/**
* Gives the byte at a place, or a settled value past the end so the
* coder can run on to the end of the picture.
*
* @param int $at which byte
* @return int the byte
*/
public function codedByteAt($at)
{
return ($at < strlen($this->bytes)) ? ord($this->bytes[$at]) : 0xff;
}
/**
* Takes in the next byte, which the coder does whenever it has used
* up the bits it holds.
*/
public function takeNextCodedByte()
{
if ($this->codedByteAt($this->at) == 0xff) {
if ($this->codedByteAt($this->at + 1) > 0x8f) {
$this->held += 0xff00;
$this->good = 8;
} else {
$this->at++;
$this->held += $this->codedByteAt($this->at) << 9;
$this->good = 7;
}
} else {
$this->at++;
$this->held += $this->codedByteAt($this->at) << 8;
$this->good = 8;
}
}
/**
* Reads one answer from the coder, given which guess is being asked.
* The guess remembers how often it has been right and moves to a
* state that spends fewer bits when it keeps being right.
*
* @param int $which which guess
* @return int the answer, 0 or 1
*/
public function readDecisionBit($which)
{
if (!isset($this->contexts[$which])) {
$this->contexts[$which] = [0, 0];
}
list($state, $likely) = $this->contexts[$which];
$chance = self::STATES[$state][0];
$this->range -= $chance;
$answer = $likely;
if ((($this->held >> 16) & 0xffff) < $chance) {
/* The less likely answer came up. */
if ($this->range < $chance) {
$this->range = $chance;
$this->contexts[$which] = [self::STATES[$state][1],
$likely];
} else {
$this->range = $chance;
$answer = 1 - $likely;
if (self::STATES[$state][3] == 1) {
$likely = 1 - $likely;
}
$this->contexts[$which] = [self::STATES[$state][2],
$likely];
}
} else {
$this->held -= $chance << 16;
if (($this->range & 0x8000) != 0) {
return $answer;
}
if ($this->range < $chance) {
$answer = 1 - $likely;
if (self::STATES[$state][3] == 1) {
$likely = 1 - $likely;
}
$this->contexts[$which] = [self::STATES[$state][2],
$likely];
} else {
$this->contexts[$which] = [self::STATES[$state][1],
$likely];
}
}
/* Whichever way it went, the range has shrunk below half, so it
and what is held are doubled until it is back above. */
do {
if ($this->good == 0) {
$this->takeNextCodedByte();
}
$this->range <<= 1;
$this->held = ($this->held << 1) & 0xffffffff;
$this->good--;
} while (($this->range & 0x8000) == 0);
return $answer;
}
/**
* readGenericRegion decodes a generic region row by row. For each
* pixel it forms a guess from the pixels already decoded near it, and
* the arithmetic coder says whether the guess holds.
*
* The pixels a guess looks at are taken as runs that lie side by side
* on one row, and each run is kept as a small number that moves one
* pixel along the row at each step, so a pixel costs a few shifts
* rather than a lookup for every pixel around it. A region that moved
* its adaptive pixels off the runs has each pixel around looked up
* instead.
*
* Only the rows a guess can reach back to are kept as lists of
* numbers; each finished row is kept as a string, one byte to a
* pixel, so a page scanned at 600 dots an inch fits in under 100
* megabytes.
*
* @param int $wide how many pixels across
* @param int $high how many rows
* @param int $template which set of surrounding pixels to look at
* @param array $adaptive the surrounding pixels the region moved
* @param bool $typical whether a row may say it repeats the one above
* @return array each row as a string of 0 for black and 1 for white
*/
public function readGenericRegion($wide, $high, $template, $adaptive,
$typical)
{
$places = self::templateOffsets($template, $adaptive);
$count = count($places);
$runs = [];
$reach = 1;
$margin = 1;
foreach ($places as $k => $place) {
$last = count($runs) - 1;
if ($last >= 0 && $runs[$last]["down"] == $place[1] &&
$runs[$last]["right"] + 1 == $place[0]) {
$runs[$last]["right"] = $place[0];
$runs[$last]["length"]++;
} else {
$runs[] = ["down" => $place[1], "right" => $place[0],
"length" => 1];
}
$runs[count($runs) - 1]["shift"] = $count - 1 - $k;
$reach = max($reach, -$place[1]);
$margin = max($margin, abs($place[0]));
}
/* Three runs at most, one to a row, is how every template lies
when the region leaves its adaptive pixels where the standard
puts them; a run of no length stands in for a missing row. */
$rows_of_runs = array_column($runs, "down");
$sliding = count($runs) <= 3 &&
count(array_unique($rows_of_runs)) == count($runs);
while (count($runs) < 3) {
$runs[] = ["down" => -1, "right" => 0, "length" => 0,
"shift" => 0];
}
$blank = array_fill(0, $wide + 2 * $margin, 0);
$states = array_fill(0, 1 << self::TEMPLATE_PIXELS, 0);
$likely = $states;
$chance_of = array_column(self::STATES, 0);
$after_right = array_column(self::STATES, 1);
$after_wrong = array_column(self::STATES, 2);
$turns = array_column(self::STATES, 3);
$bytes = $this->bytes;
$length = strlen($bytes);
$at = $this->at;
$range = $this->range;
$held = $this->held;
$good = $this->good;
$repeat_guess = [0x9b25, 0x0795, 0x00e5, 0x0195][$template] ??
0x9b25;
$recent = [];
$rows = [];
$same = 0;
for ($down = 0; $down < $high; $down++) {
$row = $blank;
$sources = [];
$masks = [];
$windows = [];
foreach ($runs as $i => $run) {
$sources[$i] = ($run["down"] == 0) ? null :
($recent[$down + $run["down"]] ?? $blank);
$masks[$i] = (1 << $run["length"]) - 1;
$windows[$i] = 0;
for ($j = $run["length"] - 1; $j >= 0; $j--) {
$windows[$i] = ($windows[$i] << 1) |
($sources[$i][$margin - 1 + $run["right"] - $j] ??
0);
}
}
list($first_source, $second_source, $third_source) = $sources;
list($first_mask, $second_mask, $third_mask) = $masks;
list($first_window, $second_window, $third_window) = $windows;
$first_reads = $margin + $runs[0]["right"];
$second_reads = $margin + $runs[1]["right"];
$third_reads = $margin + $runs[2]["right"];
$first_shift = $runs[0]["shift"];
$second_shift = $runs[1]["shift"];
$repeats = false;
for ($across = $typical ? -1 : 0; $across < $wide; $across++) {
if ($across < 0) {
$which = $repeat_guess;
} else if ($sliding) {
$first_window = (($first_window << 1) |
($first_source === null ?
$row[$first_reads + $across] :
$first_source[$first_reads + $across])) &
$first_mask;
$second_window = (($second_window << 1) |
($second_source === null ?
$row[$second_reads + $across] :
$second_source[$second_reads + $across])) &
$second_mask;
$third_window = (($third_window << 1) |
($third_source === null ?
$row[$third_reads + $across] :
$third_source[$third_reads + $across])) &
$third_mask;
$which = ($first_window << $first_shift) |
($second_window << $second_shift) | $third_window;
} else {
$which = 0;
foreach ($places as $place) {
$source = ($place[1] == 0) ? $row :
($recent[$down + $place[1]] ?? $blank);
$which = ($which << 1) |
($source[$margin + $across + $place[0]] ?? 0);
}
}
/* This is readDecisionBit worked inline, since a call
for every pixel costs more than the decision does. */
$state = $states[$which];
$chance = $chance_of[$state];
$range -= $chance;
$answer = $likely[$which];
if ((($held >> 16) & 0xffff) < $chance) {
if ($range < $chance) {
$states[$which] = $after_right[$state];
} else {
$answer = 1 - $answer;
if ($turns[$state] == 1) {
$likely[$which] = $answer;
}
$states[$which] = $after_wrong[$state];
}
$range = $chance;
} else {
$held -= $chance << 16;
if (($range & 0x8000) == 0) {
if ($range < $chance) {
$answer = 1 - $answer;
if ($turns[$state] == 1) {
$likely[$which] = $answer;
}
$states[$which] = $after_wrong[$state];
} else {
$states[$which] = $after_right[$state];
}
}
}
while (($range & 0x8000) == 0) {
if ($good == 0) {
$next = ($at + 1 < $length) ? ord($bytes[$at + 1]) :
0xff;
if ((($at < $length) ? ord($bytes[$at]) : 0xff) ==
0xff) {
if ($next > 0x8f) {
$held += 0xff00;
$good = 8;
} else {
$at++;
$held += $next << 9;
$good = 7;
}
} else {
$at++;
$held += $next << 8;
$good = 8;
}
}
$range <<= 1;
$held = ($held << 1) & 0xffffffff;
$good--;
}
if ($across >= 0) {
$row[$margin + $across] = $answer;
continue;
}
$same ^= $answer;
if ($same == 1 && $down > 0) {
$repeats = true;
break;
}
}
if ($repeats) {
$recent[$down] = $recent[$down - 1];
$rows[$down] = $rows[$down - 1];
} else {
$recent[$down] = $row;
/* A one in this coding means ink, where a picture wants 0
for black, so the two are turned around here. */
$rows[$down] = strtr(implode("", array_slice($row, $margin,
$wide)), "01", "10");
}
unset($recent[$down - $reach]);
}
$this->at = $at;
$this->range = $range;
$this->held = $held;
$this->good = $good;
return $rows;
}
/**
* Gives the pixels a template looks at, in the order their answers
* are strung together, with the ones the region moved put in place.
*
* @param int $template which set of surrounding pixels
* @param array $adaptive the pixels the region moved
* @return array each as how far across and how far up
*/
public static function templateOffsets($template, $adaptive)
{
$first = $adaptive[0] ?? [3, -1];
$second = $adaptive[1] ?? [-3, -1];
$third = $adaptive[2] ?? [2, -2];
$fourth = $adaptive[3] ?? [-2, -2];
if ($template == 0) {
return [$fourth, [-1, -2], [0, -2], [1, -2], $third,
$second, [-2, -1], [-1, -1], [0, -1], [1, -1], [2, -1],
$first, [-4, 0], [-3, 0], [-2, 0], [-1, 0]];
}
if ($template == 1) {
return [[-1, -2], [0, -2], [1, -2], [2, -2], [-2, -1],
[-1, -1], [0, -1], [1, -1], [2, -1], $first,
[-3, 0], [-2, 0], [-1, 0]];
}
if ($template == 2) {
return [[-1, -2], [0, -2], [1, -2], [-2, -1], [-1, -1],
[0, -1], [1, -1], $first, [-2, 0], [-1, 0]];
}
return [[-3, -1], [-2, -1], [-1, -1], [0, -1], [1, -1], $first,
[-4, 0], [-3, 0], [-2, 0], [-1, 0]];
}
}
- Yioop Repo - Seekquarry