<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
/**
* Reads markup one character at a time so a parser can hunt for the next
* token without copying the rest of the document. It keeps the source and
* a single cursor position; every read returns a small span cut from the
* source rather than a rebuilt tail, so a parser layered on top uses
* memory proportional to how deeply blocks are nested, not to the length
* of what remains.
*
* The set of characters that can begin a block is supplied to the
* constructor, so the same scanner drives different markup flavors: a wiki
* parser passes the characters that open wiki blocks, a markdown parser
* passes markdown's, and so on.
*
* @author Chris Pollett
*/
class MarkUpScanner
{
/**
* The whole markup document being scanned.
* @var string
*/
private $source;
/**
* Byte offset of the cursor into the source.
* @var int
*/
private $position;
/**
* Length of the source in bytes, cached so it is not recomputed.
* @var int
*/
private $length;
/**
* Characters that, when first on a line, may begin a new block (for
* example headings, list items, indents). Which ones apply is a
* property of the markup flavor, so the parser supplies them.
* @var string
*/
private $block_start_chars;
/**
* Sets up a scanner over a piece of markup, first normalizing its
* line endings to plain newlines so a stray carriage return cannot
* ride along at the end of a line and defeat a block matcher.
*
* @param string $source the markup to scan
* @param string $block_start_chars each character that may start a
* block for this markup flavor, concatenated (for wiki, for
* instance, the heading, list, indent, rule, brace, and tag
* openers)
*/
public function __construct($source, $block_start_chars)
{
$this->source = str_replace(["\r\n", "\r"], "\n", $source);
$this->position = 0;
$this->length = strlen($this->source);
$this->block_start_chars = $block_start_chars;
}
/**
* Whether the cursor has reached the end of the source.
* @return bool true if nothing remains to read
*/
public function atEnd()
{
return $this->position >= $this->length;
}
/**
* The current cursor offset, so a caller can remember a spot and
* later return to it.
* @return int the offset into the source
*/
public function tell()
{
return $this->position;
}
/**
* Moves the cursor back to a remembered offset.
* @param int $offset an offset previously returned by tell
* @return void
*/
public function seek($offset)
{
$this->position = $offset;
}
/**
* Looks at a character near the cursor without consuming it.
* @param int $ahead how many characters past the cursor to look
* @return string the character, or the empty string past the end
*/
public function peek($ahead = 0)
{
$at = $this->position + $ahead;
return $at < $this->length ? $this->source[$at] : "";
}
/**
* Moves the cursor forward.
* @param int $count how many characters to consume
* @return void
*/
public function advance($count = 1)
{
$this->position += $count;
}
/**
* Whether the source at the cursor begins with the given text.
* @param string $text the text to compare against
* @return bool true if the next characters match text exactly
*/
public function startsWith($text)
{
return substr_compare($this->source, $text, $this->position,
strlen($text)) === 0;
}
/**
* Whether the cursor sits at the start of a line, meaning it is at
* the very beginning of the source or just past a newline.
* @return bool true at a line start
*/
public function atLineStart()
{
return $this->position === 0 ||
$this->source[$this->position - 1] === "\n";
}
/**
* Returns the source from the cursor to the end without moving the
* cursor, so a caller can inspect all that remains to be read.
*
* @return string the unread remainder of the source
*/
public function rest()
{
return substr($this->source, $this->position);
}
/**
* Returns the whole source the scanner runs over, so a caller that
* works from an offset can be handed the text and the cursor rather
* than a fresh copy of all that remains. The cursor is not moved.
* @return string the full source
*/
public function source()
{
return $this->source;
}
/**
* Returns a whole line at or ahead of the cursor without moving it and
* without copying all that remains, so a caller can look at a line, or
* the one just below it, in time set by the length of that line rather
* than by the length of the rest of the source. This is what keeps a
* scan that peeks a line per step linear overall. The ahead count is in
* lines past the cursor: zero is the line the cursor sits on, one is the
* line below it.
*
* @param int $ahead how many lines past the cursor to return
* @return string the requested line without its newline, or the empty
* string when there is no such line
*/
public function peekLine($ahead = 0)
{
$start = $this->position;
for ($i = 0; $i < $ahead; $i++) {
$newline = strpos($this->source, "\n", $start);
if ($newline === false) {
return "";
}
$start = $newline + 1;
}
if ($start >= $this->length) {
return "";
}
$newline = strpos($this->source, "\n", $start);
$end = ($newline === false) ? $this->length : $newline;
return substr($this->source, $start, $end - $start);
}
/**
* Consumes characters until one of the stop characters (or a newline,
* or the end of the source) is reached, and returns what was consumed.
* The stop characters themselves are left for the caller to inspect.
*
* @param string $stops each character that ends the run, concatenated;
* a newline always ends the run as well
* @return string the characters read before the stop
*/
public function readUntil($stops)
{
$start = $this->position;
while ($this->position < $this->length) {
$current = $this->source[$this->position];
if ($current === "\n" || strpos($stops, $current) !== false) {
break;
}
$this->position++;
}
return substr($this->source, $start, $this->position - $start);
}
/**
* Consumes the rest of the current line and returns it without the
* trailing newline, leaving the cursor at the start of the next line.
* @return string the line's text
*/
public function readLine()
{
$start = $this->position;
while ($this->position < $this->length &&
$this->source[$this->position] !== "\n") {
$this->position++;
}
$line = substr($this->source, $start, $this->position - $start);
if ($this->position < $this->length) {
$this->position++;
}
return $line;
}
/**
* Consumes a run of the same character and reports how many there
* were, used for markers made of repeated characters such as a run of
* equals signs opening a heading or of colons setting an indent depth.
*
* @param string $character the single character to count
* @return int how many copies were consumed
*/
public function readRepeat($character)
{
$count = 0;
while ($this->position < $this->length &&
$this->source[$this->position] === $character) {
$this->position++;
$count++;
}
return $count;
}
/**
* Skips a run of blank lines, leaving the cursor at the first line
* that has content or at the end of the source.
* @return void
*/
public function skipBlankLines()
{
while ($this->position < $this->length) {
$mark = $this->position;
while ($this->position < $this->length &&
($this->source[$this->position] === " " ||
$this->source[$this->position] === "\t")) {
$this->position++;
}
if ($this->position < $this->length &&
$this->source[$this->position] === "\n") {
$this->position++;
} else if ($this->position < $this->length) {
$this->position = $mark;
break;
} else {
break;
}
}
}
/**
* Whether the character at the cursor is one that can begin a block
* for this markup flavor, the test a parser uses to decide whether a
* fresh line continues the current block or starts a new one.
* @return bool true if the cursor's character may open a block
*/
public function atBlockStartChar()
{
if ($this->position >= $this->length) {
return false;
}
return strpos($this->block_start_chars,
$this->source[$this->position]) !== false;
}
/**
* Whether the line at the cursor is blank, that is empty or only
* spaces and tabs before its newline, tested without consuming
* anything so a parser can decide a blank line ends the current block.
* @return bool true when the current line has no visible content
*/
public function atBlankLine()
{
$at = $this->position;
while ($at < $this->length && ($this->source[$at] === " " ||
$this->source[$at] === "\t")) {
$at++;
}
return $at >= $this->length || $this->source[$at] === "\n";
}
/**
* From a cursor sitting on an opening double brace, consumes through
* the matching close, counting nested double braces so an inner pair
* does not close the outer one, and returns the text between them.
* When there is no match the cursor is left where it began and false
* is returned, so the caller can treat the braces as ordinary text.
*
* @return string|false the text between the braces, or false when the
* opening brace is never matched
*/
public function matchBraces()
{
$start = $this->position;
$depth = 0;
while ($this->position < $this->length - 1) {
if ($this->source[$this->position] === "{" &&
$this->source[$this->position + 1] === "{") {
$depth++;
$this->position += 2;
} else if ($this->source[$this->position] === "}" &&
$this->source[$this->position + 1] === "}") {
$depth--;
$this->position += 2;
if ($depth === 0) {
return substr($this->source, $start + 2,
$this->position - $start - 4);
}
} else {
$this->position++;
}
}
$this->position = $start;
return false;
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software