17#ifndef REAL_COMPILER_HPP
18#define REAL_COMPILER_HPP
47 return ranges.size() == 1 && ranges[0].lo == 0x80U && ranges[0].hi == 0x10FFFFU;
69 std::vector<code_range> ranges {in.
ranges};
70 const auto add_partner {[&out, &ranges](std::uint32_t p) {
72 out.
ascii.
set(
static_cast<std::uint8_t
>(p));
75 ranges.push_back({.lo = p, .hi = p});
78 for (std::uint32_t cp = 0; cp < 0x80U; ++cp) {
79 if (in.
ascii.
test(
static_cast<std::uint8_t
>(cp))) {
83 for (std::uint8_t i = 0; i < entry.count; ++i) {
84 add_partner(entry.partner[i]);
91 if (std::ranges::any_of(in.
ranges,
92 [&entry](
const code_range& r) { return entry.cp >= r.lo && entry.cp <= r.hi; })) {
93 for (std::uint8_t k = 0; k < entry.count; ++k) {
94 add_partner(entry.partner[k]);
166 return static_cast<std::int32_t
>(prog.
code.size());
189 prog.
code.push_back(instruction);
199 return here(prog) - 1;
209 return here(prog) - 1;
222 prog.
code[
static_cast<std::size_t
>(pc)].primary_target = target;
235 prog.
code[
static_cast<std::size_t
>(pc)].secondary_target = target;
251 std::size_t index {prog.
classes.size()};
252 for (std::size_t i = 0; i < prog.
classes.size(); ++i) {
258 if (index == prog.
classes.size()) {
259 if (index > 0xFFFF) {
260 throw regex_error(
"too many character classes", 0);
283 for (std::size_t i = 0; i < prog.
cp_classes.size(); ++i) {
285 if (!(existing.ascii == cd.
ascii) || existing.range_count != cd.
ranges.size()) {
289 for (std::uint32_t k = 0; k < existing.range_count; ++k) {
291 if (a.lo != cd.
ranges[k].lo || a.hi != cd.
ranges[k].hi) {
306 if (index > 0xFFFF) {
307 throw regex_error(
"too many code-point classes", 0);
309 const auto begin {
static_cast<std::uint32_t
>(prog.
cp_ranges.size())};
314 .range_begin = begin,
315 .range_count =
static_cast<std::uint32_t
>(cd.
ranges.size())});
341 const std::int32_t block_start {
here(prog)};
373 const std::int32_t end {
here(prog)};
390 const std::vector<std::vector<char_class>>& branches)
const
392 std::vector<std::int32_t> jumps;
393 for (std::size_t b = 0; b + 1 < branches.size(); ++b) {
402 for (
const char_class& step : branches.back()) {
405 const std::int32_t end {
here(prog)};
406 for (
const std::int32_t
jump : jumps) {
418 const std::vector<code_range>& ranges)
const
420 std::vector<std::vector<char_class>> branches;
421 if (!
ascii.empty()) {
422 branches.push_back({
ascii});
426 std::vector<char_class> branch;
427 for (std::size_t i = 0; i < seq.length; ++i) {
429 step.
set_range(seq.parts[i].lo, seq.parts[i].hi);
430 branch.push_back(step);
432 branches.push_back(branch);
435 if (branches.empty()) {
477 folded.ascii.invert();
478 return {.ascii = folded.ascii, .ranges = {}};
480 folded.ascii.invert_ascii();
496 bool capture_free =
false)
const
515 if (
tree_.
classes[
static_cast<std::size_t
>(node.klass)].codepoint_predicate) {
540 const flags node_flags {
static_cast<flags>(node.effective_flags)};
552 head.
bits[0] &= ~newline.bits[0];
568 const flags node_flags {
static_cast<flags>(node.effective_flags)};
572 .arg8 =
static_cast<std::uint8_t
>(
assert_kind_for(node.anchor, node_flags)),
573 .arg16 = node_unicode != prog_unicode ? std::uint16_t {1} : std::uint16_t {0}});
577 for (std::int32_t child = node.child; child != -1;
578 child =
tree_.
nodes[
static_cast<std::size_t
>(child)].next) {
589 if (node.group >= 0 && !capture_free) {
590 emit(prog, {.op =
opcode::save, .arg16 =
static_cast<std::uint16_t
>(2 * node.group)});
591 emit_node(prog, node.child, capture_free);
592 emit(prog, {.op =
opcode::save, .arg16 =
static_cast<std::uint16_t
>((2 * node.group) + 1)});
596 emit_node(prog, node.child, capture_free);
617 flags node_flags)
const
670 bool capture_free)
const
672 std::vector<std::int32_t> jumps;
673 std::int32_t branch {node.
child};
674 while (branch != -1) {
675 const std::int32_t after {
tree_.
nodes[
static_cast<std::size_t
>(branch)].next};
688 const std::int32_t end {
here(prog)};
689 for (
const std::int32_t j : jumps) {
707 bool capture_free)
const
709 for (std::int32_t i = 0; i < node.
min; ++i) {
710 if (node.
max == -1 && i == node.
min - 1) {
713 const std::int32_t body {
here(prog)};
722 if (node.
max == -1) {
738 std::vector<std::int32_t> exits;
739 for (std::int32_t i = node.
min; i < node.
max; ++i) {
743 const std::int32_t end {
here(prog)};
744 for (
const std::int32_t s : exits) {
766 bool capture_free)
const
773 throw regex_error(
"nested lookaround is not supported", 0);
777 throw regex_error(
"unbounded lookaround is not supported (use a fixed repeat count)", 0);
780 throw regex_error(
"lookaround sub-pattern too long", 0);
782 const std::size_t sub_id {prog.
lookarounds.size()};
785 const std::int32_t skip {
emit_jump(prog)};
786 const std::int32_t sub_offset {
here(prog)};
790 prog.
lookarounds[sub_id] = {.code_offset = sub_offset,
791 .code_length =
here(prog) - sub_offset,
807 [[nodiscard]]
constexpr std::int32_t
l_max_bytes(std::int32_t index)
const
826 std::int32_t width {eff.
ascii.
empty() ? 0 : 1};
828 const std::int32_t w {r.hi < 0x800U ? 2 : (r.hi < 0x10000U ? 3 : 4)};
843 std::int32_t total {0};
844 for (std::int32_t child = node.child; child != -1;
845 child =
tree_.
nodes[
static_cast<std::size_t
>(child)].next) {
856 std::int32_t widest {0};
857 for (std::int32_t branch = node.child; branch != -1;
858 branch =
tree_.
nodes[
static_cast<std::size_t
>(branch)].next) {
871 if (node.max == -1) {
881 return node.max * body;
909 if (!std::is_constant_evaluated() && prog.hints.inner_literal_prefix >= 1) {
913 prog.prefix_classes = pp.classes;
914 prog.prefix_cp_classes = pp.cp_classes;
915 prog.prefix_cp_ranges = pp.cp_ranges;
Pattern text → AST, via a constexpr recursive-descent parser.
256-bit byte set with O(1) membership, fully constexpr.
Compiles an ast into a dynamic_program (NFA bytecode).
static constexpr std::int32_t emit_split(dynamic_program &prog)
Emits a split with placeholder targets.
static constexpr void emit(dynamic_program &prog, instr instruction)
Appends one instruction, enforcing the program-size cap.
constexpr void emit_alternation(dynamic_program &prog, const ast_node &node, bool capture_free) const
Emits an alternation: branches chained with leftmost-preferred splits.
constexpr assert_kind assert_kind_for(anchor_kind anchor, flags node_flags) const
Maps an AST anchor_kind to the runtime assert_kind.
constexpr void emit_node(dynamic_program &prog, std::int32_t index, bool capture_free=false) const
Emits the bytecode for the AST node at index (recursively).
static constexpr std::int32_t here(const dynamic_program &prog)
Returns the index of the next instruction.
constexpr void emit_byte_sequences(dynamic_program &prog, const std::vector< std::vector< char_class > > &branches) const
Emits an alternation of byte-range sequences (branches) as split/jump — the general form of emit_code...
static constexpr void emit_klass_cp(dynamic_program &prog, const class_def &cd)
Emits a match-time code-point predicate for a Unicode shorthand (\w \d \s and their negations) in tex...
static constexpr std::int32_t emit_jump(dynamic_program &prog)
Emits a jump with a placeholder target.
constexpr void emit_codepoint_class(dynamic_program &prog, const char_class &ascii) const
Emits "one codepoint matching \p ascii, or any non-ASCII codepoint".
static constexpr void patch_secondary(dynamic_program &prog, std::int32_t pc, std::int32_t target)
Sets the secondary branch target of the split at pc.
constexpr void emit_class_codepoints(dynamic_program &prog, const char_class &ascii, const std::vector< code_range > &ranges) const
Emits a code-point class: the ASCII bitmap (one byte, if any) OR the canonical UTF-8 byte sequences o...
const ast & tree_
The AST being compiled.
static constexpr void patch_primary(dynamic_program &prog, std::int32_t pc, std::int32_t target)
Sets the primary branch target of the instruction at pc.
static constexpr void emit_klass(dynamic_program &prog, const char_class &klass)
Emits a klass instruction, interning klass.
constexpr std::int32_t l_max_bytes(std::int32_t index) const
Upper bound, in bytes, on what the sub-AST at index can consume; -1 if unbounded (a *,...
constexpr void emit_lookaround(dynamic_program &prog, const ast_node &node, bool capture_free) const
Emits a bounded lookaround: an assert_lookaround whose sub-program is a capture-free region the main ...
constexpr class_def effective_class(const ast_node &node) const
The class a node_kind::klass node effectively accepts, after negation, icase folding and the bytes/co...
constexpr void emit_repeat(dynamic_program &prog, const ast_node &node, bool capture_free) const
Emits a quantifier (Thompson construction).
constexpr dynamic_program compile()
Emits the full NFA program for the bound AST.
flags flags_
Effective compilation flags.
constexpr compiler(const ast &tree, flags compile_flags)
Binds the compiler to a parsed pattern and its flags.
Resource limits guarding against pattern-driven resource exhaustion.
Extracts a required inner literal from a pattern's AST — the substring that every match must contain ...
constexpr std::vector< utf8_byte_seq > utf8_range_sequences(std::uint32_t lo, std::uint32_t hi)
Canonical UTF-8 byte-range sequences for the code-point range [lo, hi], excluding the surrogate block...
constexpr fold_entry unicode_fold_table[]
Fold orbits, sorted by fold_entry::cp for binary search.
constexpr bool is_any_non_ascii(const std::vector< code_range > &ranges)
Whether ranges is exactly the whole non-ASCII space [U+0080, U+10FFFF] — the "any non-ASCII code poin...
constexpr char_class utf8_lead3_set()
The lead-byte set of a 3-byte UTF-8 sequence.
constexpr std::size_t max_program_size
Maximum number of NFA instructions in a compiled program.
anchor_kind
The specific zero-width assertion of an anchor node (see node_kind::anchor).
@ word_start
< (start of word; REAL extension, not in Python re).
@ caret
^ (text or line start, depending on multiline).
@ word_end
> (end of word; REAL extension, not in Python re).
@ dollar
$ (end, before a trailing \n, or line end with m).
constexpr dynamic_program compile(const ast &tree, flags compile_flags)
Compiles tree to an NFA program (convenience over compiler).
constexpr std::size_t unicode_fold_table_size
Number of entries in unicode_fold_table.
constexpr inner_literal extract_inner_literal(const ast &tree)
Extract the best required inner literal from a pattern's AST (a pure function on the node pool)....
constexpr std::vector< code_range > complement_code_ranges(std::vector< code_range > ranges)
Complements a set of code-point ranges within [0x80, 0x10FFFF] (used by negated classes and by an in-...
constexpr class_def unicode_casefold(const class_def &in)
Expands a character class to its Unicode simple case-fold closure (text-mode icase).
constexpr std::vector< code_range > coalesce_ranges(std::vector< code_range > ranges)
Sorts ranges and merges overlapping / adjacent ones into a minimal, sorted set (the same set of code ...
constexpr void fold_ascii_case(char_class &klass)
Closes klass under ASCII case folding.
constexpr char_class utf8_lead2_set()
The lead-byte set of a 2-byte UTF-8 sequence.
constexpr char_class utf8_lead4_set()
The lead-byte set of a 4-byte UTF-8 sequence.
constexpr char_class utf8_cont_set()
The UTF-8 continuation-byte set 10xxxxxx.
constexpr std::int32_t max_lookaround_length
Maximum bytes a bounded lookaround sub-pattern may consume (its L_max). Bounds the per-position looka...
constexpr pattern_hints analyze_program(std::span< const instr > code, std::span< const char_class > classes, std::span< const cp_class > cp_classes, std::int32_t cp_mark_ascii, std::int32_t cp_mark_offset)
Walks a compiled program once to derive its search hints.
@ any
One codepoint, except newline (the . metacharacter).
@ repeat
Child repeated [min, max] times (max -1 = unbounded).
@ lookaround
Bounded lookaround: child = sub-pattern, negated = (?!/(?<!), direction = ahead/behind.
@ concat
Children matched in sequence.
@ alternation
Children are branches, leftmost preferred.
@ anchor
Zero-width assertion; kind in real::detail::ast_node::anchor.
@ klass
One codepoint constrained by classes[klass] (a class_def; negated or not).
@ empty
Matches the empty string.
@ group
Child wrapped in a group; group >= 0 when capturing.
assert_kind
Kind of zero-width assertion carried in assert_position's arg8.
@ word_start
< (non-word/start on the left, word on the right).
@ word_end
> (word on the left, non-word/end on the right).
@ line_start
^ with multiline.
@ line_end
$ with multiline.
@ word_boundary
\b (Unicode word-ness in text mode; ASCII in bytes / re.A).
@ text_start
\A, and ^ without multiline.
@ text_end_or_final_newline
$ without multiline (Python semantics).
ast build_prefix_ast(const ast &tree, std::int32_t count)
Build the prefix sub-AST: the first count top-level concat children (count >= 1). Copies the tree and...
@ klass_cp
Consume one code point tested against cp_classes[arg16] (decode + range bsearch); enters a 3-instr co...
@ byte
Consume one byte equal to arg8; fall through to pc+1.
@ save
Store current position in slot arg16; fall through (epsilon).
@ klass
Consume one byte in classes[arg16]; fall through to pc+1.
@ assert_lookaround
Epsilon; proceeds only if the lookaround sub-program arg16 holds here.
@ assert_position
Epsilon; proceeds only if assertion arg8 holds here.
@ split
Epsilon-branch to x (preferred) and y.
constexpr std::size_t find_fold_index(std::uint32_t cp)
Binary-searches unicode_fold_table for cp; returns its index, or unicode_fold_table_size if cp is not...
constexpr bool has_flag(flags value, flags flag)
Tests whether flag is set in value.
flags
Compilation flags, mirroring Python's re.I, re.M and re.S.
@ dotall
. also matches \n.
@ bytes
Binary mode: . and [^…] match raw bytes, not codepoints.
@ ascii
ASCII mode (re.A): \d \w \s \b stay ASCII and icase folds ASCII only, even in text mode....
@ icase
Case-insensitive (ASCII).
@ ecma
ECMAScript compatibility: $ (no multiline) matches only at the very end (not before a final \n,...
@ multiline
^ and $ also match at line boundaries.
@ dollar_endonly
$ (no multiline) matches only at the very end of the text, never before a final \n — the Rust/\z sema...
Search acceleration: pattern analysis and candidate-finding.
Compiled form of a pattern and the public flags / error types.
One AST node. Active fields depend on kind (noted per field).
bool lazy
repeat: prefer the shortest expansion.
std::int32_t max
repeat: maximum count (-1 = unbounded).
bool negated
klass: written as [^...] / \D \W \S.
std::int32_t klass
klass: index into ast::classes.
std::uint8_t effective_flags
Flag set in force where this node was parsed (see flags). Stamped from the scope stack; carried for s...
std::int32_t child
First child (concat, repeat, alternation, group).
std::int32_t min
repeat: minimum count.
look_dir direction
lookaround: ahead (?=/(?! or behind (?<=/(?<!.
A parsed pattern: the node pool plus side tables.
std::vector< ast_node > nodes
The node pool; root indexes it.
std::int32_t root
Index of the root node.
std::vector< named_group > names
Named capture groups.
std::vector< class_def > classes
Character classes as written, before negation.
std::int32_t group_count
Number of capturing groups.
A set of byte values (0–255) as a 256-bit bitmap.
constexpr void set_range(std::uint8_t low, std::uint8_t high)
Adds the inclusive byte range [low, high] to the set.
constexpr bool test(std::uint8_t byte) const
Tests membership of byte byte.
constexpr void set(std::uint8_t byte)
Adds byte byte to the set.
constexpr bool empty() const
Reports whether the set has no members.
std::array< std::uint64_t, 4 > bits
Bitmap; bit byte is byte byte's membership.
A parsed character class: its ASCII bitmap plus any non-ASCII code-point ranges. Bundling the two (ra...
char_class ascii
ASCII members as a bitmap (all 256 bytes in bytes mode); pre-negation.
std::vector< code_range > ranges
Non-ASCII code-point ranges (code-point mode only; empty otherwise).
An inclusive code-point range [lo, hi]. Shared by character classes (ast.hpp) and the generated Unico...
A match-time code-point class for the klass_cp opcode: an ASCII bitmap for code points < 0x80 plus a ...
Owning, heap-allocated program: the storage backing real::regex.
std::int32_t codepoint_mark_ascii
ASCII sub-class index of an emitted codepoint-class block (-1 = none).
std::vector< cp_class > cp_classes
Match-time code-point classes (for klass_cp).
pattern_hints hints
Search-acceleration hints.
std::vector< instr > prefix_code
IL: the inner-literal prefix sub-program (the part before the literal), for the reverse start-finder....
std::uint16_t slot_count
2 * (capture groups + 1).
std::vector< instr > code
The instruction stream (main program + lookaround sub-program regions).
std::int32_t codepoint_mark_offset
Where that block starts (program offset); the whole-pattern hint requires offset 1.
bool unicode_word
\b \B < > use Unicode word-ness (text mode).
std::vector< code_range > cp_ranges
Flat range buffer the cp_class slices index into.
bool byte_mode
flags::bytes mode.
std::vector< lookaround_sub > lookarounds
Bounded lookaround sub-programs (regions of code).
std::vector< char_class > classes
Interned character classes.
std::vector< named_group > names
Named capture groups.
A code point and the other members of its case-fold orbit (up to 3; orbits <= 4).
The best required inner literal of a pattern (the memmem candidate). len == 0 means the pattern decli...
One NFA instruction. Field meaning depends on op.
std::int32_t inner_literal_prefix
std::uint8_t inner_literal_len
std::array< std::uint8_t, 16 > inner_literal
A required inner literal every match must contain (the memmem candidate the inner-literal prefilter s...
A canonical UTF-8 byte-range sequence (1–4 steps) covering part of a code-point range.
Unicode simple case-folding orbits for text-mode flags::icase.
Code-point range → canonical UTF-8 byte-range sequences (RE2 / rust regex-syntax Utf8Sequences).
REAL's version macros and the C++20 language-standard guard.