REAL
Regular Expression Algorithmic Library — constexpr C++20 regex
Loading...
Searching...
No Matches
compiler.hpp
Go to the documentation of this file.
1
17#ifndef REAL_COMPILER_HPP
18#define REAL_COMPILER_HPP
19
20// Internal — do not include directly.
21// Users: #include <real/real.hpp> (or the documented opt-ins <real/dfa.hpp>, <real/std/regex.hpp>).
22
23#include "real/version.hpp"
24
25#include <algorithm>
26#include <cstdint>
27#include <vector>
28
29#include "real/frontend/ast.hpp"
31#include "real/core/config.hpp"
34#include "real/core/program.hpp"
37
38namespace real::detail {
39
40 // The code-point-range → UTF-8 byte-range algorithm (utf8_byte_seq, utf8_range_sequences,
41 // encode_utf8_bytes) now lives in utf8_ranges.hpp, shared with the lazy DFA's klass_cp expansion.
42
45 constexpr bool is_any_non_ascii(const std::vector<code_range>& ranges)
46 {
47 return ranges.size() == 1 && ranges[0].lo == 0x80U && ranges[0].hi == 0x10FFFFU;
48 }
49
66 {
67 class_def out;
68 out.ascii = in.ascii;
69 std::vector<code_range> ranges {in.ranges}; // seed with the input's non-ASCII ranges
70 const auto add_partner {[&out, &ranges](std::uint32_t p) {
71 if (p < 0x80U) {
72 out.ascii.set(static_cast<std::uint8_t>(p)); // ASCII partner -> bitmap
73 }
74 else {
75 ranges.push_back({.lo = p, .hi = p}); // non-ASCII partner (coalesced below)
76 }
77 }};
78 for (std::uint32_t cp = 0; cp < 0x80U; ++cp) {
79 if (in.ascii.test(static_cast<std::uint8_t>(cp))) {
80 const std::size_t idx {find_fold_index(cp)};
81 if (idx != unicode_fold_table_size) {
82 const fold_entry& entry {unicode_fold_table[idx]};
83 for (std::uint8_t i = 0; i < entry.count; ++i) {
84 add_partner(entry.partner[i]);
85 }
86 }
87 }
88 }
89 for (std::size_t i = 0; i < unicode_fold_table_size; ++i) {
90 const fold_entry& entry {unicode_fold_table[i]};
91 if (std::ranges::any_of(in.ranges,
92 [&entry](const code_range& r) { return entry.cp >= r.lo && entry.cp <= r.hi; })) {
93 for (std::uint8_t k = 0; k < entry.count; ++k) {
94 add_partner(entry.partner[k]);
95 }
96 }
97 }
98 // Coalesce: the fold adds many degenerate {cp, cp} ranges (a class's own members' partners, and
99 // partners of members just outside the class); merging overlapping/adjacent ranges collapses that
100 // fragmentation without changing the accepted set — pure size optimisation.
101 out.ranges = coalesce_ranges(std::move(ranges));
102 return out;
103 }
104
109 {
110 public:
111
117 constexpr compiler(const ast& tree,
118 flags compile_flags)
119 : tree_(tree),
120 flags_(compile_flags)
121 {}
122
129 {
130 dynamic_program prog;
131 prog.slot_count = static_cast<std::uint16_t>(2 * (tree_.group_count + 1));
132 prog.names = tree_.names;
133 emit(prog, {.op = opcode::save, .arg16 = 0});
134 emit_node(prog, tree_.root);
135 emit(prog, {.op = opcode::save, .arg16 = 1});
136 emit(prog, {.op = opcode::match});
140 // The required inner literal + its prefix boundary (a single AST walk). Recorded for the inner-literal
141 // search path; not yet routed on. Kept off the program code, so byte-identity is untouched.
143 prog.hints.inner_literal = il.bytes;
144 prog.hints.inner_literal_len = il.len;
145 prog.hints.inner_literal_prefix = il.prefix_child_count;
146 if (prog.code.size() > max_program_size) {
147 throw regex_error("program too large", 0);
148 }
149 return prog;
150 }
151
152 private:
153
154 const ast& tree_;
156
157 // --- low-level emission helpers -------------------------------------
158
164 static constexpr std::int32_t here(const dynamic_program& prog)
165 {
166 return static_cast<std::int32_t>(prog.code.size());
167 }
168
183 static constexpr void emit(dynamic_program& prog,
184 instr instruction)
185 {
186 if (prog.code.size() >= max_program_size) {
187 throw regex_error("program too large", 0);
188 }
189 prog.code.push_back(instruction);
190 }
191
196 static constexpr std::int32_t emit_split(dynamic_program& prog)
197 {
198 emit(prog, {.op = opcode::split, .primary_target = -1, .secondary_target = -1});
199 return here(prog) - 1;
200 }
201
206 static constexpr std::int32_t emit_jump(dynamic_program& prog)
207 {
208 emit(prog, {.op = opcode::jump, .primary_target = -1});
209 return here(prog) - 1;
210 }
211
218 static constexpr void patch_primary(dynamic_program& prog,
219 std::int32_t pc,
220 std::int32_t target)
221 {
222 prog.code[static_cast<std::size_t>(pc)].primary_target = target;
223 }
224
231 static constexpr void patch_secondary(dynamic_program& prog,
232 std::int32_t pc,
233 std::int32_t target)
234 {
235 prog.code[static_cast<std::size_t>(pc)].secondary_target = target;
236 }
237
248 static constexpr void emit_klass(dynamic_program& prog,
249 const char_class& klass)
250 {
251 std::size_t index {prog.classes.size()};
252 for (std::size_t i = 0; i < prog.classes.size(); ++i) {
253 if (prog.classes[i] == klass) {
254 index = i;
255 break;
256 }
257 }
258 if (index == prog.classes.size()) {
259 if (index > 0xFFFF) {
260 throw regex_error("too many character classes", 0);
261 }
262 prog.classes.push_back(klass);
263 }
264 emit(prog, {.op = opcode::klass, .arg16 = static_cast<std::uint16_t>(index)});
265 }
266
279 static constexpr void emit_klass_cp(dynamic_program& prog,
280 const class_def& cd)
281 {
282 std::size_t index {prog.cp_classes.size()};
283 for (std::size_t i = 0; i < prog.cp_classes.size(); ++i) {
284 const cp_class& existing {prog.cp_classes[i]};
285 if (!(existing.ascii == cd.ascii) || existing.range_count != cd.ranges.size()) {
286 continue;
287 }
288 bool same {true};
289 for (std::uint32_t k = 0; k < existing.range_count; ++k) {
290 const code_range& a {prog.cp_ranges[existing.range_begin + k]};
291 if (a.lo != cd.ranges[k].lo || a.hi != cd.ranges[k].hi) {
292 // Two code-point classes with the SAME ASCII bitmap and range COUNT but different ranges:
293 // the interner must not merge them. In practice the shorthand classes (\w/\d/\s and their
294 // complements) have distinct bitmaps and counts, so this range mismatch is a defensive
295 // arm of the dedup, not hit by the current emitters (hence uncovered by the runtime report).
296 same = false;
297 break;
298 }
299 }
300 if (same) {
301 index = i;
302 break;
303 }
304 }
305 if (index == prog.cp_classes.size()) {
306 if (index > 0xFFFF) {
307 throw regex_error("too many code-point classes", 0);
308 }
309 const auto begin {static_cast<std::uint32_t>(prog.cp_ranges.size())};
310 for (const code_range& r : cd.ranges) {
311 prog.cp_ranges.push_back(r);
312 }
313 prog.cp_classes.push_back({.ascii = cd.ascii,
314 .range_begin = begin,
315 .range_count = static_cast<std::uint32_t>(cd.ranges.size())});
316 }
317 emit(prog, {.op = opcode::klass_cp, .arg16 = static_cast<std::uint16_t>(index)});
318 emit_klass(prog, utf8_cont_set()); // three continuation slots; klass_cp's skip picks the entry
319 emit_klass(prog, utf8_cont_set());
320 emit_klass(prog, utf8_cont_set());
321 }
322
323 // --- UTF-8 byte expansion --------------------------------------------
324
339 const char_class& ascii) const
340 {
341 const std::int32_t block_start {here(prog)}; // start offset, recorded as the marker below
342 const char_class cont {utf8_cont_set()};
343 const char_class lead2 {utf8_lead2_set()};
344 const char_class lead3 {utf8_lead3_set()};
345 const char_class lead4 {utf8_lead4_set()};
346
347 const std::int32_t s1 {emit_split(prog)};
348 patch_primary(prog, s1, here(prog));
349 emit_klass(prog, ascii);
350 const std::int32_t j1 {emit_jump(prog)};
351
352 patch_secondary(prog, s1, here(prog));
353 const std::int32_t s2 {emit_split(prog)};
354 patch_primary(prog, s2, here(prog));
355 emit_klass(prog, lead2);
356 emit_klass(prog, cont);
357 const std::int32_t j2 {emit_jump(prog)};
358
359 patch_secondary(prog, s2, here(prog));
360 const std::int32_t s3 {emit_split(prog)};
361 patch_primary(prog, s3, here(prog));
362 emit_klass(prog, lead3);
363 emit_klass(prog, cont);
364 emit_klass(prog, cont);
365 const std::int32_t j3 {emit_jump(prog)};
366
367 patch_secondary(prog, s3, here(prog));
368 emit_klass(prog, lead4);
369 emit_klass(prog, cont);
370 emit_klass(prog, cont);
371 emit_klass(prog, cont);
372
373 const std::int32_t end {here(prog)};
374 patch_primary(prog, j1, end);
375 patch_primary(prog, j2, end);
376 patch_primary(prog, j3, end);
377
378 // Record the marker (offset + ASCII sub-class index) so analyze_program reads
379 // it instead of reverse-engineering this 16-instruction block's bytecode shape.
380 prog.codepoint_mark_offset = block_start;
381 prog.codepoint_mark_ascii = static_cast<std::int32_t>(prog.code[static_cast<std::size_t>(block_start) + 1].arg16);
382 }
383
390 const std::vector<std::vector<char_class>>& branches) const
391 {
392 std::vector<std::int32_t> jumps;
393 for (std::size_t b = 0; b + 1 < branches.size(); ++b) {
394 const std::int32_t split {emit_split(prog)};
395 patch_primary(prog, split, here(prog));
396 for (const char_class& step : branches[b]) {
397 emit_klass(prog, step);
398 }
399 jumps.push_back(emit_jump(prog));
400 patch_secondary(prog, split, here(prog));
401 }
402 for (const char_class& step : branches.back()) {
403 emit_klass(prog, step);
404 }
405 const std::int32_t end {here(prog)};
406 for (const std::int32_t jump : jumps) {
407 patch_primary(prog, jump, end);
408 }
409 }
410
417 const char_class& ascii,
418 const std::vector<code_range>& ranges) const
419 {
420 std::vector<std::vector<char_class>> branches;
421 if (!ascii.empty()) {
422 branches.push_back({ascii});
423 }
424 for (const code_range& range : ranges) {
425 for (const utf8_byte_seq& seq : utf8_range_sequences(range.lo, range.hi)) {
426 std::vector<char_class> branch;
427 for (std::size_t i = 0; i < seq.length; ++i) {
428 char_class step;
429 step.set_range(seq.parts[i].lo, seq.parts[i].hi);
430 branch.push_back(step);
431 }
432 branches.push_back(branch);
433 }
434 }
435 if (branches.empty()) {
436 // An impossible class (e.g. the negation of the whole code-point space): match nothing. An
437 // empty bitmap rejects every byte, so the thread dies — a never-match, not a crash.
438 emit_klass(prog, char_class {});
439 return;
440 }
441 emit_byte_sequences(prog, branches);
442 }
443
451 [[nodiscard]] constexpr class_def effective_class(const ast_node& node) const
452 {
453 // icase and ascii are read from the node's own scope (stamped by the parser from the flag-scope
454 // stack), so a scoped (?i:...) / (?a:...) folds and picks tables per scope. bytes is not scopable
455 // and stays global.
456 const flags node_flags {static_cast<flags>(node.effective_flags)};
457 class_def folded {tree_.classes[static_cast<std::size_t>(node.klass)]};
458 if (has_flag(node_flags, flags::icase)) {
459 if (has_flag(flags_, flags::bytes) || has_flag(node_flags, flags::ascii)) {
460 fold_ascii_case(folded.ascii); // bytes / ASCII mode (re.A): ASCII-only fold, no Unicode partners
461 }
462 else {
463 folded = unicode_casefold(folded); // text: full Unicode fold of the whole class, both directions
464 }
465 }
466 // The fold is applied BEFORE negation (Python order): [^k] under icase is the complement of
467 // {k, K, Kelvin}, so it rejects Kelvin.
468 if (!node.negated) {
469 // Members accumulate in PARSE order — a predicate's range table (`\d`/`\w`/`\p{…}`), then any literal
470 // or range that follows it — but `klass_cp` binary-searches the ranges at match time, so they must be
471 // sorted and merged. Without this, a non-ASCII member after a predicate (`[\dЩ]`, `[\w\W]`) landed out
472 // of order and was silently missed. The negated path already coalesces via complement_code_ranges.
473 folded.ranges = coalesce_ranges(std::move(folded.ranges));
474 return folded;
475 }
477 folded.ascii.invert(); // raw bytes: plain 256-bit complement, no code-point ranges
478 return {.ascii = folded.ascii, .ranges = {}};
479 }
480 folded.ascii.invert_ascii();
481 return {.ascii = folded.ascii, .ranges = complement_code_ranges(folded.ranges)};
482 }
483
484 // --- node emission ----------------------------------------------------
485
494 constexpr void emit_node(dynamic_program& prog,
495 std::int32_t index,
496 bool capture_free = false) const
497 {
498 const ast_node& node {tree_.nodes[static_cast<std::size_t>(index)]};
499 switch (node.kind) {
500 case node_kind::empty:
501 break;
502 case node_kind::byte:
503 // A `byte` node is a raw byte with byte provenance (a `\xHH` / octal escape, or a non-cased
504 // literal), never case-folded: under icase a cased literal was promoted to a foldable
505 // singleton class at the parser, so it never reaches here. This preserves the deliberate
506 // `\xHH` provenance split (see emit_literal_codepoint / divergences.dox).
507 emit(prog, {.op = opcode::byte, .arg8 = node.byte});
508 break;
509 case node_kind::klass:
510 {
511 // A text-mode class with a Unicode-shorthand contribution (\w/\d/\s, bare or in a class):
512 // a match-time code-point predicate, not the byte-NFA. effective_class materialises the
513 // fold and the external negation, so the stored cp_class needs no negation flag -- this is
514 // also what gives [^\W] == \w, [^\D] == \d, [^\S] == \s.
515 if (tree_.classes[static_cast<std::size_t>(node.klass)].codepoint_predicate) {
516 emit_klass_cp(prog, effective_class(node));
517 break;
518 }
519 const class_def eff {effective_class(node)};
520 if (has_flag(flags_, flags::bytes) || eff.ranges.empty()) {
521 // Bytes mode is a single 256-bit bitmap; a code-point class with no non-ASCII members is
522 // just its ASCII bitmap. An empty bitmap here (impossible class) is a never-match.
523 emit_klass(prog, eff.ascii);
524 break;
525 }
526 if (is_any_non_ascii(eff.ranges)) {
527 // "ASCII bitmap OR any non-ASCII code point" (`.`-family, `[^x]`): the compact
528 // emit_codepoint_class shape the prefilter/DFA fast path recognizes.
529 emit_codepoint_class(prog, eff.ascii);
530 break;
531 }
532 emit_class_codepoints(prog, eff.ascii, eff.ranges);
533 break;
534 }
535 case node_kind::any:
536 {
537 // dotall is read from this node's own scope (a scoped (?s:.) matches \n inside the island
538 // only); bytes and ecma are not scopable and stay global. A non-scoped node carries the
539 // global dotall, so its emitted class is byte-identical to before.
540 const flags node_flags {static_cast<flags>(node.effective_flags)};
541 char_class head;
542 head.set_range(0x00, 0x7F);
544 head.set_range(0x80, 0xFF); // any raw byte
545 }
546 if (!has_flag(node_flags, flags::dotall)) {
547 char_class newline;
548 newline.set('\n');
550 newline.set('\r'); // ECMAScript `.` excludes \n AND \r (byte-level; U+2028/2029 are multi-byte)
551 }
552 head.bits[0] &= ~newline.bits[0];
553 }
555 emit_klass(prog, head);
556 }
557 else {
558 emit_codepoint_class(prog, head);
559 }
560 break;
561 }
563 {
564 // The assert_kind for ^/$ follows this node's own multiline (a scoped (?m:...)); \b \B < >
565 // additionally carry a per-instruction FLIP bit: 1 when this node's word-ness differs from
566 // the program default (a scoped (?a:...) / (?-a:...) island), else 0. A non-scoped pattern
567 // is all-0 here and maps to the same assert_kinds, so its program is byte-identical.
568 const flags node_flags {static_cast<flags>(node.effective_flags)};
569 const bool prog_unicode {!has_flag(flags_, flags::bytes) && !has_flag(flags_, flags::ascii)};
570 const bool node_unicode {!has_flag(flags_, flags::bytes) && !has_flag(node_flags, flags::ascii)};
571 emit(prog, {.op = opcode::assert_position,
572 .arg8 = static_cast<std::uint8_t>(assert_kind_for(node.anchor, node_flags)),
573 .arg16 = node_unicode != prog_unicode ? std::uint16_t {1} : std::uint16_t {0}});
574 }
575 break;
577 for (std::int32_t child = node.child; child != -1;
578 child = tree_.nodes[static_cast<std::size_t>(child)].next) {
579 emit_node(prog, child, capture_free);
580 }
581 break;
583 emit_repeat(prog, node, capture_free);
584 break;
586 emit_alternation(prog, node, capture_free);
587 break;
588 case node_kind::group:
589 if (node.group >= 0 && !capture_free) {
590 emit(prog, {.op = opcode::save, .arg16 = static_cast<std::uint16_t>(2 * node.group)});
591 emit_node(prog, node.child, capture_free);
592 emit(prog, {.op = opcode::save, .arg16 = static_cast<std::uint16_t>((2 * node.group) + 1)});
593 }
594 else {
595 // capture-free (a lookaround sub-pattern) or non-capturing group.
596 emit_node(prog, node.child, capture_free);
597 }
598 break;
600 emit_lookaround(prog, node, capture_free);
601 break;
602 }
603 }
604
617 flags node_flags) const
618 {
619 // multiline is read from the anchor node's own scope (a scoped (?m:^...$) is line-relative inside
620 // the island only); ecma is not scopable and stays global. A non-scoped node carries the global
621 // multiline, so `^`/`$` map to the same assert_kind as before — byte-identical.
622 const bool multiline {has_flag(node_flags, flags::multiline)};
623 assert_kind result {};
624 switch (anchor) {
627 break;
629 // Default (Python): `$` matches at end OR just before a final `\n`. With the ecma OR dollar_endonly
630 // flag, `$` (no multiline) matches only at the very end — ECMAScript / Rust (`\z`) semantics.
631 result = multiline
636 break;
639 break;
641 result = assert_kind::text_end;
642 break;
645 break;
648 break;
651 break;
653 result = assert_kind::word_end;
654 break;
655 }
656 return result;
657 }
658
668 constexpr void emit_alternation(dynamic_program& prog,
669 const ast_node& node,
670 bool capture_free) const
671 {
672 std::vector<std::int32_t> jumps;
673 std::int32_t branch {node.child};
674 while (branch != -1) {
675 const std::int32_t after {tree_.nodes[static_cast<std::size_t>(branch)].next};
676 if (after != -1) {
677 const std::int32_t s {emit_split(prog)};
678 patch_primary(prog, s, here(prog));
679 emit_node(prog, branch, capture_free);
680 jumps.push_back(emit_jump(prog));
681 patch_secondary(prog, s, here(prog));
682 }
683 else {
684 emit_node(prog, branch, capture_free); // last branch: falls through
685 }
686 branch = after;
687 }
688 const std::int32_t end {here(prog)};
689 for (const std::int32_t j : jumps) {
690 patch_primary(prog, j, end);
691 }
692 }
693
705 constexpr void emit_repeat(dynamic_program& prog,
706 const ast_node& node,
707 bool capture_free) const
708 {
709 for (std::int32_t i = 0; i < node.min; ++i) {
710 if (node.max == -1 && i == node.min - 1) {
711 // Last mandatory copy doubles as the loop body: e+ patterns
712 // emit the body exactly once.
713 const std::int32_t body {here(prog)};
714 emit_node(prog, node.child, capture_free);
715 const std::int32_t s {emit_split(prog)};
716 patch_primary(prog, s, node.lazy ? here(prog) : body);
717 patch_secondary(prog, s, node.lazy ? body : here(prog));
718 return;
719 }
720 emit_node(prog, node.child, capture_free);
721 }
722 if (node.max == -1) { // min == 0: a star loop
723 const std::int32_t s {emit_split(prog)};
724 patch_primary(prog, s, node.lazy ? -1 : here(prog)); // body side set below
725 emit_node(prog, node.child, capture_free);
726 const std::int32_t j {emit_jump(prog)};
727 patch_primary(prog, j, s);
728 if (node.lazy) {
729 patch_primary(prog, s, here(prog));
730 patch_secondary(prog, s, s + 1);
731 }
732 else {
733 patch_secondary(prog, s, here(prog));
734 }
735 return;
736 }
737 // Optional copies: each split can bail out to the common exit.
738 std::vector<std::int32_t> exits;
739 for (std::int32_t i = node.min; i < node.max; ++i) {
740 exits.push_back(emit_split(prog));
741 emit_node(prog, node.child, capture_free);
742 }
743 const std::int32_t end {here(prog)};
744 for (const std::int32_t s : exits) {
745 patch_primary(prog, s, node.lazy ? end : s + 1);
746 patch_secondary(prog, s, node.lazy ? s + 1 : end);
747 }
748 }
749
764 constexpr void emit_lookaround(dynamic_program& prog,
765 const ast_node& node,
766 bool capture_free) const
767 {
768 if (capture_free) {
769 // intentionally uncovered: fail-loud net for a parser-guaranteed invariant. The
770 // parser rejects nested lookarounds first, so capture_free is never true here; a
771 // nested lookaround reaching the compiler would break the linear-time guarantee,
772 // so a throw beats a silent miscompile if that parser guard ever regresses.
773 throw regex_error("nested lookaround is not supported", 0);
774 }
775 const std::int32_t lmax {l_max_bytes(node.child)};
776 if (lmax < 0) {
777 throw regex_error("unbounded lookaround is not supported (use a fixed repeat count)", 0);
778 }
779 if (lmax > max_lookaround_length) {
780 throw regex_error("lookaround sub-pattern too long", 0);
781 }
782 const std::size_t sub_id {prog.lookarounds.size()};
783 prog.lookarounds.push_back({}); // placeholder, filled once the region is emitted
784 emit(prog, {.op = opcode::assert_lookaround, .arg16 = static_cast<std::uint16_t>(sub_id)});
785 const std::int32_t skip {emit_jump(prog)}; // main flow jumps over the sub-region
786 const std::int32_t sub_offset {here(prog)};
787 emit_node(prog, node.child, /*capture_free=*/ true);
788 emit(prog, {.op = opcode::match}); // sub-program terminator
789 patch_primary(prog, skip, here(prog));
790 prog.lookarounds[sub_id] = {.code_offset = sub_offset,
791 .code_length = here(prog) - sub_offset,
792 .l_max = lmax,
793 .direction = node.direction,
794 .negative = node.negated};
795 }
796
807 [[nodiscard]] constexpr std::int32_t l_max_bytes(std::int32_t index) const
808 {
809 const ast_node& node {tree_.nodes[static_cast<std::size_t>(index)]};
810 switch (node.kind) {
811 case node_kind::empty:
813 return 0;
814 case node_kind::byte:
815 return 1;
816 case node_kind::klass:
817 {
818 // Widest UTF-8 encoding the class can match, from the SAME effective (post-negation) class
819 // that emit_node compiles — so width and emission never disagree. Bytes mode: one byte.
820 // Otherwise 1 for any ASCII member, plus the widest code-point range (2/3/4 by top code
821 // point); an impossible class matches nothing, reported as 1 (harmless — it never matches).
823 return 1;
824 }
825 const class_def eff {effective_class(node)};
826 std::int32_t width {eff.ascii.empty() ? 0 : 1};
827 for (const code_range& r : eff.ranges) {
828 const std::int32_t w {r.hi < 0x800U ? 2 : (r.hi < 0x10000U ? 3 : 4)};
829 if (w > width) {
830 width = w;
831 }
832 }
833 // An impossible (never-match) class contributes 0: it consumes nothing, so a dead branch
834 // in a bounded lookaround (the negation of the whole code-point space, repeated) does not
835 // inflate the width -- the alternation `a | <impossible>{300}` stays width 1.
836 // Its emitted never-match still makes the branch fail — a width of 0 is not an empty match.
837 return width;
838 }
839 case node_kind::any:
840 return has_flag(flags_, flags::bytes) ? 1 : 4;
842 {
843 std::int32_t total {0};
844 for (std::int32_t child = node.child; child != -1;
845 child = tree_.nodes[static_cast<std::size_t>(child)].next) {
846 const std::int32_t c {l_max_bytes(child)};
847 if (c < 0) {
848 return -1;
849 }
850 total += c;
851 }
852 return total;
853 }
855 {
856 std::int32_t widest {0};
857 for (std::int32_t branch = node.child; branch != -1;
858 branch = tree_.nodes[static_cast<std::size_t>(branch)].next) {
859 const std::int32_t c {l_max_bytes(branch)};
860 if (c < 0) {
861 return -1;
862 }
863 if (c > widest) {
864 widest = c;
865 }
866 }
867 return widest;
868 }
870 {
871 if (node.max == -1) {
872 return -1; // *, +, {n,} are not statically bounded
873 }
874 const std::int32_t body {l_max_bytes(node.child)};
875 if (body < 0) {
876 return -1;
877 }
878 if (body > 0 && node.max > max_lookaround_length / body) {
879 return -1; // the product would exceed the cap (nested {n}{m}...) -> reject; no int32 overflow
880 }
881 return node.max * body;
882 }
883 case node_kind::group:
884 return l_max_bytes(node.child);
886 // intentionally uncovered: -Wswitch exhaustiveness arm; the parser rejects nested
887 // lookarounds first, so l_max_bytes never recurses into one. Treated as unbounded.
888 return -1;
889 }
890 return -1;
891 }
892 };
893
901 constexpr dynamic_program compile(const ast& tree,
902 flags compile_flags)
903 {
904 dynamic_program prog {compiler(tree, compile_flags).compile()};
905 // IL: compile the inner-literal prefix sub-program (the part before the literal) for the reverse
906 // start-finder. Dynamic-only — a static_regex compiles in a constant-evaluated context and keeps the core
907 // search, sidestepping the constexpr budget of a second compile (the "dynamic-only if it would blow the
908 // budget" choice, taken up front). The prefix program is a subset, so this does not recurse into itself.
909 if (!std::is_constant_evaluated() && prog.hints.inner_literal_prefix >= 1) {
910 const dynamic_program pp {
911 compiler(build_prefix_ast(tree, prog.hints.inner_literal_prefix), compile_flags).compile()};
912 prog.prefix_code = pp.code;
913 prog.prefix_classes = pp.classes;
914 prog.prefix_cp_classes = pp.cp_classes;
915 prog.prefix_cp_ranges = pp.cp_ranges;
916 }
917 return prog;
918 }
919} // namespace real::detail
920
921#endif // REAL_COMPILER_HPP
Pattern text → AST, via a constexpr recursive-descent parser.
256-bit byte set with O(1) membership, fully constexpr.
Compiles an ast into a dynamic_program (NFA bytecode).
Definition compiler.hpp:109
static constexpr std::int32_t emit_split(dynamic_program &prog)
Emits a split with placeholder targets.
Definition compiler.hpp:196
static constexpr void emit(dynamic_program &prog, instr instruction)
Appends one instruction, enforcing the program-size cap.
Definition compiler.hpp:183
constexpr void emit_alternation(dynamic_program &prog, const ast_node &node, bool capture_free) const
Emits an alternation: branches chained with leftmost-preferred splits.
Definition compiler.hpp:668
constexpr assert_kind assert_kind_for(anchor_kind anchor, flags node_flags) const
Maps an AST anchor_kind to the runtime assert_kind.
Definition compiler.hpp:616
constexpr void emit_node(dynamic_program &prog, std::int32_t index, bool capture_free=false) const
Emits the bytecode for the AST node at index (recursively).
Definition compiler.hpp:494
static constexpr std::int32_t here(const dynamic_program &prog)
Returns the index of the next instruction.
Definition compiler.hpp:164
constexpr void emit_byte_sequences(dynamic_program &prog, const std::vector< std::vector< char_class > > &branches) const
Emits an alternation of byte-range sequences (branches) as split/jump — the general form of emit_code...
Definition compiler.hpp:389
static constexpr void emit_klass_cp(dynamic_program &prog, const class_def &cd)
Emits a match-time code-point predicate for a Unicode shorthand (\w \d \s and their negations) in tex...
Definition compiler.hpp:279
static constexpr std::int32_t emit_jump(dynamic_program &prog)
Emits a jump with a placeholder target.
Definition compiler.hpp:206
constexpr void emit_codepoint_class(dynamic_program &prog, const char_class &ascii) const
Emits "one codepoint matching \p ascii, or any non-ASCII codepoint".
Definition compiler.hpp:338
static constexpr void patch_secondary(dynamic_program &prog, std::int32_t pc, std::int32_t target)
Sets the secondary branch target of the split at pc.
Definition compiler.hpp:231
constexpr void emit_class_codepoints(dynamic_program &prog, const char_class &ascii, const std::vector< code_range > &ranges) const
Emits a code-point class: the ASCII bitmap (one byte, if any) OR the canonical UTF-8 byte sequences o...
Definition compiler.hpp:416
const ast & tree_
The AST being compiled.
Definition compiler.hpp:154
static constexpr void patch_primary(dynamic_program &prog, std::int32_t pc, std::int32_t target)
Sets the primary branch target of the instruction at pc.
Definition compiler.hpp:218
static constexpr void emit_klass(dynamic_program &prog, const char_class &klass)
Emits a klass instruction, interning klass.
Definition compiler.hpp:248
constexpr std::int32_t l_max_bytes(std::int32_t index) const
Upper bound, in bytes, on what the sub-AST at index can consume; -1 if unbounded (a *,...
Definition compiler.hpp:807
constexpr void emit_lookaround(dynamic_program &prog, const ast_node &node, bool capture_free) const
Emits a bounded lookaround: an assert_lookaround whose sub-program is a capture-free region the main ...
Definition compiler.hpp:764
constexpr class_def effective_class(const ast_node &node) const
The class a node_kind::klass node effectively accepts, after negation, icase folding and the bytes/co...
Definition compiler.hpp:451
constexpr void emit_repeat(dynamic_program &prog, const ast_node &node, bool capture_free) const
Emits a quantifier (Thompson construction).
Definition compiler.hpp:705
constexpr dynamic_program compile()
Emits the full NFA program for the bound AST.
Definition compiler.hpp:128
flags flags_
Effective compilation flags.
Definition compiler.hpp:155
constexpr compiler(const ast &tree, flags compile_flags)
Binds the compiler to a parsed pattern and its flags.
Definition compiler.hpp:117
Resource limits guarding against pattern-driven resource exhaustion.
Extracts a required inner literal from a pattern's AST — the substring that every match must contain ...
constexpr std::vector< utf8_byte_seq > utf8_range_sequences(std::uint32_t lo, std::uint32_t hi)
Canonical UTF-8 byte-range sequences for the code-point range [lo, hi], excluding the surrogate block...
constexpr fold_entry unicode_fold_table[]
Fold orbits, sorted by fold_entry::cp for binary search.
constexpr bool is_any_non_ascii(const std::vector< code_range > &ranges)
Whether ranges is exactly the whole non-ASCII space [U+0080, U+10FFFF] — the "any non-ASCII code poin...
Definition compiler.hpp:45
constexpr char_class utf8_lead3_set()
The lead-byte set of a 3-byte UTF-8 sequence.
constexpr std::size_t max_program_size
Maximum number of NFA instructions in a compiled program.
Definition config.hpp:29
anchor_kind
The specific zero-width assertion of an anchor node (see node_kind::anchor).
Definition ast.hpp:64
@ word_start
< (start of word; REAL extension, not in Python re).
@ caret
^ (text or line start, depending on multiline).
@ word_end
> (end of word; REAL extension, not in Python re).
@ dollar
$ (end, before a trailing \n, or line end with m).
constexpr dynamic_program compile(const ast &tree, flags compile_flags)
Compiles tree to an NFA program (convenience over compiler).
Definition compiler.hpp:901
constexpr std::size_t unicode_fold_table_size
Number of entries in unicode_fold_table.
constexpr inner_literal extract_inner_literal(const ast &tree)
Extract the best required inner literal from a pattern's AST (a pure function on the node pool)....
constexpr std::vector< code_range > complement_code_ranges(std::vector< code_range > ranges)
Complements a set of code-point ranges within [0x80, 0x10FFFF] (used by negated classes and by an in-...
Definition ast.hpp:139
constexpr class_def unicode_casefold(const class_def &in)
Expands a character class to its Unicode simple case-fold closure (text-mode icase).
Definition compiler.hpp:65
constexpr std::vector< code_range > coalesce_ranges(std::vector< code_range > ranges)
Sorts ranges and merges overlapping / adjacent ones into a minimal, sorted set (the same set of code ...
Definition ast.hpp:106
constexpr void fold_ascii_case(char_class &klass)
Closes klass under ASCII case folding.
constexpr char_class utf8_lead2_set()
The lead-byte set of a 2-byte UTF-8 sequence.
constexpr char_class utf8_lead4_set()
The lead-byte set of a 4-byte UTF-8 sequence.
constexpr char_class utf8_cont_set()
The UTF-8 continuation-byte set 10xxxxxx.
constexpr std::int32_t max_lookaround_length
Maximum bytes a bounded lookaround sub-pattern may consume (its L_max). Bounds the per-position looka...
Definition config.hpp:42
constexpr pattern_hints analyze_program(std::span< const instr > code, std::span< const char_class > classes, std::span< const cp_class > cp_classes, std::int32_t cp_mark_ascii, std::int32_t cp_mark_offset)
Walks a compiled program once to derive its search hints.
@ any
One codepoint, except newline (the . metacharacter).
@ repeat
Child repeated [min, max] times (max -1 = unbounded).
@ lookaround
Bounded lookaround: child = sub-pattern, negated = (?!/(?<!), direction = ahead/behind.
@ byte
One exact byte.
@ concat
Children matched in sequence.
@ alternation
Children are branches, leftmost preferred.
@ anchor
Zero-width assertion; kind in real::detail::ast_node::anchor.
@ klass
One codepoint constrained by classes[klass] (a class_def; negated or not).
@ empty
Matches the empty string.
@ group
Child wrapped in a group; group >= 0 when capturing.
assert_kind
Kind of zero-width assertion carried in assert_position's arg8.
Definition program.hpp:208
@ word_start
< (non-word/start on the left, word on the right).
@ word_end
> (word on the left, non-word/end on the right).
@ line_start
^ with multiline.
@ line_end
$ with multiline.
@ word_boundary
\b (Unicode word-ness in text mode; ASCII in bytes / re.A).
@ text_start
\A, and ^ without multiline.
@ text_end_or_final_newline
$ without multiline (Python semantics).
ast build_prefix_ast(const ast &tree, std::int32_t count)
Build the prefix sub-AST: the first count top-level concat children (count >= 1). Copies the tree and...
@ klass_cp
Consume one code point tested against cp_classes[arg16] (decode + range bsearch); enters a 3-instr co...
@ byte
Consume one byte equal to arg8; fall through to pc+1.
@ save
Store current position in slot arg16; fall through (epsilon).
@ klass
Consume one byte in classes[arg16]; fall through to pc+1.
@ assert_lookaround
Epsilon; proceeds only if the lookaround sub-program arg16 holds here.
@ assert_position
Epsilon; proceeds only if assertion arg8 holds here.
@ jump
Epsilon-jump to x.
@ split
Epsilon-branch to x (preferred) and y.
constexpr std::size_t find_fold_index(std::uint32_t cp)
Binary-searches unicode_fold_table for cp; returns its index, or unicode_fold_table_size if cp is not...
constexpr bool has_flag(flags value, flags flag)
Tests whether flag is set in value.
Definition program.hpp:92
flags
Compilation flags, mirroring Python's re.I, re.M and re.S.
Definition program.hpp:42
@ dotall
. also matches \n.
@ none
No flags.
@ bytes
Binary mode: . and [^…] match raw bytes, not codepoints.
@ ascii
ASCII mode (re.A): \d \w \s \b stay ASCII and icase folds ASCII only, even in text mode....
@ icase
Case-insensitive (ASCII).
@ ecma
ECMAScript compatibility: $ (no multiline) matches only at the very end (not before a final \n,...
@ multiline
^ and $ also match at line boundaries.
@ dollar_endonly
$ (no multiline) matches only at the very end of the text, never before a final \n — the Rust/\z sema...
Search acceleration: pattern analysis and candidate-finding.
Compiled form of a pattern and the public flags / error types.
One AST node. Active fields depend on kind (noted per field).
Definition ast.hpp:79
bool lazy
repeat: prefer the shortest expansion.
Definition ast.hpp:84
std::int32_t max
repeat: maximum count (-1 = unbounded).
Definition ast.hpp:88
bool negated
klass: written as [^...] / \D \W \S.
Definition ast.hpp:83
std::int32_t klass
klass: index into ast::classes.
Definition ast.hpp:86
std::uint8_t effective_flags
Flag set in force where this node was parsed (see flags). Stamped from the scope stack; carried for s...
Definition ast.hpp:92
std::int32_t child
First child (concat, repeat, alternation, group).
Definition ast.hpp:90
std::int32_t min
repeat: minimum count.
Definition ast.hpp:87
look_dir direction
lookaround: ahead (?=/(?! or behind (?<=/(?<!.
Definition ast.hpp:85
A parsed pattern: the node pool plus side tables.
Definition ast.hpp:164
std::vector< ast_node > nodes
The node pool; root indexes it.
Definition ast.hpp:165
std::int32_t root
Index of the root node.
Definition ast.hpp:170
std::vector< named_group > names
Named capture groups.
Definition ast.hpp:167
std::vector< class_def > classes
Character classes as written, before negation.
Definition ast.hpp:166
std::int32_t group_count
Number of capturing groups.
Definition ast.hpp:169
A set of byte values (0–255) as a 256-bit bitmap.
Definition charclass.hpp:30
constexpr void set_range(std::uint8_t low, std::uint8_t high)
Adds the inclusive byte range [low, high] to the set.
Definition charclass.hpp:48
constexpr bool test(std::uint8_t byte) const
Tests membership of byte byte.
constexpr void set(std::uint8_t byte)
Adds byte byte to the set.
Definition charclass.hpp:37
constexpr bool empty() const
Reports whether the set has no members.
std::array< std::uint64_t, 4 > bits
Bitmap; bit byte is byte byte's membership.
Definition charclass.hpp:31
A parsed character class: its ASCII bitmap plus any non-ASCII code-point ranges. Bundling the two (ra...
Definition ast.hpp:98
char_class ascii
ASCII members as a bitmap (all 256 bytes in bytes mode); pre-negation.
Definition ast.hpp:99
std::vector< code_range > ranges
Non-ASCII code-point ranges (code-point mode only; empty otherwise).
Definition ast.hpp:100
An inclusive code-point range [lo, hi]. Shared by character classes (ast.hpp) and the generated Unico...
Definition program.hpp:168
A match-time code-point class for the klass_cp opcode: an ASCII bitmap for code points < 0x80 plus a ...
Definition program.hpp:179
Owning, heap-allocated program: the storage backing real::regex.
Definition program.hpp:368
std::int32_t codepoint_mark_ascii
ASCII sub-class index of an emitted codepoint-class block (-1 = none).
Definition program.hpp:386
std::vector< cp_class > cp_classes
Match-time code-point classes (for klass_cp).
Definition program.hpp:373
pattern_hints hints
Search-acceleration hints.
Definition program.hpp:382
std::vector< instr > prefix_code
IL: the inner-literal prefix sub-program (the part before the literal), for the reverse start-finder....
Definition program.hpp:375
std::uint16_t slot_count
2 * (capture groups + 1).
Definition program.hpp:379
std::vector< instr > code
The instruction stream (main program + lookaround sub-program regions).
Definition program.hpp:369
std::int32_t codepoint_mark_offset
Where that block starts (program offset); the whole-pattern hint requires offset 1.
Definition program.hpp:387
bool unicode_word
\b \B < > use Unicode word-ness (text mode).
Definition program.hpp:381
std::vector< code_range > cp_ranges
Flat range buffer the cp_class slices index into.
Definition program.hpp:374
bool byte_mode
flags::bytes mode.
Definition program.hpp:380
std::vector< lookaround_sub > lookarounds
Bounded lookaround sub-programs (regions of code).
Definition program.hpp:372
std::vector< char_class > classes
Interned character classes.
Definition program.hpp:370
std::vector< named_group > names
Named capture groups.
Definition program.hpp:371
A code point and the other members of its case-fold orbit (up to 3; orbits <= 4).
The best required inner literal of a pattern (the memmem candidate). len == 0 means the pattern decli...
One NFA instruction. Field meaning depends on op.
Definition program.hpp:251
std::int32_t inner_literal_prefix
Definition program.hpp:319
std::uint8_t inner_literal_len
Definition program.hpp:318
std::array< std::uint8_t, 16 > inner_literal
A required inner literal every match must contain (the memmem candidate the inner-literal prefilter s...
Definition program.hpp:317
A canonical UTF-8 byte-range sequence (1–4 steps) covering part of a code-point range.
Unicode simple case-folding orbits for text-mode flags::icase.
Code-point range → canonical UTF-8 byte-range sequences (RE2 / rust regex-syntax Utf8Sequences).
REAL's version macros and the C++20 language-standard guard.