REAL
Regular Expression Algorithmic Library — constexpr C++20 regex
Loading...
Searching...
No Matches
ast.hpp
Go to the documentation of this file.
1
18#ifndef REAL_AST_HPP
19#define REAL_AST_HPP
20
21// Internal — do not include directly.
22// Users: #include <real/real.hpp> (or the documented opt-ins <real/dfa.hpp>, <real/std/regex.hpp>).
23
24#include "real/version.hpp"
25
26#include <algorithm>
27#include <cstdint>
28#include <span>
29#include <string_view>
30#include <vector>
31
33#include "real/core/config.hpp"
34#include "real/core/program.hpp"
39#include "real/unicode/utf8.hpp"
40
41namespace real::detail {
42
46 enum class node_kind : std::uint8_t
47 {
48 empty,
49 byte,
50 klass,
51 any,
52 concat,
53 repeat,
55 group,
56 anchor,
58 };
59
63 enum class anchor_kind : std::uint8_t
64 {
65 caret,
66 dollar,
68 text_end,
72 word_end,
73 };
74
78 struct ast_node
79 {
81 std::uint8_t byte {};
83 bool negated {};
84 bool lazy {};
86 std::int32_t klass {-1};
87 std::int32_t min {};
88 std::int32_t max {-1};
89 std::int32_t group {-1};
90 std::int32_t child {-1};
91 std::int32_t next {-1};
92 std::uint8_t effective_flags {};
93 };
94
97 struct class_def
98 {
100 std::vector<code_range> ranges;
102 };
103
106 constexpr std::vector<code_range> coalesce_ranges(std::vector<code_range> ranges)
107 {
108 // Fast path: already sorted and disjoint (no overlap, no adjacency). The shorthand / property tables come
109 // this way, so a bare `\w`/`\d`/`\p{…}` class skips the O(n log n) sort and the second allocation entirely —
110 // which also keeps it within a static_regex's constexpr step budget (the sort of the large word table would
111 // otherwise blow it). Only a genuinely mixed class (a predicate plus a literal/range, or two predicates)
112 // falls through to the sort.
113 bool tidy {true};
114 for (std::size_t i = 1; i < ranges.size(); ++i) {
115 if (ranges[i].lo <= ranges[i - 1].hi + 1U) { // unsorted, overlapping, or adjacent
116 tidy = false;
117 break;
118 }
119 }
120 if (tidy) {
121 return ranges;
122 }
123 std::sort(ranges.begin(), ranges.end(),
124 [](const code_range& a, const code_range& b) { return a.lo < b.lo; });
125 std::vector<code_range> merged;
126 for (const code_range& r : ranges) {
127 if (!merged.empty() && r.lo <= merged.back().hi + 1U) { // overlapping OR adjacent
128 merged.back().hi = merged.back().hi > r.hi ? merged.back().hi : r.hi;
129 }
130 else {
131 merged.push_back(r);
132 }
133 }
134 return merged;
135 }
136
139 constexpr std::vector<code_range> complement_code_ranges(std::vector<code_range> ranges)
140 {
141 const std::vector<code_range> merged {coalesce_ranges(std::move(ranges))};
142 std::vector<code_range> gaps;
143 std::uint32_t next {0x80U};
144 for (const code_range& r : merged) {
145 if (r.lo > next) {
146 gaps.push_back({.lo = next, .hi = r.lo - 1U});
147 }
148 next = r.hi + 1U;
149 }
150 if (next <= 0x10FFFFU) {
151 gaps.push_back({.lo = next, .hi = 0x10FFFFU});
152 }
153 return gaps;
154 }
155
163 struct ast
164 {
165 std::vector<ast_node> nodes;
166 std::vector<class_def> classes;
167 std::vector<named_group> names;
169 std::int32_t group_count {};
170 std::int32_t root {-1};
171 };
172
174 enum class digit_escape_kind : std::uint8_t
175 {
176 octal,
177 group_ref,
179 };
180
188
202 constexpr digit_escape_result decode_digit_escape(std::string_view text,
203 std::size_t first)
204 {
205 const auto is_octal {[](char c) { return c >= '0' && c <= '7'; }};
206 const auto is_digit {[](char c) { return c >= '0' && c <= '9'; }};
207 if (text[first] == '0') {
208 unsigned value {};
209 std::size_t taken {};
210 while (taken < 3 && first + taken < text.size() && is_octal(text[first + taken])) {
211 value = (value * 8U) + static_cast<unsigned>(text[first + taken] - '0');
212 ++taken;
213 }
214 return {.kind = digit_escape_kind::octal, .value = value & 0xFFU, .length = taken};
215 }
216 std::size_t length {1};
217 if (first + 1 < text.size() && is_digit(text[first + 1])) {
218 length = 2;
219 if (is_octal(text[first]) && is_octal(text[first + 1]) && first + 2 < text.size() &&
220 is_octal(text[first + 2])) {
221 const unsigned value {(static_cast<unsigned>(text[first] - '0') * 8U * 8U) +
222 (static_cast<unsigned>(text[first + 1] - '0') * 8U) +
223 static_cast<unsigned>(text[first + 2] - '0')};
225 .value = value,
226 .length = 3}
227 : digit_escape_result {.kind = digit_escape_kind::octal, .value = value, .length = 3};
228 }
229 }
230 unsigned group {};
231 for (std::size_t k = 0; k < length; ++k) {
232 group = (group * 10U) + static_cast<unsigned>(text[first + k] - '0');
233 }
234 return {.kind = digit_escape_kind::group_ref, .value = group, .length = length};
235 }
236
240 class parser
241 {
242 public:
243
250 constexpr explicit parser(std::string_view pattern,
251 flags initial_flags = flags::none)
252 : pattern_(pattern),
253 bytes_(has_flag(initial_flags, flags::bytes)),
254 ecma_(has_flag(initial_flags, flags::ecma))
255 {
256 // The scope stack holds the flag set in force at each nesting level; the base is the
257 // constructor's flags. A scoped group (?flags:...) pushes a modified copy for its body.
258 flag_scopes_.push_back(initial_flags);
259 }
260
266 constexpr ast parse()
267 {
268 ast out;
269 while (parse_global_flags_prefix(out)) {}
270 out.root = parse_alternation(out);
271 if (pos_ != pattern_.size()) {
272 fail("unbalanced parenthesis"); // only a stray ')' stops earlier
273 }
274 return out;
275 }
276
277 private:
278
279 std::string_view pattern_;
280 std::size_t pos_ {};
281 std::int32_t depth_ {};
282 std::vector<flags> flag_scopes_;
284 bool bytes_ {};
285 bool ecma_ {};
286
288 [[nodiscard]] constexpr flags current_flags() const
289 {
290 return flag_scopes_.back();
291 }
292
295 [[nodiscard]] constexpr bool is_verbose() const
296 {
298 }
299
301 [[nodiscard]] constexpr bool is_icase() const
302 {
304 }
305
307 [[nodiscard]] constexpr bool is_ascii_mode() const
308 {
310 }
311
319 constexpr void skip_insignificant()
320 {
321 if (!is_verbose()) {
322 return;
323 }
324 while (!eof()) {
325 const char ch {peek()};
326 if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f' || ch == '\v') {
327 ++pos_;
328 }
329 else if (ch == '#') {
330 while (!eof() && peek() != '\n') {
331 ++pos_;
332 }
333 }
334 else {
335 break;
336 }
337 }
338 }
339
351 template <typename Error = regex_error>
352 [[noreturn]] constexpr void fail(const char* message) const
353 {
354 throw Error(message, pos_);
355 }
356
360 template <typename = void>
361 [[noreturn]] constexpr void fail_unsupported(const char* message) const
362 {
364 }
365
369 [[nodiscard]] constexpr bool eof() const
370 {
371 return pos_ >= pattern_.size();
372 }
373
377 [[nodiscard]] constexpr char peek() const
378 {
379 return pattern_[pos_];
380 }
381
387 [[nodiscard]] constexpr bool accept(char ch)
388 {
389 if (!eof() && peek() == ch) {
390 ++pos_;
391 return true;
392 }
393 return false;
394 }
395
401 static constexpr bool is_ascii_alnum(char ch)
402 {
403 return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z');
404 }
405
412 constexpr std::int32_t add_node(ast& out,
413 ast_node node)
414 {
415 // Stamp the flag set in force where this node was parsed (from the scope stack). Carried for
416 // scoped-flag semantics; the compiler does not read it yet, so it does not change any program.
417 node.effective_flags = static_cast<std::uint8_t>(current_flags());
418 out.nodes.push_back(node);
419 return static_cast<std::int32_t>(out.nodes.size()) - 1;
420 }
421
432 constexpr std::int32_t add_class_node(ast& out,
433 const char_class& klass,
434 bool negated,
435 const std::vector<code_range>& ranges = {},
436 bool codepoint_predicate = false)
437 {
438 out.classes.push_back({.ascii = klass, .ranges = ranges, .codepoint_predicate = codepoint_predicate});
439 const auto index {static_cast<std::int32_t>(out.classes.size()) - 1};
440 return add_node(out, {.kind = node_kind::klass, .negated = negated, .klass = index});
441 }
442
450 [[nodiscard]] constexpr bool text_shorthand() const
451 {
452 return !bytes_ && !is_ascii_mode();
453 }
454
462 std::vector<code_range>& ranges,
463 const char_class& prop_ascii,
464 std::span<const code_range> table,
465 bool negated,
466 bool& property_derived) const
467 {
468 if (negated) {
469 char_class inverted {prop_ascii};
470 if (bytes_) {
471 inverted.invert(); // raw bytes: plain 256-bit complement, no ranges
472 klass.merge(inverted);
473 return;
474 }
475 inverted.invert_ascii();
476 klass.merge(inverted);
477 // Text: the gaps between the property's ranges. ASCII: shorthand_ranges is empty, so the
478 // complement is the whole non-ASCII space (`\W` under re.A still matches é).
479 const std::vector<code_range> comp {complement_code_ranges(shorthand_ranges(table))};
480 ranges.insert(ranges.end(), comp.begin(), comp.end());
481 }
482 else {
483 klass.merge(prop_ascii);
484 const std::vector<code_range> high {shorthand_ranges(table)};
485 ranges.insert(ranges.end(), high.begin(), high.end());
486 }
487 property_derived = property_derived || text_shorthand();
488 }
489
490 [[nodiscard]] constexpr std::vector<code_range> shorthand_ranges(std::span<const code_range> table) const
491 {
492 std::vector<code_range> out;
493 if (bytes_ || is_ascii_mode()) {
494 return out;
495 }
496 for (const code_range& r : table) {
497 if (r.hi < 0x80U) {
498 continue; // wholly ASCII: already covered by the bitmap
499 }
500 out.push_back({.lo = r.lo < 0x80U ? 0x80U : r.lo, .hi = r.hi});
501 }
502 return out;
503 }
504
508 {
510 std::span<const code_range> ranges;
511 bool negated;
512 };
513
519 [[nodiscard]] static constexpr shorthand_spec shorthand_class(char letter)
520 {
521 switch (letter) {
522 case 'd': return {.set = digit_set(), .ranges = digit_ranges, .negated = false};
523 case 'D': return {.set = digit_set(), .ranges = digit_ranges, .negated = true};
524 case 'w': return {.set = word_set(), .ranges = word_ranges, .negated = false};
525 case 'W': return {.set = word_set(), .ranges = word_ranges, .negated = true};
526 case 's': return {.set = space_set(), .ranges = space_ranges, .negated = false};
527 case 'S': return {.set = space_set(), .ranges = space_ranges, .negated = true};
528 default: break;
529 }
530 // Unreachable: both call sites dispatch only on the six shorthand letters (see parse_escape /
531 // parse_class_item). Kept as a structural fallback; never hit at run time (coverage-honest).
532 return {.set = space_set(), .ranges = space_ranges, .negated = true};
533 }
534
538 {
539 std::array<char, 64> data {};
540 std::size_t len {};
541 [[nodiscard]] constexpr std::string_view view() const
542 {
543 return {data.data(), len};
544 }
545 };
546
547 [[nodiscard]] static constexpr loose_buf loose_key(std::string_view s)
548 {
549 loose_buf b;
550 for (const char c : s) {
551 if (c == '_' || c == '-' || c == ' ') {
552 continue;
553 }
554 if (b.len < b.data.size()) {
555 b.data[b.len++] = (c >= 'A' && c <= 'Z') ? static_cast<char>(c - 'A' + 'a') : c;
556 }
557 }
558 return b;
559 }
560
568 [[nodiscard]] constexpr std::vector<code_range> resolve_property(std::string_view name) const
569 {
570 std::string_view ns;
571 std::string_view value {name};
572 for (std::size_t i = 0; i < name.size(); ++i) {
573 if (name[i] == '=') {
574 ns = name.substr(0, i);
575 value = name.substr(i + 1);
576 break;
577 }
578 }
579 const loose_buf ns_key {loose_key(ns)};
580 const std::string_view nk {ns_key.view()};
581 const bool want_gc {nk.empty() || nk == "gc" || nk == "generalcategory"};
582 const bool want_script {nk.empty() || nk == "sc" || nk == "script"};
583 if (!want_gc && !want_script) {
584 // well-formed `\p{ns=...}` but a namespace REAL does not offer (a binding may delegate it).
585 fail_unsupported("unknown Unicode property namespace in \\p{...} (use gc= or sc=)");
586 }
587 const loose_buf value_key {loose_key(value)};
588 if (want_gc) {
589 const gc_property prop {resolve_gc(value_key.view())};
590 if (prop != gc_property::count) {
591 const std::span<const code_range> t {gc_property_ranges[static_cast<std::size_t>(prop)]};
592 return {t.begin(), t.end()};
593 }
594 }
595 if (want_script) {
596 const script sc {resolve_script(value_key.view())};
597 if (sc != script::count) {
598 std::vector<code_range> out;
599 for (const script_range& r : script_ranges) {
600 if (r.sc == sc) {
601 out.push_back({.lo = r.lo, .hi = r.hi});
602 }
603 }
604 return out;
605 }
606 }
607 // well-formed `\p{Name}` but a property REAL does not yet tabulate (e.g. a binary property like
608 // `\p{Alphabetic}`, or a script/category REAL lacks): unsupported, so a binding can delegate it.
609 fail_unsupported("unsupported Unicode property in \\p{...} (only General_Category and Script are built in)");
610 }
611
615 constexpr std::vector<code_range> parse_property_table()
616 {
617 // Read bytes-mode from the scope stack, not the global `bytes_` member (the flag-scope ratchet): bytes is
618 // never scoped, so this equals `bytes_` while keeping the parser's global-read count flat.
620 fail_unsupported("\\p{...} Unicode property classes are not available in bytes mode");
621 }
622 ++pos_; // consume the 'p' / 'P'
623 std::string_view name;
624 if (!eof() && peek() == '{') {
625 ++pos_;
626 const std::size_t start {pos_};
627 while (!eof() && peek() != '}') {
628 ++pos_;
629 }
630 if (eof()) {
631 fail("unterminated \\p{...} property");
632 }
633 name = pattern_.substr(start, pos_ - start);
634 ++pos_; // consume '}'
635 }
636 else {
637 if (eof()) {
638 fail("\\p must be followed by a property name or {name}");
639 }
640 name = pattern_.substr(pos_, 1);
641 ++pos_;
642 }
643 if (name.empty()) {
644 fail("empty Unicode property name in \\p{}");
645 }
646 return resolve_property(name);
647 }
648
652 constexpr void property_ascii_high(const std::vector<code_range>& table,
654 std::vector<code_range>& high) const
655 {
656 for (char32_t c = 0; c < 0x80U; ++c) {
657 if (cp_in_ranges(table, c)) {
658 ascii.set(static_cast<std::uint8_t>(c));
659 }
660 }
661 for (const code_range& r : table) {
662 if (r.hi < 0x80U) {
663 continue; // wholly ASCII: already in the bitmap
664 }
665 high.push_back({.lo = r.lo < 0x80U ? 0x80U : r.lo, .hi = r.hi});
666 }
667 }
668
674 constexpr std::int32_t parse_unicode_property(ast& out,
675 bool negated)
676 {
677 const std::vector<code_range> table {parse_property_table()};
679 std::vector<code_range> high;
680 property_ascii_high(table, ascii, high);
681 return add_class_node(out, ascii, negated, high, /*codepoint_predicate=*/ true);
682 }
683
693 std::vector<code_range>& ranges,
694 const std::vector<code_range>& table,
695 bool negated,
696 bool& property_derived) const
697 {
699 std::vector<code_range> high;
700 property_ascii_high(table, ascii, high);
701 if (negated) {
702 ascii.invert_ascii();
703 klass.merge(ascii);
704 const std::vector<code_range> comp {complement_code_ranges(high)};
705 ranges.insert(ranges.end(), comp.begin(), comp.end());
706 }
707 else {
708 klass.merge(ascii);
709 ranges.insert(ranges.end(), high.begin(), high.end());
710 }
711 property_derived = true;
712 }
713
722 constexpr std::int32_t parse_alternation(ast& out)
723 {
724 const std::int32_t first {parse_sequence(out)};
725 if (eof() || peek() != '|') {
726 return first;
727 }
728 std::int32_t last {first};
729 while (accept('|')) {
730 const std::int32_t branch {parse_sequence(out)}; // may be empty
731 out.nodes[static_cast<std::size_t>(last)].next = branch;
732 last = branch;
733 }
734 const std::int32_t alt = add_node(out, {.kind = node_kind::alternation});
735 out.nodes[static_cast<std::size_t>(alt)].child = first;
736 return alt;
737 }
738
744 constexpr std::int32_t parse_sequence(ast& out)
745 {
746 std::int32_t first {-1};
747 std::int32_t last {-1};
748 while (true) {
749 skip_insignificant(); // verbose: between elements and before '|' / ')'
750 if (eof() || peek() == '|' || peek() == ')') {
751 break;
752 }
753 std::int32_t atom {parse_atom(out)};
754 skip_insignificant(); // verbose: whitespace between an atom and its quantifier
755 atom = parse_quantifier(out, atom);
756 if (first == -1) {
757 first = atom;
758 }
759 else {
760 out.nodes[static_cast<std::size_t>(last)].next = atom;
761 }
762 last = atom;
763 }
764 if (first == -1) {
765 return add_node(out, {.kind = node_kind::empty});
766 }
767 if (out.nodes[static_cast<std::size_t>(first)].next == -1) {
768 return first; // single atom: no concat wrapper needed
769 }
770 const std::int32_t seq = add_node(out, {.kind = node_kind::concat});
771 out.nodes[static_cast<std::size_t>(seq)].child = first;
772 return seq;
773 }
774
780 constexpr std::int32_t parse_atom(ast& out)
781 {
782 const char ch {peek()};
783 switch (ch) {
784 case '*':
785 case '+':
786 case '?':
787 fail("nothing to repeat");
788 case '^':
789 ++pos_;
790 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::caret});
791 case '$':
792 ++pos_;
793 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::dollar});
794 case '(':
795 return parse_group(out);
796 case ')':
797 fail("unbalanced parenthesis");
798 case '.':
799 ++pos_;
800 return add_node(out, {.kind = node_kind::any});
801 case '[':
802 return parse_class(out);
803 case '\\':
804 return parse_escape(out);
805 default:
806 {
807 // In code-point mode a raw non-ASCII byte begins a UTF-8 sequence: decode the WHOLE
808 // code point and emit it as one atom (the same emission as `\uHHHH`), so a following
809 // quantifier applies to the code point, not just its last byte (the é+ bug). A malformed
810 // sequence is a pattern error, not a silent literal. In bytes mode, and for ASCII, a raw
811 // byte stays a single byte node (so the compat layer's bytes|ecma path is unchanged).
812 if (!bytes_ && static_cast<std::uint8_t>(ch) >= 0x80U) {
814 if (!decoded.valid) {
815 fail("invalid UTF-8 byte in pattern");
816 }
817 pos_ += decoded.length;
818 return emit_literal_codepoint(out, static_cast<std::int32_t>(decoded.cp));
819 }
820 // Like Python: lone '{', ']' and '}' are ordinary characters.
821 ++pos_;
822 return emit_literal_codepoint(out, static_cast<std::uint8_t>(ch));
823 }
824 }
825 }
826
839 constexpr std::int32_t parse_quantifier(ast& out,
840 std::int32_t atom)
841 {
842 if (eof()) {
843 return atom;
844 }
845 // Like Python: a bare anchor cannot be repeated ((?:^)* is fine).
846 if (out.nodes[static_cast<std::size_t>(atom)].kind == node_kind::anchor &&
847 (peek() == '*' || peek() == '+' || peek() == '?' || peek() == '{')) {
848 std::int32_t ignored_min {};
849 std::int32_t ignored_max {-1};
850 if (peek() != '{' || try_parse_braces(ignored_min, ignored_max)) {
851 fail("nothing to repeat");
852 }
853 }
854 std::int32_t min {};
855 std::int32_t max {-1};
856 switch (peek()) {
857 case '*':
858 ++pos_;
859 break;
860 case '+':
861 ++pos_;
862 min = 1;
863 break;
864 case '?':
865 ++pos_;
866 max = 1;
867 break;
868 case '{':
869 if (!try_parse_braces(min, max)) {
870 return atom; // literal '{': handled as the next atom
871 }
872 break;
873 default:
874 return atom;
875 }
876 const bool lazy {accept('?')};
877 if (!eof()) {
878 const char ch {peek()};
879 std::int32_t ignored_min {};
880 std::int32_t ignored_max {-1};
881 if (ch == '*' || ch == '+' || ch == '?' ||
882 (ch == '{' && try_parse_braces(ignored_min, ignored_max))) {
883 fail("multiple repeat");
884 }
885 }
886 return add_node(out, {.kind = node_kind::repeat, .lazy = lazy, .min = min, .max = max, .child = atom});
887 }
888
897 constexpr bool try_parse_braces(std::int32_t& min,
898 std::int32_t& max)
899 {
900 const std::size_t saved_pos {pos_};
901 ++pos_; // consume '{'
902 const std::int32_t repeat_min {parse_repeat_count()};
903 std::int32_t repeat_max {repeat_min};
904 bool has_comma {};
905 if (accept(',')) {
906 has_comma = true;
907 repeat_max = parse_repeat_count();
908 }
909 if (!accept('}') || (repeat_min < 0 && repeat_max < 0)) {
910 pos_ = saved_pos;
911 return false; // "{", "{}", "{,}", "{x"…: literal text
912 }
913 min = repeat_min < 0 ? 0 : repeat_min;
914 max = (has_comma && repeat_max < 0) ? -1 : repeat_max;
915 if (max != -1 && max < min) {
916 pos_ = saved_pos;
917 fail("min repeat greater than max repeat");
918 }
919 return true;
920 }
921
928 constexpr std::int32_t parse_repeat_count()
929 {
930 std::int32_t value {-1};
931 while (!eof() && peek() >= '0' && peek() <= '9') {
932 value = value < 0 ? 0 : value;
933 value = (value * 10) + (peek() - '0');
934 if (value > max_repeat_count) {
935 fail("repetition count too large");
936 }
937 ++pos_;
938 }
939 return value;
940 }
941
948 constexpr void expect(char ch,
949 const char* message)
950 {
951 if (!accept(ch)) {
952 fail(message);
953 }
954 }
955
962 static constexpr flags flag_for_letter(char letter)
963 {
964 switch (letter) {
965 case 'i':
966 return flags::icase;
967 case 'm':
968 return flags::multiline;
969 case 's':
970 return flags::dotall;
971 case 'x':
972 return flags::verbose;
973 case 'a': // ASCII mode: `\d \w \s \b` stay ASCII, icase folds ASCII only.
974 return flags::ascii;
975 default:
976 return flags::none;
977 }
978 }
979
985 static constexpr bool is_flag_letter(char letter)
986 {
987 return letter == 'i' || letter == 'm' || letter == 's' || letter == 'a' || letter == 'x';
988 }
989
991 static constexpr flags without(flags value,
992 flags bit)
993 {
994 return static_cast<flags>(
995 static_cast<std::uint8_t>(static_cast<unsigned>(value) & ~static_cast<unsigned>(bit)));
996 }
997
1008 constexpr bool parse_global_flags_prefix(ast& out)
1009 {
1010 const std::size_t saved_pos {pos_};
1011 if (!accept('(') || !accept('?')) {
1012 pos_ = saved_pos;
1013 return false;
1014 }
1015 flags found {flags::none};
1016 bool any_letter {};
1017 while (!eof() && is_flag_letter(peek())) {
1018 found = found | flag_for_letter(peek());
1019 any_letter = true;
1020 ++pos_;
1021 }
1022 if (!any_letter || !accept(')')) {
1023 pos_ = saved_pos; // some other (?...) construct: let parse_group decide
1024 return false;
1025 }
1026 out.inline_flags = out.inline_flags | found;
1027 // A leading (?flags) group sets the base scope, so the rest of the pattern is parsed under it —
1028 // verbose affects tokenization, icase/ascii affect literal folding and the \w\d\s tables. This
1029 // mirrors the constructor flags, which seed the same base scope.
1030 flag_scopes_.back() = flag_scopes_.back() | found;
1031 return true;
1032 }
1033
1052 constexpr std::int32_t parse_group(ast& out)
1053 {
1054 const std::size_t open_pos {pos_};
1055 if (++depth_ > max_nesting_depth) {
1056 fail("pattern nesting too deep");
1057 }
1058 ++pos_; // consume '('
1059 std::int32_t group {-1};
1060 bool scoped_flags {false};
1061 if (accept('?')) {
1062 if (accept('#')) {
1063 // (?#...) comment: skip to the first ')' (a backslash is not special here, like re);
1064 // emits nothing. Works the same in verbose and non-verbose mode.
1065 while (!eof() && peek() != ')') {
1066 ++pos_;
1067 }
1068 if (!accept(')')) {
1069 pos_ = open_pos;
1070 fail("missing ), unterminated comment");
1071 }
1072 --depth_;
1073 return add_node(out, {.kind = node_kind::empty});
1074 }
1075 if (accept(':')) {
1076 // non-capturing
1077 }
1078 else if (accept('P')) {
1079 if (accept('<')) {
1080 group = new_group(out, open_pos);
1081 parse_group_name(out, group);
1082 }
1083 else if (!eof() && peek() == '=') {
1084 fail_unsupported("named backreferences are not supported");
1085 }
1086 else {
1087 fail("unknown extension");
1088 }
1089 }
1090 else if (accept('<')) {
1091 if (!eof() && (peek() == '=' || peek() == '!')) {
1092 return parse_lookaround(out, look_dir::behind, open_pos);
1093 }
1094 group = new_group(out, open_pos);
1095 parse_group_name(out, group);
1096 }
1097 else if (!eof() && (peek() == '=' || peek() == '!')) {
1098 return parse_lookaround(out, look_dir::ahead, open_pos);
1099 }
1100 else if (!eof() && peek() == '>') {
1101 fail("atomic groups are not supported");
1102 }
1103 else if (!eof() && peek() == '(') {
1104 fail("conditional groups are not supported");
1105 }
1106 else if (!eof() && (is_flag_letter(peek()) || peek() == '-')) {
1107 // (?flags:...) / (?-flags:...) / (?flags-flags:...) — a scoped-flags group. Parse the
1108 // added flags, an optional '-' and the removed flags.
1109 flags added {flags::none};
1110 flags removed {flags::none};
1111 while (!eof() && is_flag_letter(peek())) {
1112 added = added | flag_for_letter(peek());
1113 ++pos_;
1114 }
1115 if (accept('-')) {
1116 bool saw_negative {false};
1117 while (!eof() && is_flag_letter(peek())) {
1118 removed = removed | flag_for_letter(peek());
1119 saw_negative = true;
1120 ++pos_;
1121 }
1122 if (!saw_negative) {
1123 fail("missing flag after '-'");
1124 }
1125 }
1126 if (!accept(':')) {
1127 // An unscoped (?flags) is only legal at the very start of the pattern (consumed by
1128 // parse_global_flags_prefix); anything else here is a misplaced global-flags group.
1129 fail("global flags not at the start of the expression");
1130 }
1131 // Every inline flag (i m s x a) is honoured per scope: verbose changes tokenization, icase/
1132 // ascii govern folding and the \w\d\s tables, dotall the dot, multiline the ^/$ anchors — all
1133 // read from the scope stack. The added set is applied and the removed set cleared for the body.
1134 flag_scopes_.push_back(without(current_flags() | added, removed));
1135 scoped_flags = true; // group stays non-capturing (-1)
1136 }
1137 else {
1138 fail("unknown extension");
1139 }
1140 }
1141 else {
1142 group = new_group(out, open_pos);
1143 }
1144 const std::int32_t body {parse_alternation(out)};
1145 if (scoped_flags) {
1146 flag_scopes_.pop_back(); // the scoped flags apply only to the body just parsed
1147 }
1148 if (!accept(')')) {
1149 pos_ = open_pos;
1150 fail("missing ), unterminated subpattern");
1151 }
1152 --depth_;
1153 return add_node(out, {.kind = node_kind::group, .group = group, .child = body});
1154 }
1155
1171 constexpr std::int32_t parse_lookaround(ast& out,
1172 look_dir direction,
1173 std::size_t open_pos)
1174 {
1175 if (in_lookaround_) {
1176 fail_unsupported("nested lookaround is not supported");
1177 }
1178 const bool negative {peek() == '!'};
1179 ++pos_; // consume '=' or '!'
1180 in_lookaround_ = true;
1181 const std::int32_t sub {parse_alternation(out)};
1182 in_lookaround_ = false;
1183 if (!accept(')')) {
1184 pos_ = open_pos;
1185 fail("missing ), unterminated subpattern");
1186 }
1187 --depth_;
1188 return add_node(out, {.kind = node_kind::lookaround,
1189 .negated = negative,
1190 .direction = direction,
1191 .child = sub});
1192 }
1193
1201 constexpr std::int32_t new_group(ast& out,
1202 std::size_t open_pos)
1203 {
1204 if (out.group_count >= max_group_count) {
1205 pos_ = open_pos;
1206 fail("too many capture groups");
1207 }
1208 return ++out.group_count;
1209 }
1210
1216 static constexpr bool is_name_start(char ch)
1217 {
1218 return ch == '_' || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z');
1219 }
1220
1227 constexpr void parse_group_name(ast& out,
1228 std::int32_t group)
1229 {
1230 const std::size_t begin {pos_};
1231 if (eof() || !is_name_start(peek())) {
1232 fail("bad character in group name");
1233 }
1234 while (!eof() && (is_ascii_alnum(peek()) || peek() == '_')) {
1235 ++pos_;
1236 }
1237 const std::size_t end {pos_};
1238 expect('>', "bad character in group name");
1239 for (const named_group& existing : out.names) {
1240 const std::string_view name {pattern_.substr(begin, end - begin)};
1241 const auto e_begin {static_cast<std::size_t>(existing.begin)};
1242 const auto e_end {static_cast<std::size_t>(existing.end)};
1243 if (pattern_.substr(e_begin, e_end - e_begin) == name) {
1244 fail("redefinition of group name");
1245 }
1246 }
1247 out.names.push_back({.group = group,
1248 .begin = static_cast<std::int32_t>(begin),
1249 .end = static_cast<std::int32_t>(end)});
1250 }
1251
1262 constexpr std::int32_t parse_byte_escape()
1263 {
1264 const char ch {peek()};
1265 if (ch >= '0' && ch <= '9') {
1266 return parse_digit_escape(); // octal byte, or a rejected back-reference
1267 }
1268 switch (ch) {
1269 case 'n':
1270 ++pos_;
1271 return '\n';
1272 case 't':
1273 ++pos_;
1274 return '\t';
1275 case 'r':
1276 ++pos_;
1277 return '\r';
1278 case 'f':
1279 ++pos_;
1280 return '\f';
1281 case 'v':
1282 ++pos_;
1283 return '\v';
1284 case 'a':
1285 ++pos_;
1286 // REAL/Python `\a` is the bell (0x07). ECMAScript has no `\a` escape — it is an identity
1287 // escape (the literal 'a'). Gate under ecma; `\n \t \r \f \v` are ECMAScript ControlEscapes
1288 // and stay unchanged. This covers both contexts (parse_byte_escape is shared with classes).
1289 if (ecma_) { return 'a'; }
1290 return '\a';
1291 case 'x':
1292 {
1293 ++pos_;
1294 const std::int32_t high_nibble {hex_digit()};
1295 const std::int32_t low_nibble {hex_digit()};
1296 return (high_nibble * 16) + low_nibble; // arithmetic, not signed bitwise (MISRA)
1297 }
1298 default:
1299 // Any escaped ASCII punctuation is that literal character.
1300 if (static_cast<std::uint8_t>(ch) < 0x80 && !is_ascii_alnum(ch)) {
1301 ++pos_;
1302 return static_cast<std::uint8_t>(ch);
1303 }
1304 return -1;
1305 }
1306 }
1307
1318 constexpr std::int32_t parse_digit_escape()
1319 {
1321 pos_ += decoded.length;
1322 if (decoded.kind == digit_escape_kind::octal) {
1323 return static_cast<std::int32_t>(decoded.value); // a single byte, like \xHH
1324 }
1325 if (decoded.kind == digit_escape_kind::octal_overflow) {
1326 fail("octal escape value outside of range 0-0o377");
1327 }
1328 fail_unsupported("backreferences are not supported"); // a decimal group number = a back-reference
1329 }
1330
1336 constexpr std::int32_t hex_digit()
1337 {
1338 if (eof()) {
1339 fail("invalid \\x escape: expected two hex digits");
1340 }
1341 const char ch {peek()};
1342 ++pos_;
1343 if (ch >= '0' && ch <= '9') {
1344 return ch - '0';
1345 }
1346 if (ch >= 'a' && ch <= 'f') {
1347 return ch - 'a' + 10;
1348 }
1349 if (ch >= 'A' && ch <= 'F') {
1350 return ch - 'A' + 10;
1351 }
1352 --pos_;
1353 fail("invalid \\x escape: expected two hex digits");
1354 }
1355
1366 constexpr std::int32_t parse_unicode_codepoint(bool capital)
1367 {
1368 if (bytes_) {
1369 fail("\\u and \\U escapes are not allowed in bytes patterns");
1370 }
1371 const int width {capital ? 8 : 4};
1372 std::int32_t value {};
1373 for (int i = 0; i < width; ++i) {
1374 std::int32_t digit {-1};
1375 if (!eof()) {
1376 const char ch {peek()};
1377 if (ch >= '0' && ch <= '9') {
1378 digit = ch - '0';
1379 }
1380 else if (ch >= 'a' && ch <= 'f') {
1381 digit = (ch - 'a') + 10;
1382 }
1383 else if (ch >= 'A' && ch <= 'F') {
1384 digit = (ch - 'A') + 10;
1385 }
1386 }
1387 if (digit < 0) {
1388 fail(capital ? "invalid \\U escape: expected 8 hex digits"
1389 : "invalid \\u escape: expected 4 hex digits");
1390 }
1391 value = (value * 16) + digit;
1392 ++pos_;
1393 }
1394 if (value >= 0xD800 && value <= 0xDFFF) {
1395 fail("invalid Unicode escape: surrogate code point");
1396 }
1397 if (value > 0x10FFFF) {
1398 fail("invalid Unicode escape: code point out of range");
1399 }
1400 return value;
1401 }
1402
1414 constexpr std::int32_t parse_named_codepoint()
1415 {
1416 if (bytes_) {
1417 fail("\\N escapes are not allowed in bytes patterns");
1418 }
1419 if (eof() || peek() != '{') {
1420 fail("expected '{' after \\N (\\N{U+XXXX})");
1421 }
1422 ++pos_; // consume '{'
1423 if (eof() || peek() != 'U') {
1424 fail("\\N{...} takes a U+XXXX code point; a character name is resolved by the Python binding");
1425 }
1426 ++pos_; // consume 'U'
1427 if (eof() || peek() != '+') {
1428 fail("expected '+' in \\N{U+XXXX}");
1429 }
1430 ++pos_; // consume '+'
1431 std::int32_t value {};
1432 int count {};
1433 while (!eof() && count < 6) {
1434 const char ch {peek()};
1435 std::int32_t digit {-1};
1436 if (ch >= '0' && ch <= '9') {
1437 digit = ch - '0';
1438 }
1439 else if (ch >= 'a' && ch <= 'f') {
1440 digit = (ch - 'a') + 10;
1441 }
1442 else if (ch >= 'A' && ch <= 'F') {
1443 digit = (ch - 'A') + 10;
1444 }
1445 if (digit < 0) {
1446 break;
1447 }
1448 value = (value * 16) + digit;
1449 ++pos_;
1450 ++count;
1451 }
1452 if (count == 0) {
1453 fail("expected 1 to 6 hex digits in \\N{U+XXXX}");
1454 }
1455 if (eof() || peek() != '}') {
1456 fail("unterminated \\N{U+XXXX} (expected '}')");
1457 }
1458 ++pos_; // consume '}'
1459 if (value >= 0xD800 && value <= 0xDFFF) {
1460 fail("invalid \\N escape: surrogate code point");
1461 }
1462 if (value > 0x10FFFF) {
1463 fail("invalid \\N escape: code point out of range");
1464 }
1465 return value;
1466 }
1467
1475 constexpr std::int32_t emit_codepoint_utf8(ast& out,
1476 std::int32_t cp)
1477 {
1478 const auto value {static_cast<std::uint32_t>(cp)};
1479 if (value < 0x80U) {
1480 return add_node(out, {.kind = node_kind::byte, .byte = static_cast<std::uint8_t>(value)});
1481 }
1482 std::int32_t first {-1};
1483 std::int32_t last {-1};
1484 const auto emit_byte {[&](std::uint8_t one) {
1485 const std::int32_t node {add_node(out, {.kind = node_kind::byte, .byte = one})};
1486 if (first < 0) {
1487 first = node;
1488 }
1489 else {
1490 out.nodes[static_cast<std::size_t>(last)].next = node;
1491 }
1492 last = node;
1493 }};
1494 if (value < 0x800U) {
1495 emit_byte(static_cast<std::uint8_t>(0xC0U | (value >> 6U)));
1496 emit_byte(static_cast<std::uint8_t>(0x80U | (value & 0x3FU)));
1497 }
1498 else if (value < 0x10000U) {
1499 emit_byte(static_cast<std::uint8_t>(0xE0U | (value >> 12U)));
1500 emit_byte(static_cast<std::uint8_t>(0x80U | ((value >> 6U) & 0x3FU)));
1501 emit_byte(static_cast<std::uint8_t>(0x80U | (value & 0x3FU)));
1502 }
1503 else {
1504 emit_byte(static_cast<std::uint8_t>(0xF0U | (value >> 18U)));
1505 emit_byte(static_cast<std::uint8_t>(0x80U | ((value >> 12U) & 0x3FU)));
1506 emit_byte(static_cast<std::uint8_t>(0x80U | ((value >> 6U) & 0x3FU)));
1507 emit_byte(static_cast<std::uint8_t>(0x80U | (value & 0x3FU)));
1508 }
1509 const std::int32_t seq {add_node(out, {.kind = node_kind::concat})};
1510 out.nodes[static_cast<std::size_t>(seq)].child = first;
1511 return seq;
1512 }
1513
1523 constexpr std::int32_t emit_literal_codepoint(ast& out,
1524 std::int32_t cp)
1525 {
1526 if (is_icase()) {
1527 const bool ascii_letter {(cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z')};
1528 if (ascii_letter) {
1529 char_class bitmap;
1530 bitmap.set(static_cast<std::uint8_t>(cp));
1531 return add_class_node(out, bitmap, false);
1532 }
1533 if (!bytes_ && cp >= 0x80 &&
1534 detail::find_fold_index(static_cast<std::uint32_t>(cp)) != detail::unicode_fold_table_size) {
1535 const std::vector<code_range> single {
1536 {.lo = static_cast<std::uint32_t>(cp), .hi = static_cast<std::uint32_t>(cp)}};
1537 return add_class_node(out, char_class {}, false, single);
1538 }
1539 }
1540 // Not promoted: a raw byte (ASCII, or any byte in bytes mode) is a byte node; a non-ASCII
1541 // code point in text mode is emitted as its UTF-8 bytes.
1542 if (bytes_ || cp < 0x80) {
1543 return add_node(out, {.kind = node_kind::byte, .byte = static_cast<std::uint8_t>(cp)});
1544 }
1545 return emit_codepoint_utf8(out, cp);
1546 }
1547
1558 constexpr std::int32_t parse_escape(ast& out)
1559 {
1560 ++pos_; // consume the backslash
1561 if (eof()) {
1562 fail("dangling backslash");
1563 }
1564 switch (peek()) {
1565 // A bare shorthand in text mode (not bytes, not re.A) is emitted as a match-time code-point
1566 // predicate (klass_cp): O(decode + range bsearch) per position, independent of the range count.
1567 // In bytes / ASCII mode shorthand_ranges() is empty and the flag is off -> the ASCII byte-NFA.
1568 case 'd':
1569 case 'D':
1570 case 'w':
1571 case 'W':
1572 case 's':
1573 case 'S': {
1574 const shorthand_spec sc {shorthand_class(peek())};
1575 ++pos_;
1576 return add_class_node(out, sc.set, sc.negated, shorthand_ranges(sc.ranges), text_shorthand());
1577 }
1578 // `\p{Name}` / `\P{Name}` / `\pX` — Unicode General_Category and Script property classes (text mode).
1579 case 'p':
1580 case 'P':
1581 return parse_unicode_property(out, peek() == 'P');
1582 // `\A \Z < >` are REAL extensions (text-start/end, word-start/end). ECMAScript has no
1583 // such escapes — they are identity escapes (the literal character). Under the ecma flag
1584 // (the std-compat layer), emit the literal; otherwise keep REAL's anchor. `\b`/`\B` are
1585 // standard word boundaries in both and stay unchanged.
1586 case 'A':
1587 ++pos_;
1588 // ecma: `\A` is the literal 'A' (Annex B identity escape); a cased letter, so it folds under icase.
1589 if (ecma_) {
1590 return emit_literal_codepoint(out, 'A');
1591 }
1592 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::text_start});
1593 case 'Z':
1594 ++pos_;
1595 if (ecma_) {
1596 return emit_literal_codepoint(out, 'Z'); // ecma: literal 'Z', folds under icase
1597 }
1598 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::text_end});
1599 case 'z':
1600 // `\z` is an exact alias of `\Z` (end of the text, no MULTILINE interaction) — Python 3.14
1601 // added it with that meaning. Same anchor, so it is byte-identical to `\Z`.
1602 ++pos_;
1603 if (ecma_) {
1604 return emit_literal_codepoint(out, 'z'); // ecma: literal 'z' (identity escape), folds under icase
1605 }
1606 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::text_end});
1607 case 'b':
1608 ++pos_;
1609 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::word_boundary});
1610 case 'B':
1611 ++pos_;
1612 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::not_word_boundary});
1613 case '<':
1614 ++pos_;
1615 if (ecma_) {
1616 return emit_literal_codepoint(out, '<'); // ecma: literal '<' (non-cased -> a plain byte)
1617 }
1618 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::word_start});
1619 case '>':
1620 ++pos_;
1621 if (ecma_) {
1622 return emit_literal_codepoint(out, '>'); // ecma: literal '>'
1623 }
1624 return add_node(out, {.kind = node_kind::anchor, .anchor = anchor_kind::word_end});
1625 case 'u':
1626 ++pos_;
1628 case 'U':
1629 ++pos_;
1631 case 'N':
1632 ++pos_;
1634 default:
1635 {
1636 const std::int32_t byte_value {parse_byte_escape()};
1637 if (byte_value < 0) {
1638 fail_unsupported("unsupported escape sequence");
1639 }
1640 // A `\xHH` / octal escape with value < 0x80 is an ASCII character (byte == code point): a
1641 // cased one folds under icase like a raw ASCII literal (`\x4B` == `K`). A value >= 0x80
1642 // keeps byte provenance and is never folded — the documented text-mode divergence.
1643 if (byte_value < 0x80) {
1644 return emit_literal_codepoint(out, byte_value);
1645 }
1646 return add_node(out, {.kind = node_kind::byte, .byte = static_cast<std::uint8_t>(byte_value)});
1647 }
1648 }
1649 }
1650
1663 constexpr std::int32_t parse_class_item(char_class& klass,
1664 std::vector<code_range>& ranges,
1665 bool& property_derived)
1666 {
1667 const char ch {peek()};
1668 if (static_cast<std::uint8_t>(ch) >= 0x80) {
1669 // bytes mode keeps rejecting non-ASCII in a class (the compat layer relies on that rejection
1670 // to fall back to std). Code-point mode decodes the whole code point as a class member.
1671 if (bytes_) {
1672 fail("non-ASCII character class member not supported");
1673 }
1675 if (!decoded.valid) {
1676 fail("invalid UTF-8 byte in character class");
1677 }
1678 pos_ += decoded.length;
1679 return static_cast<std::int32_t>(decoded.cp); // a code point (may be >= 0x80)
1680 }
1681 if (ch != '\\') {
1682 ++pos_;
1683 return static_cast<std::uint8_t>(ch);
1684 }
1685 ++pos_; // consume the backslash
1686 if (eof()) {
1687 fail("dangling backslash");
1688 }
1689 switch (peek()) {
1690 case 'd':
1691 case 'D':
1692 case 'w':
1693 case 'W':
1694 case 's':
1695 case 'S': {
1696 const shorthand_spec sc {shorthand_class(peek())};
1697 ++pos_;
1698 merge_property(klass, ranges, sc.set, sc.ranges, sc.negated, property_derived);
1699 return -1;
1700 }
1701 // `\p{Name}` / `\P{Name}` / `\pX` inside a class — a Unicode General_Category / Script property member.
1702 case 'p':
1703 case 'P': {
1704 const bool negated {peek() == 'P'};
1705 const std::vector<code_range> table {parse_property_table()};
1706 merge_unicode_property(klass, ranges, table, negated, property_derived);
1707 return -1;
1708 }
1709 case 'b':
1710 ++pos_;
1711 return 0x08; // backspace, only inside classes
1712 case 'u':
1713 case 'U':
1714 {
1715 const bool capital {peek() == 'U'};
1716 ++pos_;
1717 // A non-ASCII code point is now a valid class member (code-point mode); `parse_unicode_codepoint`
1718 // already rejects `\u`/`\U` in bytes mode, so a class in bytes mode still has ASCII-only members.
1719 return parse_unicode_codepoint(capital);
1720 }
1721 case 'N':
1722 ++pos_;
1723 return parse_named_codepoint(); // \N{U+XXXX} is a valid class member (a code point)
1724 case '0':
1725 case '1':
1726 case '2':
1727 case '3':
1728 case '4':
1729 case '5':
1730 case '6':
1731 case '7':
1732 {
1733 // Inside a class every `\digit` is octal — there are no back-references in a class (re's
1734 // rule). Up to 3 octal digits; a value above 0o377 (255) is out of range, as in re. The
1735 // first non-octal digit ends the escape, so `[\18]` is `\x01` then a literal '8'.
1736 unsigned value {};
1737 std::size_t taken {};
1738 while (taken < 3 && !eof() && peek() >= '0' && peek() <= '7') {
1739 value = (value * 8U) + static_cast<unsigned>(peek() - '0');
1740 ++pos_;
1741 ++taken;
1742 }
1743 if (value > 0xFFU) {
1744 fail("octal escape value out of range (\\0 to \\377)");
1745 }
1746 return static_cast<std::int32_t>(value);
1747 }
1748 case '8':
1749 case '9':
1750 fail("invalid escape (\\8 and \\9 are not octal and there are no back-references in a class)");
1751 default:
1752 {
1753 const std::int32_t byte_value {parse_byte_escape()};
1754 if (byte_value < 0) {
1755 fail_unsupported("unsupported escape sequence");
1756 }
1757 return byte_value;
1758 }
1759 }
1760 }
1761
1772 constexpr std::int32_t parse_class(ast& out)
1773 {
1774 const std::size_t open_pos {pos_};
1775 ++pos_; // consume '['
1776 const bool negated {accept('^')};
1778 std::vector<code_range> ranges; // non-ASCII members (code-point mode); empty in bytes/ASCII-only classes
1779 bool property_derived {}; // a \w/\d/\s (text mode) contributed -> emit as klass_cp
1780 bool first {true};
1781 // Add one member. In bytes mode a member >= 0x80 (from `\xHH`) is a raw byte in the bitmap, NOT
1782 // a code point — so class_ranges stays empty and a bytes-mode class is byte-for-byte a
1783 // std::basic_regex<char> class (what the compat layer relies on). In code-point mode, >= 0x80 is
1784 // a (degenerate) code-point range.
1785 const auto add_cp {[&](std::int32_t cp) {
1786 if (bytes_ || cp < 0x80) {
1787 klass.set(static_cast<std::uint8_t>(cp));
1788 }
1789 else {
1790 ranges.push_back({static_cast<std::uint32_t>(cp), static_cast<std::uint32_t>(cp)});
1791 }
1792 }};
1793 // Add an inclusive range [lo, hi]. Bytes mode: the whole range is bytes in the bitmap.
1794 // Code-point mode: a range crossing 0x7F/0x80 splits (the ASCII part -> bitmap).
1795 const auto add_range {[&](std::int32_t lo, std::int32_t hi) {
1796 if (bytes_) {
1797 klass.set_range(static_cast<std::uint8_t>(lo), static_cast<std::uint8_t>(hi));
1798 }
1799 else if (lo < 0x80) {
1800 klass.set_range(static_cast<std::uint8_t>(lo), static_cast<std::uint8_t>(hi < 0x80 ? hi : 0x7F));
1801 if (hi >= 0x80) {
1802 ranges.push_back({0x80U, static_cast<std::uint32_t>(hi)});
1803 }
1804 }
1805 else {
1806 ranges.push_back({static_cast<std::uint32_t>(lo), static_cast<std::uint32_t>(hi)});
1807 }
1808 }};
1809 while (true) {
1810 if (eof()) {
1811 pos_ = open_pos;
1812 fail("unterminated character class");
1813 }
1814 // Python (default): a ']' right after '[' or '[^' is a literal member, so `[]`/`[^]`
1815 // continue. ECMAScript (ecma): ']' always closes — `[]` is the empty class (matches
1816 // nothing) and `[^]` is its negation (matches any character, the "any incl. newline" idiom).
1817 if (peek() == ']' && (!first || ecma_)) {
1818 ++pos_;
1819 break;
1820 }
1821 first = false;
1822 const std::size_t item_pos {pos_};
1823 const std::int32_t range_start {parse_class_item(klass, ranges, property_derived)};
1824 if (range_start < 0) {
1825 continue; // set item (e.g. \d): its bitmap and any Unicode ranges are already merged
1826 }
1827 // Possible range: 'x-y', where a trailing '-]' is a literal '-'.
1828 if (!eof() && peek() == '-' && pos_ + 1 < pattern_.size() &&
1829 pattern_[pos_ + 1] != ']') {
1830 ++pos_; // consume '-'
1831 const std::int32_t range_end {parse_class_item(klass, ranges, property_derived)};
1832 if (range_end < 0 || range_end < range_start) {
1833 pos_ = item_pos;
1834 fail("bad character range");
1835 }
1836 add_range(range_start, range_end);
1837 }
1838 else {
1839 add_cp(range_start);
1840 }
1841 }
1842 return add_class_node(out, klass, negated, ranges, property_derived);
1843 }
1844 };
1845
1853 constexpr ast parse(std::string_view pattern,
1854 flags initial_flags = flags::none)
1855 {
1856 return parser(pattern, initial_flags).parse();
1857 }
1858} // namespace real::detail
1859
1860#endif // REAL_AST_HPP
256-bit byte set with O(1) membership, fully constexpr.
Recursive-descent parser: a pattern string in, an ast out.
Definition ast.hpp:241
constexpr bool eof() const
Returns true if the read offset is at or past the end of the pattern.
Definition ast.hpp:369
constexpr std::vector< code_range > shorthand_ranges(std::span< const code_range > table) const
Definition ast.hpp:490
static constexpr flags without(flags value, flags bit)
value with bit cleared.
Definition ast.hpp:991
constexpr void merge_property(char_class &klass, std::vector< code_range > &ranges, const char_class &prop_ascii, std::span< const code_range > table, bool negated, bool &property_derived) const
Merges an in-class shorthand (\w \d \s or a negated \W \D \S) into the class being built: its ASCII b...
Definition ast.hpp:461
static constexpr loose_buf loose_key(std::string_view s)
Definition ast.hpp:547
std::vector< flags > flag_scopes_
Stack of the flag set in force per nesting level; the top is current. Replaces a global verbose_ read...
Definition ast.hpp:282
constexpr std::vector< code_range > resolve_property(std::string_view name) const
Resolves a \p{...} property name to its code-point ranges, or fails with a clear error....
Definition ast.hpp:568
constexpr std::int32_t parse_atom(ast &out)
Parses one atom: a literal, ., a class, a group, an anchor or an escape.
Definition ast.hpp:780
constexpr char peek() const
Returns the current character without consuming it (undefined at eof()).
Definition ast.hpp:377
constexpr std::int32_t parse_escape(ast &out)
Parses an escape outside a character class.
Definition ast.hpp:1558
constexpr std::int32_t parse_digit_escape()
Parses a <digit> escape via the shared decode_digit_escape().
Definition ast.hpp:1318
constexpr ast parse()
Parses the whole pattern.
Definition ast.hpp:266
constexpr bool is_ascii_mode() const
True when ascii (re.A) is in force at the current scope (a scoped (?a:...) honoured).
Definition ast.hpp:307
constexpr std::vector< code_range > parse_property_table()
Rejects bytes mode, consumes the p/P and the {Name} (or single letter), and resolves it to the proper...
Definition ast.hpp:615
constexpr std::int32_t parse_lookaround(ast &out, look_dir direction, std::size_t open_pos)
Parses a lookaround after (?= / (?! (ahead) or (?<= / (?<! (behind) — the =/! is not yet consumed.
Definition ast.hpp:1171
constexpr std::int32_t add_class_node(ast &out, const char_class &klass, bool negated, const std::vector< code_range > &ranges={}, bool codepoint_predicate=false)
Interns a class bitmap and appends a node_kind::klass node.
Definition ast.hpp:432
constexpr void merge_unicode_property(char_class &klass, std::vector< code_range > &ranges, const std::vector< code_range > &table, bool negated, bool &property_derived) const
Merges a \p{Name} / \P{Name} property into the character class being built (the in-class form) — the ...
Definition ast.hpp:692
constexpr std::int32_t parse_named_codepoint()
Decodes a \N{U+XXXX} named-code-point escape (1–6 hex digits) — the same code-point path as \u/\U,...
Definition ast.hpp:1414
constexpr std::int32_t parse_quantifier(ast &out, std::int32_t atom)
Wraps atom in a repeat node if a quantifier follows.
Definition ast.hpp:839
constexpr std::int32_t parse_unicode_property(ast &out, bool negated)
Parses \p{Name} / \P{Name} / \pX (outside a class) into a negatable Unicode code-point class (klass_c...
Definition ast.hpp:674
constexpr std::int32_t parse_group(ast &out)
Parses a group construct.
Definition ast.hpp:1052
std::string_view pattern_
The pattern being parsed.
Definition ast.hpp:279
constexpr bool is_verbose() const
True when verbose mode (re.X) is in force here — read from the scope stack, so a scoped (?...
Definition ast.hpp:295
constexpr bool parse_global_flags_prefix(ast &out)
Consumes a leading (?ims) global-flags group, if present.
Definition ast.hpp:1008
constexpr std::int32_t parse_byte_escape()
Parses a single-byte escape (valid inside and outside classes).
Definition ast.hpp:1262
constexpr std::int32_t parse_sequence(ast &out)
Parses sequence := (atom quantifier?)*, stopping at | or ).
Definition ast.hpp:744
constexpr std::int32_t parse_class_item(char_class &klass, std::vector< code_range > &ranges, bool &property_derived)
Parses one member inside a character class.
Definition ast.hpp:1663
std::int32_t depth_
Current group nesting (see max_nesting_depth).
Definition ast.hpp:281
constexpr std::int32_t new_group(ast &out, std::size_t open_pos)
Allocates the next capture group number.
Definition ast.hpp:1201
constexpr void parse_group_name(ast &out, std::int32_t group)
Parses ‘name := [A-Za-z_][A-Za-z0-9_]* ’>'` and records it.
Definition ast.hpp:1227
bool bytes_
In flags::bytes mode, rejects code-point escapes (\u/\U).
Definition ast.hpp:284
constexpr void fail(const char *message) const
Aborts the parse with a real::regex_error at the current offset.
Definition ast.hpp:352
constexpr std::int32_t parse_class(ast &out)
Parses a bracketed character class [...] or [^...].
Definition ast.hpp:1772
constexpr std::int32_t add_node(ast &out, ast_node node)
Appends node to the pool.
Definition ast.hpp:412
bool ecma_
ECMAScript grammar: \A \Z < > are identity-escape literals, not anchors.
Definition ast.hpp:285
static constexpr flags flag_for_letter(char letter)
Maps a flag letter to its flags value.
Definition ast.hpp:962
constexpr std::int32_t parse_unicode_codepoint(bool capital)
Decodes a \uHHHH (4 hex) or \UHHHHHHHH (8 hex) code-point escape (str only).
Definition ast.hpp:1366
constexpr std::int32_t parse_alternation(ast &out)
Parses ‘alternation := sequence (’|' sequence)*`.
Definition ast.hpp:722
std::size_t pos_
Current read offset into pattern_.
Definition ast.hpp:280
static constexpr shorthand_spec shorthand_class(char letter)
Maps a shorthand letter to its shorthand_spec. The single place the letter -> (set,...
Definition ast.hpp:519
static constexpr bool is_name_start(char ch)
Returns true if ch may start a group name.
Definition ast.hpp:1216
constexpr bool text_shorthand() const
The non-ASCII (>= 0x80) code-point ranges of a Unicode property table (\d/\s/\w), for a text-mode sho...
Definition ast.hpp:450
constexpr std::int32_t emit_literal_codepoint(ast &out, std::int32_t cp)
Emits a code-point literal (code-point provenance: a raw character or \\u/\\U).
Definition ast.hpp:1523
bool in_lookaround_
True while parsing a lookaround sub-pattern (rejects nesting).
Definition ast.hpp:283
constexpr void skip_insignificant()
In verbose mode, consumes insignificant whitespace and # comments.
Definition ast.hpp:319
constexpr void fail_unsupported(const char *message) const
Like fail, but tags the error as unsupported (well-formed but beyond REAL's linear engine — a backref...
Definition ast.hpp:361
constexpr void expect(char ch, const char *message)
Consumes ch or fails.
Definition ast.hpp:948
constexpr std::int32_t hex_digit()
Consumes one hexadecimal digit.
Definition ast.hpp:1336
constexpr bool accept(char ch)
Consumes the current character if it equals ch.
Definition ast.hpp:387
constexpr bool try_parse_braces(std::int32_t &min, std::int32_t &max)
Tries to parse {n} / {n,} / {,m} / {n,m} starting at {.
Definition ast.hpp:897
constexpr void property_ascii_high(const std::vector< code_range > &table, char_class &ascii, std::vector< code_range > &high) const
Splits a property's ranges into its ASCII bitmap (< 0x80) and its non-ASCII ranges....
Definition ast.hpp:652
constexpr bool is_icase() const
True when icase (re.I) is in force at the current scope (a scoped (?i:...) honoured).
Definition ast.hpp:301
static constexpr bool is_flag_letter(char letter)
Returns true if letter is a flag letter (imsax).
Definition ast.hpp:985
constexpr parser(std::string_view pattern, flags initial_flags=flags::none)
Binds the parser to a pattern and the constructor flags.
Definition ast.hpp:250
constexpr flags current_flags() const
The flag set in force at the current nesting level (the scope-stack top).
Definition ast.hpp:288
constexpr std::int32_t emit_codepoint_utf8(ast &out, std::int32_t cp)
Emits a code point as its 1–4 UTF-8 bytes — the same byte-level form a literal multi-byte character p...
Definition ast.hpp:1475
constexpr std::int32_t parse_repeat_count()
Reads an optional decimal repeat count.
Definition ast.hpp:928
static constexpr bool is_ascii_alnum(char ch)
Returns true if ch is in [0-9A-Za-z].
Definition ast.hpp:401
Resource limits guarding against pattern-driven resource exhaustion.
constexpr char_class digit_set()
The ASCII digit set behind \d (Python re.ASCII semantics).
anchor_kind
The specific zero-width assertion of an anchor node (see node_kind::anchor).
Definition ast.hpp:64
@ word_start
< (start of word; REAL extension, not in Python re).
@ caret
^ (text or line start, depending on multiline).
@ word_end
> (end of word; REAL extension, not in Python re).
@ dollar
$ (end, before a trailing \n, or line end with m).
constexpr gc_property resolve_gc(std::string_view loose)
Resolve a loose-normalized General_Category name to its property, or count if unknown.
constexpr decoded_codepoint decode_codepoint_strict(std::string_view text, std::size_t pos)
Strictly decodes and validates the UTF-8 sequence at text[pos].
Definition utf8.hpp:44
constexpr digit_escape_result decode_digit_escape(std::string_view text, std::size_t first)
Decodes a <digit> escape per CPython's exact rule (shared by the pattern parser and the replacement-...
Definition ast.hpp:202
constexpr bool cp_in_ranges(std::span< const code_range > ranges, char32_t cp)
Binary-searches a sorted, non-overlapping range table for cp. Returns a bool (not a pointer into the ...
constexpr std::int32_t max_group_count
Maximum capture groups; bounds slot_count = 2 * (groups + 1).
Definition config.hpp:35
constexpr std::span< const code_range > gc_property_ranges[]
Range table indexed by gc_property (parallel to the enum order).
constexpr char_class word_set()
The ASCII word set behind \w.
constexpr std::size_t unicode_fold_table_size
Number of entries in unicode_fold_table.
digit_escape_kind
What a <digit> escape decoded to (see decode_digit_escape()).
Definition ast.hpp:175
@ octal
An octal byte escape; value is the byte (0-255).
@ group_ref
A decimal group number; value is the group (a back-reference in a pattern).
@ octal_overflow
A 3-octal-digit escape greater than 0o377 (an error in CPython).
constexpr std::vector< code_range > complement_code_ranges(std::vector< code_range > ranges)
Complements a set of code-point ranges within [0x80, 0x10FFFF] (used by negated classes and by an in-...
Definition ast.hpp:139
gc_property
A Unicode General_Category property: the 29 assignable categories then the 7 groups.
constexpr std::vector< code_range > coalesce_ranges(std::vector< code_range > ranges)
Sorts ranges and merges overlapping / adjacent ones into a minimal, sorted set (the same set of code ...
Definition ast.hpp:106
look_dir
Direction of a lookaround sub-pattern.
Definition program.hpp:224
@ ahead
(?= / (?! — the sub matches starting at the position.
@ behind
(?<= / (?<! — the sub matches ending exactly at the position.
constexpr ast parse(std::string_view pattern, flags initial_flags=flags::none)
Parses pattern into an ast (convenience over parser).
Definition ast.hpp:1853
constexpr code_range word_ranges[]
Code-point ranges matched by \w (771 ranges, 142940 code points).
constexpr code_range space_ranges[]
Code-point ranges matched by \s (10 ranges, 29 code points).
constexpr script resolve_script(std::string_view loose)
Resolve a loose-normalized Script name to its value, or count if unknown.
script
A Unicode Script value; Unknown (0) is every code point no script assigns.
node_kind
Kind of an AST node; selects which fields of real::detail::ast_node are meaningful.
Definition ast.hpp:47
@ any
One codepoint, except newline (the . metacharacter).
@ repeat
Child repeated [min, max] times (max -1 = unbounded).
@ lookaround
Bounded lookaround: child = sub-pattern, negated = (?!/(?<!), direction = ahead/behind.
@ byte
One exact byte.
@ concat
Children matched in sequence.
@ alternation
Children are branches, leftmost preferred.
@ anchor
Zero-width assertion; kind in real::detail::ast_node::anchor.
@ klass
One codepoint constrained by classes[klass] (a class_def; negated or not).
@ empty
Matches the empty string.
@ group
Child wrapped in a group; group >= 0 when capturing.
@ word_start
< (non-word/start on the left, word on the right).
@ word_end
> (word on the left, non-word/end on the right).
@ word_boundary
\b (Unicode word-ness in text mode; ASCII in bytes / re.A).
@ text_start
\A, and ^ without multiline.
constexpr code_range digit_ranges[]
Code-point ranges matched by \d (71 ranges, 760 code points).
constexpr char_class space_set()
The ASCII whitespace set behind \s.
constexpr std::int32_t max_repeat_count
Per-quantifier bounded-repeat cap, enforced at parse time.
Definition config.hpp:32
constexpr script_range script_ranges[]
Script partition — 979 ranges, sorted and disjoint.
constexpr std::int32_t max_nesting_depth
Maximum parser recursion depth; prevents stack overflow on deep nesting.
Definition config.hpp:38
@ byte
Consume one byte equal to arg8; fall through to pc+1.
@ klass
Consume one byte in classes[arg16]; fall through to pc+1.
constexpr std::size_t find_fold_index(std::uint32_t cp)
Binary-searches unicode_fold_table for cp; returns its index, or unicode_fold_table_size if cp is not...
@ first
Leftmost-first (Perl / Python re / the crate): source-order thread priority decides....
constexpr bool has_flag(flags value, flags flag)
Tests whether flag is set in value.
Definition program.hpp:92
flags
Compilation flags, mirroring Python's re.I, re.M and re.S.
Definition program.hpp:42
@ dotall
. also matches \n.
@ verbose
Verbose mode (re.X): ignore unescaped whitespace and # comments outside classes.
@ none
No flags.
@ bytes
Binary mode: . and [^…] match raw bytes, not codepoints.
@ ascii
ASCII mode (re.A): \d \w \s \b stay ASCII and icase folds ASCII only, even in text mode....
@ icase
Case-insensitive (ASCII).
@ ecma
ECMAScript compatibility: $ (no multiline) matches only at the very end (not before a final \n,...
@ multiline
^ and $ also match at line boundaries.
Compiled form of a pattern and the public flags / error types.
One AST node. Active fields depend on kind (noted per field).
Definition ast.hpp:79
bool lazy
repeat: prefer the shortest expansion.
Definition ast.hpp:84
std::int32_t max
repeat: maximum count (-1 = unbounded).
Definition ast.hpp:88
bool negated
klass: written as [^...] / \D \W \S.
Definition ast.hpp:83
std::int32_t klass
klass: index into ast::classes.
Definition ast.hpp:86
std::int32_t group
group: capture number, -1 for (?:...).
Definition ast.hpp:89
anchor_kind anchor
anchor: the assertion kind.
Definition ast.hpp:82
std::uint8_t effective_flags
Flag set in force where this node was parsed (see flags). Stamped from the scope stack; carried for s...
Definition ast.hpp:92
std::int32_t child
First child (concat, repeat, alternation, group).
Definition ast.hpp:90
std::int32_t next
Next sibling in the parent's child list.
Definition ast.hpp:91
std::int32_t min
repeat: minimum count.
Definition ast.hpp:87
look_dir direction
lookaround: ahead (?=/(?! or behind (?<=/(?<!.
Definition ast.hpp:85
node_kind kind
Which fields below are meaningful.
Definition ast.hpp:80
A parsed pattern: the node pool plus side tables.
Definition ast.hpp:164
std::vector< ast_node > nodes
The node pool; root indexes it.
Definition ast.hpp:165
flags inline_flags
Flags from a leading (?ims).
Definition ast.hpp:168
std::int32_t root
Index of the root node.
Definition ast.hpp:170
std::vector< named_group > names
Named capture groups.
Definition ast.hpp:167
std::vector< class_def > classes
Character classes as written, before negation.
Definition ast.hpp:166
std::int32_t group_count
Number of capturing groups.
Definition ast.hpp:169
A set of byte values (0–255) as a 256-bit bitmap.
Definition charclass.hpp:30
constexpr void invert()
Full 256-bit complement (binary mode: raw bytes, no UTF-8).
Definition charclass.hpp:95
constexpr void set(std::uint8_t byte)
Adds byte byte to the set.
Definition charclass.hpp:37
A parsed character class: its ASCII bitmap plus any non-ASCII code-point ranges. Bundling the two (ra...
Definition ast.hpp:98
char_class ascii
ASCII members as a bitmap (all 256 bytes in bytes mode); pre-negation.
Definition ast.hpp:99
bool codepoint_predicate
Emit as a match-time klass_cp (a Unicode shorthand \w/\d/\s in text mode), not the byte-NFA.
Definition ast.hpp:101
std::vector< code_range > ranges
Non-ASCII code-point ranges (code-point mode only; empty otherwise).
Definition ast.hpp:100
An inclusive code-point range [lo, hi]. Shared by character classes (ast.hpp) and the generated Unico...
Definition program.hpp:168
The result of a strict UTF-8 decode: the code point, its byte length, and validity.
Definition utf8.hpp:25
Result of decode_digit_escape().
Definition ast.hpp:183
unsigned value
Octal byte, or decimal group number.
Definition ast.hpp:185
digit_escape_kind kind
Which interpretation applies.
Definition ast.hpp:184
std::size_t length
Characters consumed from the first digit.
Definition ast.hpp:186
A named capture group.
Definition program.hpp:329
A loose-match key (lowercase, no _/-/space; UAX44-LM3-ish) built into a fixed buffer,...
Definition ast.hpp:538
std::array< char, 64 > data
Definition ast.hpp:539
constexpr std::string_view view() const
Definition ast.hpp:541
The classification of a \d \D \w \W \s \S shorthand: its ASCII bitmap, its Unicode range table,...
Definition ast.hpp:508
bool negated
True for the uppercase form (\D \W \S).
Definition ast.hpp:511
std::span< const code_range > ranges
The full Unicode range table (used in text mode).
Definition ast.hpp:510
char_class set
The ASCII bitmap (digit / word / space set).
Definition ast.hpp:509
One code-point range and the Script it belongs to (the table partitions the code space).
Unicode simple case-folding orbits for text-mode flags::icase.
Unicode General_Category ranges for \p{...} (UCD contract; distinct from the re-contract shorthands).
Unicode \w / \d / \s property ranges and their lookups.
Unicode Script ranges for \p{sc=...} (UCD contract, parsed from tools/ucd/Scripts....
UTF-8 position arithmetic for match iteration.
REAL's version macros and the C++20 language-standard guard.