lexer.hpp

Go to the documentation of this file.

 
#ifndef SCILEX_LEXER_HPP
#define SCILEX_LEXER_HPP
 
#include <algorithm>
#include <array>
#include <iterator>
#include <map>
#include <memory>
#include <optional>
#include <random>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <tuple>
#include <unordered_set>
#include <utility>
#include <vector>
 
#include <real/dfa.hpp>
#include <real/real.hpp>
 
#include "token.hpp"
 
namespace scilex {
 
  class token_iterator;
  class token_range;
 
  enum class eof_policy
  {
    omit,   
    append, 
  };
 
  struct mode_action
  {
    enum class op
    {
      push, 
      pop,  
      set,  
    };
 
    op          operation;   
    std::string target {};   
 
    std::size_t target_id {0};
  };
 
  struct rule
  {
    int                        kind;            
    real::regex                pattern;         
    bool                       skip    {false}; 
    std::vector<std::string>   in_mode {};      
    std::optional<mode_action> action  {};      
  };
 
  class lex_error : public std::runtime_error
  {
  public:
 
    lex_error(const std::string& message,
              position           where)
      : std::runtime_error(message),
        where_(where)
    {}
 
    [[nodiscard]] position where() const noexcept
    {
      return where_;
    }
 
  private:
 
    position where_; 
  };
 
  struct frame
  {
    std::size_t mode_id;   
    position    entry_pos; 
  };
 
  inline void apply_transition(const rule&         r,
                               position            start,
                               std::vector<frame>& stack)
  {
    if (!r.action) {
      return;
    }
    if (r.action->operation == mode_action::op::push) {
      stack.push_back(frame {.mode_id = r.action->target_id, .entry_pos = start});
    }
    else if (r.action->operation == mode_action::op::pop) {
      if (stack.size() == 1) {
        throw lex_error("cannot pop the mode stack: already at the root mode", start);
      }
      stack.pop_back();
    }
    else { // set: replace the active mode in place (depth unchanged)
      stack.back().mode_id = r.action->target_id;
    }
  }
 
  enum class error_policy
  {
    raise, 
    token,
  };
 
  enum class column_unit
  {
    bytes,      
    codepoints, 
    utf16,      
  };
 
  class lexer
  {
  public:
 
    explicit lexer(std::vector<rule>               rules,
                   std::unordered_set<std::string> insignificant_modes = {},
                   std::unordered_set<std::string> dfa_modes           = {},
                   error_policy                    errors              = error_policy::raise,
                   column_unit                     columns             = column_unit::bytes)
      : rules_(std::move(rules)),
        errors_(errors),
        columns_(columns)
    {
      build_dispatch();
      build_significance(insignificant_modes);
      build_dfa_modes(dfa_modes);
    }
 
    [[nodiscard]] column_unit columns() const noexcept
    {
      return columns_;
    }
 
    [[nodiscard]] std::vector<token> tokenize(std::string_view source,
                                              eof_policy       policy = eof_policy::omit) const
    {
      std::vector<token> out;
      position           cursor {0, 1, 1};
      std::vector<frame> stack {frame {.mode_id = 0, .entry_pos = cursor}}; // start in "default"
      token              next   {};
      while (scan_next(source, cursor, stack, next)) {
        out.push_back(next);
      }
      if (policy == eof_policy::append) {
        // The cursor now sits at the end position (past any trailing trivia).
        out.push_back(token {end_of_input, source.substr(cursor.offset), cursor});
      }
      return out;
    }
 
    [[nodiscard]] token_range scan(std::string_view source,
                                   eof_policy       policy = eof_policy::omit) const&;
 
    token_range scan(std::string_view source,
                     eof_policy       policy = eof_policy::omit) const&& = delete;
 
    [[nodiscard]] const std::vector<bool>& mode_significant() const noexcept
    {
      return mode_significant_;
    }
 
    [[nodiscard]] const std::string& mode_name(std::size_t id) const noexcept
    {
      return mode_names_[id];
    }
 
    [[nodiscard]] std::vector<std::string> dfa_modes_active() const
    {
      std::vector<std::string> active;
      for (std::size_t m {0}; m < per_mode_dfa_.size(); ++m) {
        if (per_mode_dfa_[m]) {
          active.push_back(mode_names_[m]);
        }
      }
      return active;
    }
 
  private:
 
    friend class token_iterator;
 
    static std::string position_label(position where)
    {
      return std::to_string(where.line) + ":" + std::to_string(where.column);
    }
 
    bool scan_next(std::string_view    source,
                   position&           cursor,
                   std::vector<frame>& stack,
                   token&              out) const
    {
      while (cursor.offset < source.size()) {
        const std::string_view rest {source.substr(cursor.offset)};
        const std::size_t      mode {stack.back().mode_id};
        const munch_result     m    {munch_at(mode, rest, static_cast<unsigned char>(source[cursor.offset]))};
 
        if (!m.have) {
          if (errors_ == error_policy::raise) {
            throw lex_error("no rule matches in mode '" + mode_names_[mode] + "' (entered at "
                            + position_label(stack.back().entry_pos) + ")", cursor); // #1
          }
          // Recovery (error_policy::token): accumulate the maximal run of bytes that no rule in this
          // mode can begin into ONE reserved-kind error token, then resume — no throw, no transition
          // (the run stays in its mode). The run ends at the first position where a rule matches (>0);
          // may_start is an O(1) first-byte pre-filter that skips the bulk of the noise without a full
          // match attempt. The lexeme is the exact offending bytes.
          const position err_start {cursor};
          advance(source, cursor, 1); // the byte at err_start is unmatched by definition
          while (cursor.offset < source.size()
                 && !starts_a_match(mode, source, cursor.offset)) {
            advance(source, cursor, 1);
          }
          out = token {scilex::error, source.substr(err_start.offset, cursor.offset - err_start.offset),
                       err_start, mode};
          return true;
        }
        if (m.len == 0) {
          // A rule won with a zero-length match (a nullable rule and no longer match at this
          // position). Advancing by 0 would spin forever, so report it as a lexical error — fatal
          // under either policy (recovery cannot make progress here). The shared advance point, so
          // both the Pike and DFA paths are covered.
          throw lex_error("zero-length match in mode '" + mode_names_[mode]
                          + "' (rule never advances)", cursor); // #4
        }
 
        const std::size_t best_idx {m.idx};
        const std::size_t best_len {m.len};
        const position    start    {cursor};
        advance(source, cursor, best_len);
 
        apply_transition(rules_[best_idx], start, stack); // advances, then transitions (#2 on a bad pop)
 
        if (!rules_[best_idx].skip) {
          // Tag the token with the mode it was lexed in (captured before the
          // transition above) — Layout Awareness reads it; the scan is untouched.
          out = token {rules_[best_idx].kind, source.substr(start.offset, best_len), start, mode};
          return true;
        }
        // Skip rule: keep scanning for the next emitted token (possibly in a new mode).
      }
      if (stack.size() > 1) {
        if (errors_ == error_policy::raise) {
          throw lex_error("unterminated mode '" + mode_names_[stack.back().mode_id] + "' (entered at "
                          + position_label(stack.back().entry_pos) + ")", stack.back().entry_pos); // #3
        }
        // Recovery (error_policy::token): a mode was still pushed at end of input. Emit one zero-width
        // error token positioned at the EOF (the partial tokens already emitted stay), then unwind to
        // the root so the next call reports a clean end of input.
        out = token {scilex::error, source.substr(cursor.offset, 0), cursor, stack.back().mode_id};
        stack.resize(1);
        return true;
      }
      return false;
    }
 
    std::size_t intern_mode(const std::string& name)
    {
      const auto [it, inserted] {mode_id_.emplace(name, mode_names_.size())};
      if (inserted) {
        mode_names_.push_back(name);
      }
      return it->second;
    }
 
    void add_to_mode(std::size_t m,
                     std::size_t idx)
    {
      const real::regex& pattern {rules_[idx].pattern};
      dispatch&          target  {per_mode_[m]};
      if (!pattern.has_first_byte_set()) {
        target.general.push_back(idx);
      }
      else if (const std::optional<unsigned char> byte {pattern.unique_first_byte()}) {
        target.first_byte_index[*byte].push_back(idx);
      }
      else {
        for (int candidate {0}; candidate < 256; ++candidate) {
          if (pattern.may_start_with(static_cast<unsigned char>(candidate))) {
            target.first_byte_index[static_cast<unsigned char>(candidate)].push_back(idx);
          }
        }
      }
    }
 
    [[nodiscard]] bool mode_is_empty(std::size_t m) const
    {
      const dispatch& d {per_mode_[m]};
      return d.general.empty()
             && std::all_of(d.first_byte_index.begin(), d.first_byte_index.end(),
                            [](const std::vector<std::size_t>& bucket) { return bucket.empty(); });
    }
 
    void validate_transitions() const
    {
      for (const rule& candidate : rules_) {
        if (!candidate.action) {
          continue;
        }
        if (candidate.pattern.pattern().empty()) {
          throw std::invalid_argument("a transition rule must consume input (empty pattern)");
        }
        if (candidate.action->operation != mode_action::op::pop
            && mode_is_empty(mode_id_.at(candidate.action->target))) {
          throw std::invalid_argument("a transition targets the empty mode '"
                                      + candidate.action->target + "' (no rule is active in it)");
        }
      }
    }
 
    void build_dispatch()
    {
      intern_mode("default"); // mode 0, always present
      for (const rule& candidate : rules_) {
        for (const std::string& name : candidate.in_mode) {
          intern_mode(name);
        }
        if (const std::optional<mode_action> action {candidate.action};
            action.has_value() && action->operation != mode_action::op::pop) {
          intern_mode(action->target);
        }
      }
      per_mode_.resize(mode_names_.size());
 
      for (std::size_t idx {0}; idx < rules_.size(); ++idx) {
        if (rules_[idx].in_mode.empty()) {
          add_to_mode(0, idx); // an undeclared rule is active in "default" only
        }
        else {
          for (const std::string& name : rules_[idx].in_mode) {
            add_to_mode(mode_id_.at(name), idx);
          }
        }
      }
      validate_transitions();
 
      // Pre-resolve each transition's target mode id once, now that every mode is
      // interned and validated, so the per-token apply_transition reads a field instead
      // of a name→id map lookup. The target string stays for diagnostics; pop has none.
      for (rule& candidate : rules_) {
        if (candidate.action && candidate.action->operation != mode_action::op::pop) {
          candidate.action->target_id = mode_id_.at(candidate.action->target);
        }
      }
    }
 
    void build_significance(const std::unordered_set<std::string>& insignificant_modes)
    {
      if (insignificant_modes.empty()) {
        return;
      }
      mode_significant_.assign(mode_names_.size(), true);
      for (const std::string& name : insignificant_modes) {
        const auto found {mode_id_.find(name)};
        if (found == mode_id_.end()) {
          throw std::invalid_argument("insignificant_modes names an unknown mode: " + name);
        }
        mode_significant_[found->second] = false;
      }
    }
 
    struct mode_dfa
    {
      real::dfa                dfa;       
      std::vector<std::size_t> to_global; 
    };
 
    struct munch_result
    {
      bool        have {false};
      std::size_t idx  {0};
      std::size_t len  {0};
    };
 
    munch_result munch_at(std::size_t      mode,
                          std::string_view rest,
                          unsigned char    lead) const
    {
      if (per_mode_dfa_[mode]) {
        if (const std::optional<real::dfa_match> matched {per_mode_dfa_[mode]->dfa.match(rest)}) {
          return munch_result {.have = true, .idx = per_mode_dfa_[mode]->to_global[matched->rule_index],
                               .len = matched->length};
        }
        return munch_result {};
      }
      return pike_munch_in_mode(mode, rest, lead);
    }
 
    bool may_start(std::size_t   mode,
                   unsigned char byte) const
    {
      return !per_mode_[mode].first_byte_index[byte].empty();
    }
 
    bool starts_a_match(std::size_t      mode,
                        std::string_view source,
                        std::size_t      offset) const
    {
      const unsigned char lead {static_cast<unsigned char>(source[offset])};
      if (!may_start(mode, lead)) {
        return false;
      }
      return munch_at(mode, source.substr(offset), lead).have;
    }
 
    void advance(std::string_view source,
                 position&        cursor,
                 std::size_t      n) const
    {
      for (std::size_t i {0}; i < n; ++i) {
        if (source[cursor.offset] == '\n') {
          ++cursor.line;
          cursor.column = 1;
        }
        else {
          cursor.column += column_step(source, cursor.offset, columns_);
        }
        ++cursor.offset;
      }
    }
 
    static std::size_t valid_utf8_len(std::string_view s,
                                      std::size_t      off)
    {
      const unsigned char b0  {static_cast<unsigned char>(s[off])};
      std::size_t         len {0};
      unsigned int        cp  {0};
      if (b0 < 0x80U) {
        return 1; // ASCII
      }
      if ((b0 & 0xE0U) == 0xC0U) {
        len = 2;
        cp  = b0 & 0x1FU;
      }
      else if ((b0 & 0xF0U) == 0xE0U) {
        len = 3;
        cp  = b0 & 0x0FU;
      }
      else if ((b0 & 0xF8U) == 0xF0U) {
        len = 4;
        cp  = b0 & 0x07U;
      }
      else {
        return 0; // a continuation byte (0x80–0xBF) or an invalid lead (0xF8–0xFF)
      }
      if (off + len > s.size()) {
        return 0; // truncated
      }
      for (std::size_t i {1}; i < len; ++i) {
        const unsigned char bi {static_cast<unsigned char>(s[off + i])};
        if ((bi & 0xC0U) != 0x80U) {
          return 0; // a missing continuation
        }
        cp = (cp << 6U) | (bi & 0x3FU);
      }
      static constexpr unsigned int min_for_len[5] {0, 0, 0x80U, 0x800U, 0x10000U};
      if (cp < min_for_len[len] || (cp >= 0xD800U && cp <= 0xDFFFU) || cp > 0x10FFFFU) {
        return 0; // overlong, a UTF-16 surrogate, or beyond U+10FFFF
      }
      return len;
    }
 
    static std::size_t column_step(std::string_view        source,
                                   std::size_t             off,
                                   scilex::column_unit     unit)
    {
      if (unit == scilex::column_unit::bytes) {
        return 1;
      }
      const unsigned char byte {static_cast<unsigned char>(source[off])};
      if ((byte & 0xC0U) == 0x80U) { // a continuation byte
        // Score 0 only if it belongs to a valid codepoint whose lead is 1–3 bytes back; an orphan
        // continuation is malformed and scores 1. (A codepoint never spans a newline, so this
        // fixed look-back cannot cross a line boundary in a way that matters.)
        for (std::size_t back {1}; back <= 3 && back <= off; ++back) {
          if (valid_utf8_len(source, off - back) > back) {
            return 0;
          }
        }
        return 1;
      }
      if (unit == scilex::column_unit::utf16) {
        return valid_utf8_len(source, off) == 4 ? 2 : 1; // an astral codepoint is a surrogate pair
      }
      return 1; // codepoints: an ASCII byte or a lead (its continuations already scored 0)
    }
 
    munch_result pike_munch_in_mode(std::size_t      mode,
                                    std::string_view rest,
                                    unsigned char    lead) const
    {
      std::size_t best_len {0};
      std::size_t best_idx {0};
      bool        have     {false};
      const auto  consider {[&](std::size_t idx) {
                              // idx comes from this mode's first-byte dispatch, populated
                              // in build_dispatch() from rules_ indices, so it is always in
                              // range. The analyzer cannot prove that cross-vector invariant
                              // once this munch is a standalone shared method; a bounds guard
                              // would be an unreachable branch the 100%-4D gate rejects, so the
                              // proven false positive is suppressed here (see REPORT note).
                              // NOLINTNEXTLINE(clang-analyzer-core.NonNullParamChecker)
                              const auto matched {rules_[idx].pattern.match(rest)};
                              // A zero-length match participates (it can only win when no rule
                              // matches >0 here); the shared guard in scan_next turns that win
                              // into a lexical error rather than a stalled scan. Maximal munch
                              // still prefers any longer non-empty match.
                              if (matched
                                  && (!have || matched.end() > best_len
                                      || (matched.end() == best_len && idx < best_idx))) {
                                best_len = matched.end();
                                best_idx = idx;
                                have     = true;
                              }
                            }};
      const dispatch& active {per_mode_[mode]};
      for (const std::size_t idx : active.first_byte_index[lead]) {
        consider(idx);
      }
      for (const std::size_t idx : active.general) {
        consider(idx);
      }
      return {.have = have, .idx = best_idx, .len = best_len};
    }
 
    [[nodiscard]] bool rule_active_in_mode(std::size_t idx,
                                           std::size_t mode) const
    {
      const std::vector<std::string>& modes {rules_[idx].in_mode};
      if (modes.empty()) {
        return mode == 0;
      }
      for (const std::string& name : modes) {
        if (mode_id_.at(name) == mode) {
          return true;
        }
      }
      return false;
    }
 
    std::vector<std::string> audit_probes(const std::vector<std::size_t>& to_global) const
    {
      std::array<bool, 256>      seen {};
      std::vector<unsigned char> alpha;
      const auto                 add {[&](unsigned char b) {
                                        if (!seen[b]) {
                                          seen[b] = true;
                                          alpha.push_back(b);
                                        }
                                      }};
      for (const std::size_t g : to_global) {
        for (int b {0}; b < 256; ++b) {
          if (rules_[g].pattern.may_start_with(static_cast<unsigned char>(b))) {
            add(static_cast<unsigned char>(b));
          }
        }
      }
      for (const char structural : std::string_view {" \t\n\"'/*-+=<>()[]{};.:,aAz09_"}) {
        add(static_cast<unsigned char>(structural));
      }
 
      // alpha is always non-empty (the structural bytes above are unconditional), so
      // the probe count is O(alphabet) + a fixed random batch — deterministic, bounded,
      // and free of cap branches. Singletons + short repeats expose lazy delimiters and
      // quantifier boundaries (the hard cases); the random batch broadens coverage.
      std::vector<std::string> probes;
      for (const unsigned char b : alpha) {
        for (const std::size_t n : std::array<std::size_t, 5> {1, 2, 3, 6, 8}) {
          probes.emplace_back(n, static_cast<char>(b));
        }
      }
      // Fixed seed by design: this RNG only generates local probe strings for the
      // build-time DFA equivalence audit, which must be reproducible. No security
      // role (no tokens, crypto or identifiers) — a constant seed is correct here.
      // NOLINTNEXTLINE(bugprone-random-generator-seed,cert-msc32-c,cert-msc51-cpp)
      std::mt19937                               rng   {0x5C11EFU}; // fixed seed: the audit is reproducible
      std::uniform_int_distribution<std::size_t> len_d {1, 48};
      std::uniform_int_distribution<std::size_t> sym_d {0, alpha.size() - 1};
      for (int batch {0}; batch < 256; ++batch) {
        std::string       input;
        const std::size_t len {len_d(rng)};
        for (std::size_t i {0}; i < len; ++i) {
          input.push_back(static_cast<char>(alpha[sym_d(rng)]));
        }
        probes.push_back(std::move(input));
      }
      return probes;
    }
 
    [[nodiscard]] bool audit_passes(const real::dfa&                candidate,
                                    const std::vector<std::size_t>& to_global,
                                    std::size_t                     mode) const
    {
      const std::vector<std::string> probes {audit_probes(to_global)};
      for (const std::string& probe : probes) {
        const std::string_view               rest    {probe}; // probes always have length >= 1
        const std::optional<real::dfa_match> hit     {candidate.match(rest)};
        const munch_result                   pike    {pike_munch_in_mode(mode, rest, static_cast<unsigned char>(rest[0]))};
        std::size_t                          dfa_idx {0};
        std::size_t                          dfa_len {0};
        if (hit.has_value()) {
          dfa_idx = to_global[hit->rule_index];
          dfa_len = hit->length;
        }
        // One comparison — the tuple's element-wise short-circuit lives in <tuple>, not
        // here — so any divergence (chiefly a lazy rule's shortest-vs-longest) rejects.
        if (std::tuple {hit.has_value(), dfa_idx, dfa_len} != std::tuple {pike.have, pike.idx, pike.len}) {
          return false;
        }
      }
      return true;
    }
 
    std::optional<mode_dfa> try_build_mode_dfa(std::vector<std::size_t> to_global,
                                               std::size_t              mode)
    {
      std::vector<real::detail::program_view> programs;
      programs.reserve(to_global.size());
      for (const std::size_t g : to_global) {
        programs.push_back(rules_[g].pattern.raw_program());
      }
      try {
        real::dfa candidate {std::span<const real::detail::program_view>(programs)};
        if (!audit_passes(candidate, to_global, mode)) {
          return std::nullopt; // a divergence (e.g. a lazy rule) → keep this mode on Pike
        }
        return mode_dfa {.dfa = std::move(candidate), .to_global = std::move(to_global)};
      }
      catch (const real::dfa_error&) {
        return std::nullopt; // un-DFA-able assertion ($, \b, multiline ^/$): keep on Pike
      }
    }
 
    void build_dfa_modes(const std::unordered_set<std::string>& dfa_modes)
    {
      per_mode_dfa_.assign(mode_names_.size(), nullptr);
      for (const std::string& name : dfa_modes) {
        const auto found {mode_id_.find(name)};
        if (found == mode_id_.end()) {
          throw std::invalid_argument("dfa_modes names an unknown mode: " + name);
        }
        const std::size_t        mode {found->second};
        std::vector<std::size_t> to_global;
        for (std::size_t idx {0}; idx < rules_.size(); ++idx) {
          if (rule_active_in_mode(idx, mode)) {
            to_global.push_back(idx);
          }
        }
        if (auto built {try_build_mode_dfa(std::move(to_global), mode)}) {
          per_mode_dfa_[mode] = std::make_shared<const mode_dfa>(std::move(*built));
        }
      }
    }
 
    struct dispatch
    {
      std::array<std::vector<std::size_t>, 256> first_byte_index; 
      std::vector<std::size_t>                  general;          
    };
 
    std::vector<rule>                            rules_;            
    error_policy                                 errors_;           
    scilex::column_unit                          columns_;          
    std::vector<std::string>                     mode_names_;       
    std::map<std::string, std::size_t>           mode_id_;          
    std::vector<dispatch>                        per_mode_;         
    std::vector<std::shared_ptr<const mode_dfa>> per_mode_dfa_;     
    std::vector<bool>                            mode_significant_; 
  };
 
  class token_iterator
  {
  public:
 
    using iterator_category = std::input_iterator_tag;     
    using value_type        = token;                       
    using difference_type   = std::ptrdiff_t;              
    using pointer           = const token*;                
    using reference         = const token&;                
 
    token_iterator() = default;
 
    token_iterator(const lexer&     owner,
                   std::string_view source,
                   eof_policy       policy)
      : owner_(&owner),
        source_(source),
        policy_(policy),
        done_(false)
    {
      advance();
    }
 
    reference operator*() const
    {
      return current_;
    }
 
    pointer operator->() const
    {
      return &current_;
    }
 
    token_iterator& operator++()
    {
      advance();
      return *this;
    }
 
    void operator++(int)
    {
      advance();
    }
 
    [[nodiscard]] bool operator==(const token_iterator& other) const
    {
      return done_ == other.done_ && (done_ || cursor_.offset == other.cursor_.offset);
    }
 
    [[nodiscard]] bool operator!=(const token_iterator& other) const
    {
      return !(*this == other);
    }
 
  private:
 
    const lexer*       owner_  {nullptr};                                               
    std::string_view   source_;                                                         
    position           cursor_ {0, 1, 1};                                               
    std::vector<frame> stack_  {frame {.mode_id = 0, .entry_pos = position {0, 1, 1}}}; 
    token              current_  {};                                                    
    eof_policy         policy_   {eof_policy::omit};                                    
    bool               eof_done_ {false};                                               
    bool               done_     {true};                                                
 
    void advance()
    {
      if (done_) {
        return;
      }
      if (owner_->scan_next(source_, cursor_, stack_, current_)) {
        return;
      }
      // Input exhausted: yield one end-of-input token if requested, else stop.
      if (policy_ == eof_policy::append && !eof_done_) {
        current_  = token {end_of_input, source_.substr(cursor_.offset), cursor_};
        eof_done_ = true;
        return;
      }
      done_ = true;
    }
  };
 
  class token_range
  {
  public:
 
    token_range(const lexer&     owner,
                std::string_view source,
                eof_policy       policy)
      : owner_(&owner),
        source_(source),
        policy_(policy)
    {}
 
    [[nodiscard]] token_iterator begin() const
    {
      return token_iterator(*owner_, source_, policy_);
    }
 
    [[nodiscard]] token_iterator end() const
    {
      return token_iterator();
    }
 
  private:
 
    const lexer*     owner_  {nullptr};          
    std::string_view source_;                    
    eof_policy       policy_ {eof_policy::omit}; 
  };
 
  inline token_range lexer::scan(std::string_view source,
                                 eof_policy       policy) const&
  {
    return token_range(*this, source, policy);
  }
} // namespace scilex
 
#endif // SCILEX_LEXER_HPP