# File lib/ritex/lexer.rb, line 44
  def lex #:yields: token, value
    ## actually this function does nothing right now except call
    ## lex_inner. if we switch to more stateful tokenization this
    ## might do something more.
    lex_inner do |sym, val|
      yield [sym, val]
    end
  end