Struct Tokenizer 

struct Tokenizer<'input, 'db> {
    db: &'db dyn Db,
    anchor: Anchor<'db>,
    input: &'input str,
    chars: Peekable<CharIndices<'input>>,
    tokens: Vec<Token<'input, 'db>>,
    kws: &'static Map<String, Keyword>,
    input_offset: Offset,
    error_start: Option<usize>,
    skipped_accum: Option<Skipped>,
}

Fields

db: &'db dyn Db

anchor: Anchor<'db>

input: &'input str

chars: Peekable<CharIndices<'input>>

tokens: Vec<Token<'input, 'db>>

kws: &'static Map<String, Keyword>

input_offset: Offset

error_start: Option<usize>

skipped_accum: Option<Skipped>

Implementations

impl<'input, 'db> Tokenizer<'input, 'db>

fn tokenize(self) -> Vec<Token<'input, 'db>>

fn accumulate_skipped(&mut self, skipped: Skipped)

fn clear_accumulated(&mut self, index: usize) -> Option<Skipped>

Clears various accumulated state in prep for a new token being issued (or the final token). Returns the Skipped value that should be used for the next token issued (if any). Reports errors for any invalid characters seen thus far.

fn span(&self, start: usize, end: usize) -> Span<'db>

fn comment(&mut self, index: usize)

fn identifier(&mut self, start: usize, ch: char)

fn integer(&mut self, start: usize, ch: char)

fn escape_sequence(&mut self, backslash_offset: usize, content: &mut String)

Process an escape sequence after consuming \. backslash_offset is the byte index of the \ character.

fn emit_string_literal( &mut self, span: Span<'db>, skipped: Option<Skipped>, content: String, quote_len: usize, raw: bool, )

Emit a string literal token with the given span and processed content.

quote_len is the number of quote characters in the delimiter (1 for ", 3 for """). When the raw source content begins with a newline, multiline dedenting is applied: the raw content is dedented and escape sequences are re-processed on the result, replacing the content that was built during scanning.

fn emit_unterminated_string( &mut self, start: usize, skipped: Option<Skipped>, content: String, message: &str, )

Emit tokens for an unterminated string literal: a literal token with whatever content was accumulated, plus an error token. No multiline dedenting is applied since the string is malformed.

fn string_literal(&mut self, start: usize)

fn triple_quoted_string_literal( &mut self, start: usize, skipped: Option<Skipped>, )

Lex a triple-quoted string literal. Called after the opening """ has been consumed. Scans until the closing """ is found.

fn delimited(&mut self, start: usize, delim: Delimiter, close: char)

fn ops(&mut self, start: usize, ch: char)