/rust/registry/src/index.crates.io-6f17d22bba15001f/toml-0.5.9/src/tokens.rs

Source (jump to first uncovered line)
use std::borrow::Cow;
use std::char;
use std::str;
use std::string;
use std::string::String as StdString;

use self::Token::*;

/// A span, designating a range of bytes where a token is located.
#[derive(Eq, PartialEq, Debug, Clone, Copy)]
pub struct Span {
    /// The start of the range.
    pub start: usize,
    /// The end of the range (exclusive).
    pub end: usize,
}

impl From<Span> for (usize, usize) {
    fn from(Span { start, end }: Span) -> (usize, usize) {
        (start, end)
    }
}

#[derive(Eq, PartialEq, Debug)]
pub enum Token<'a> {
    Whitespace(&'a str),
    Newline,
    Comment(&'a str),

    Equals,
    Period,
    Comma,
    Colon,
    Plus,
    LeftBrace,
    RightBrace,
    LeftBracket,
    RightBracket,

    Keylike(&'a str),
    String {
        src: &'a str,
        val: Cow<'a, str>,
        multiline: bool,
    },
}

#[derive(Eq, PartialEq, Debug)]
pub enum Error {
    InvalidCharInString(usize, char),
    InvalidEscape(usize, char),
    InvalidHexEscape(usize, char),
    InvalidEscapeValue(usize, u32),
    NewlineInString(usize),
    Unexpected(usize, char),
    UnterminatedString(usize),
    NewlineInTableKey(usize),
    MultilineStringKey(usize),
    Wanted {
        at: usize,
        expected: &'static str,
        found: &'static str,
    },
}

#[derive(Clone)]
pub struct Tokenizer<'a> {
    input: &'a str,
    chars: CrlfFold<'a>,
}

#[derive(Clone)]
struct CrlfFold<'a> {
    chars: str::CharIndices<'a>,
}

#[derive(Debug)]
enum MaybeString {
    NotEscaped(usize),
    Owned(string::String),
}

impl<'a> Tokenizer<'a> {
    pub fn new(input: &'a str) -> Tokenizer<'a> {
        let mut t = Tokenizer {
            input,
            chars: CrlfFold {
                chars: input.char_indices(),
            },
        };
        // Eat utf-8 BOM
        t.eatc('\u{feff}');
        t
    }

    pub fn next(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> {
        let (start, token) = match self.one() {
            Some((start, '\n')) => (start, Newline),
            Some((start, ' ')) => (start, self.whitespace_token(start)),
            Some((start, '\t')) => (start, self.whitespace_token(start)),
            Some((start, '#')) => (start, self.comment_token(start)),
            Some((start, '=')) => (start, Equals),
            Some((start, '.')) => (start, Period),
            Some((start, ',')) => (start, Comma),
            Some((start, ':')) => (start, Colon),
            Some((start, '+')) => (start, Plus),
            Some((start, '{')) => (start, LeftBrace),
            Some((start, '}')) => (start, RightBrace),
            Some((start, '[')) => (start, LeftBracket),
            Some((start, ']')) => (start, RightBracket),
            Some((start, '\'')) => {
                return self
                    .literal_string(start)
                    .map(|t| Some((self.step_span(start), t)))
            }
            Some((start, '"')) => {
                return self
                    .basic_string(start)
                    .map(|t| Some((self.step_span(start), t)))
            }
            Some((start, ch)) if is_keylike(ch) => (start, self.keylike(start)),

            Some((start, ch)) => return Err(Error::Unexpected(start, ch)),
            None => return Ok(None),
        };

        let span = self.step_span(start);
        Ok(Some((span, token)))
    }

    pub fn peek(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> {
        self.clone().next()
    }

    pub fn eat(&mut self, expected: Token<'a>) -> Result<bool, Error> {
        self.eat_spanned(expected).map(|s| s.is_some())
    }

    /// Eat a value, returning it's span if it was consumed.
    pub fn eat_spanned(&mut self, expected: Token<'a>) -> Result<Option<Span>, Error> {
        let span = match self.peek()? {
            Some((span, ref found)) if expected == *found => span,
            Some(_) => return Ok(None),
            None => return Ok(None),
        };

        drop(self.next());
        Ok(Some(span))
    }

    pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> {
        // ignore span
        let _ = self.expect_spanned(expected)?;
        Ok(())
    }

    /// Expect the given token returning its span.
    pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error> {
        let current = self.current();
        match self.next()? {
            Some((span, found)) => {
                if expected == found {
                    Ok(span)
                } else {
                    Err(Error::Wanted {
                        at: current,
                        expected: expected.describe(),
                        found: found.describe(),
                    })
                }
            }
            None => Err(Error::Wanted {
                at: self.input.len(),
                expected: expected.describe(),
                found: "eof",
            }),
        }
    }

    pub fn table_key(&mut self) -> Result<(Span, Cow<'a, str>), Error> {
        let current = self.current();
        match self.next()? {
            Some((span, Token::Keylike(k))) => Ok((span, k.into())),
            Some((
                span,
                Token::String {
                    src,
                    val,
                    multiline,
                },
            )) => {
                let offset = self.substr_offset(src);
                if multiline {
                    return Err(Error::MultilineStringKey(offset));
                }
                match src.find('\n') {
                    None => Ok((span, val)),
                    Some(i) => Err(Error::NewlineInTableKey(offset + i)),
                }
            }
            Some((_, other)) => Err(Error::Wanted {
                at: current,
                expected: "a table key",
                found: other.describe(),
            }),
            None => Err(Error::Wanted {
                at: self.input.len(),
                expected: "a table key",
                found: "eof",
            }),
        }
    }

    pub fn eat_whitespace(&mut self) -> Result<(), Error> {
        while self.eatc(' ') || self.eatc('\t') {
            // ...
        }
        Ok(())
    }

    pub fn eat_comment(&mut self) -> Result<bool, Error> {
        if !self.eatc('#') {
            return Ok(false);
        }
        drop(self.comment_token(0));
        self.eat_newline_or_eof().map(|()| true)
    }

    pub fn eat_newline_or_eof(&mut self) -> Result<(), Error> {
        let current = self.current();
        match self.next()? {
            None | Some((_, Token::Newline)) => Ok(()),
            Some((_, other)) => Err(Error::Wanted {
                at: current,
                expected: "newline",
                found: other.describe(),
            }),
        }
    }

    pub fn skip_to_newline(&mut self) {
        loop {
            match self.one() {
                Some((_, '\n')) | None => break,
                _ => {}
            }
        }
    }

    fn eatc(&mut self, ch: char) -> bool {
        match self.chars.clone().next() {
            Some((_, ch2)) if ch == ch2 => {
                self.one();
                true
            }
            _ => false,
        }
    }

    pub fn current(&mut self) -> usize {
        self.chars
            .clone()
            .next()
            .map(|i| i.0)
            .unwrap_or_else(|| self.input.len())
    }

    pub fn input(&self) -> &'a str {
        self.input
    }

    fn whitespace_token(&mut self, start: usize) -> Token<'a> {
        while self.eatc(' ') || self.eatc('\t') {
            // ...
        }
        Whitespace(&self.input[start..self.current()])
    }

    fn comment_token(&mut self, start: usize) -> Token<'a> {
        while let Some((_, ch)) = self.chars.clone().next() {
            if ch != '\t' && (ch < '\u{20}' || ch > '\u{10ffff}') {
                break;
            }
            self.one();
        }
        Comment(&self.input[start..self.current()])
    }

    fn read_string(
        &mut self,
        delim: char,
        start: usize,
        new_ch: &mut dyn FnMut(
            &mut Tokenizer<'_>,
            &mut MaybeString,
            bool,
            usize,
            char,
        ) -> Result<(), Error>,
    ) -> Result<Token<'a>, Error> {
        let mut multiline = false;
        if self.eatc(delim) {
            if self.eatc(delim) {
                multiline = true;
            } else {
                return Ok(String {
                    src: &self.input[start..start + 2],
                    val: Cow::Borrowed(""),
                    multiline: false,
                });
            }
        }
        let mut val = MaybeString::NotEscaped(self.current());
        let mut n = 0;
        'outer: loop {
            n += 1;
            match self.one() {
                Some((i, '\n')) => {
                    if multiline {
                        if self.input.as_bytes()[i] == b'\r' {
                            val.to_owned(&self.input[..i]);
                        }
                        if n == 1 {
                            val = MaybeString::NotEscaped(self.current());
                        } else {
                            val.push('\n');
                        }
                        continue;
                    } else {
                        return Err(Error::NewlineInString(i));
                    }
                }
                Some((mut i, ch)) if ch == delim => {
                    if multiline {
                        if !self.eatc(delim) {
                            val.push(delim);
                            continue 'outer;
                        }
                        if !self.eatc(delim) {
                            val.push(delim);
                            val.push(delim);
                            continue 'outer;
                        }
                        if self.eatc(delim) {
                            val.push(delim);
                            i += 1;
                        }
                        if self.eatc(delim) {
                            val.push(delim);
                            i += 1;
                        }
                    }
                    return Ok(String {
                        src: &self.input[start..self.current()],
                        val: val.into_cow(&self.input[..i]),
                        multiline,
                    });
                }
                Some((i, c)) => new_ch(self, &mut val, multiline, i, c)?,
                None => return Err(Error::UnterminatedString(start)),
            }
        }
    }

    fn literal_string(&mut self, start: usize) -> Result<Token<'a>, Error> {
        self.read_string('\'', start, &mut |_me, val, _multi, i, ch| {
            if ch == '\u{09}' || ('\u{20}' <= ch && ch <= '\u{10ffff}' && ch != '\u{7f}') {
                val.push(ch);
                Ok(())
            } else {
                Err(Error::InvalidCharInString(i, ch))
            }
        })
    }

    fn basic_string(&mut self, start: usize) -> Result<Token<'a>, Error> {
        self.read_string('"', start, &mut |me, val, multi, i, ch| match ch {
            '\\' => {
                val.to_owned(&me.input[..i]);
                match me.chars.next() {
                    Some((_, '"')) => val.push('"'),
                    Some((_, '\\')) => val.push('\\'),
                    Some((_, 'b')) => val.push('\u{8}'),
                    Some((_, 'f')) => val.push('\u{c}'),
                    Some((_, 'n')) => val.push('\n'),
                    Some((_, 'r')) => val.push('\r'),
                    Some((_, 't')) => val.push('\t'),
                    Some((i, c @ 'u')) | Some((i, c @ 'U')) => {
                        let len = if c == 'u' { 4 } else { 8 };
                        val.push(me.hex(start, i, len)?);
                    }
                    Some((i, c @ ' ')) | Some((i, c @ '\t')) | Some((i, c @ '\n')) if multi => {
                        if c != '\n' {
                            while let Some((_, ch)) = me.chars.clone().next() {
                                match ch {
                                    ' ' | '\t' => {
                                        me.chars.next();
                                        continue;
                                    }
                                    '\n' => {
                                        me.chars.next();
                                        break;
                                    }
                                    _ => return Err(Error::InvalidEscape(i, c)),
                                }
                            }
                        }
                        while let Some((_, ch)) = me.chars.clone().next() {
                            match ch {
                                ' ' | '\t' | '\n' => {
                                    me.chars.next();
                                }
                                _ => break,
                            }
                        }
                    }
                    Some((i, c)) => return Err(Error::InvalidEscape(i, c)),
                    None => return Err(Error::UnterminatedString(start)),
                }
                Ok(())
            }
            ch if ch == '\u{09}' || ('\u{20}' <= ch && ch <= '\u{10ffff}' && ch != '\u{7f}') => {
                val.push(ch);
                Ok(())
            }
            _ => Err(Error::InvalidCharInString(i, ch)),
        })
    }

    fn hex(&mut self, start: usize, i: usize, len: usize) -> Result<char, Error> {
        let mut buf = StdString::with_capacity(len);
        for _ in 0..len {
            match self.one() {
                Some((_, ch)) if ch as u32 <= 0x7F && ch.is_digit(16) => buf.push(ch),
                Some((i, ch)) => return Err(Error::InvalidHexEscape(i, ch)),
                None => return Err(Error::UnterminatedString(start)),
            }
        }
        let val = u32::from_str_radix(&buf, 16).unwrap();
        match char::from_u32(val) {
            Some(ch) => Ok(ch),
            None => Err(Error::InvalidEscapeValue(i, val)),
        }
    }

    fn keylike(&mut self, start: usize) -> Token<'a> {
        while let Some((_, ch)) = self.peek_one() {
            if !is_keylike(ch) {
                break;
            }
            self.one();
        }
        Keylike(&self.input[start..self.current()])
    }

    pub fn substr_offset(&self, s: &'a str) -> usize {
        assert!(s.len() <= self.input.len());
        let a = self.input.as_ptr() as usize;
        let b = s.as_ptr() as usize;
        assert!(a <= b);
        b - a
    }

    /// Calculate the span of a single character.
    fn step_span(&mut self, start: usize) -> Span {
        let end = self
            .peek_one()
            .map(|t| t.0)
            .unwrap_or_else(|| self.input.len());
        Span { start, end }
    }

    /// Peek one char without consuming it.
    fn peek_one(&mut self) -> Option<(usize, char)> {
        self.chars.clone().next()
    }

    /// Take one char.
    pub fn one(&mut self) -> Option<(usize, char)> {
        self.chars.next()
    }
}

impl<'a> Iterator for CrlfFold<'a> {
    type Item = (usize, char);

    fn next(&mut self) -> Option<(usize, char)> {
        self.chars.next().map(|(i, c)| {
            if c == '\r' {
                let mut attempt = self.chars.clone();
                if let Some((_, '\n')) = attempt.next() {
                    self.chars = attempt;
                    return (i, '\n');
                }
            }
            (i, c)
        })
    }
}

impl MaybeString {
    fn push(&mut self, ch: char) {
        match *self {
            MaybeString::NotEscaped(..) => {}
            MaybeString::Owned(ref mut s) => s.push(ch),
        }
    }

    fn to_owned(&mut self, input: &str) {
        match *self {
            MaybeString::NotEscaped(start) => {
                *self = MaybeString::Owned(input[start..].to_owned());
            }
            MaybeString::Owned(..) => {}
        }
    }

    fn into_cow(self, input: &str) -> Cow<'_, str> {
        match self {
            MaybeString::NotEscaped(start) => Cow::Borrowed(&input[start..]),
            MaybeString::Owned(s) => Cow::Owned(s),
        }
    }
}

fn is_keylike(ch: char) -> bool {
    ('A' <= ch && ch <= 'Z')
        || ('a' <= ch && ch <= 'z')
        || ('0' <= ch && ch <= '9')
        || ch == '-'
        || ch == '_'
}

impl<'a> Token<'a> {
    pub fn describe(&self) -> &'static str {
        match *self {
            Token::Keylike(_) => "an identifier",
            Token::Equals => "an equals",
            Token::Period => "a period",
            Token::Comment(_) => "a comment",
            Token::Newline => "a newline",
            Token::Whitespace(_) => "whitespace",
            Token::Comma => "a comma",
            Token::RightBrace => "a right brace",
            Token::LeftBrace => "a left brace",
            Token::RightBracket => "a right bracket",
            Token::LeftBracket => "a left bracket",
            Token::String { multiline, .. } => {
                if multiline {
                    "a multiline string"
                } else {
                    "a string"
                }
            }
            Token::Colon => "a colon",
            Token::Plus => "a plus",
        }
    }
}

#[cfg(test)]
mod tests {
    use super::{Error, Token, Tokenizer};
    use std::borrow::Cow;

    fn err(input: &str, err: Error) {
        let mut t = Tokenizer::new(input);
        let token = t.next().unwrap_err();
        assert_eq!(token, err);
        assert!(t.next().unwrap().is_none());
    }

    #[test]
    fn literal_strings() {
        fn t(input: &str, val: &str, multiline: bool) {
            let mut t = Tokenizer::new(input);
            let (_, token) = t.next().unwrap().unwrap();
            assert_eq!(
                token,
                Token::String {
                    src: input,
                    val: Cow::Borrowed(val),
                    multiline: multiline,
                }
            );
            assert!(t.next().unwrap().is_none());
        }

        t("''", "", false);
        t("''''''", "", true);
        t("'''\n'''", "", true);
        t("'a'", "a", false);
        t("'\"a'", "\"a", false);
        t("''''a'''", "'a", true);
        t("'''\n'a\n'''", "'a\n", true);
        t("'''a\n'a\r\n'''", "a\n'a\n", true);
    }

    #[test]
    fn basic_strings() {
        fn t(input: &str, val: &str, multiline: bool) {
            let mut t = Tokenizer::new(input);
            let (_, token) = t.next().unwrap().unwrap();
            assert_eq!(
                token,
                Token::String {
                    src: input,
                    val: Cow::Borrowed(val),
                    multiline: multiline,
                }
            );
            assert!(t.next().unwrap().is_none());
        }

        t(r#""""#, "", false);
        t(r#""""""""#, "", true);
        t(r#""a""#, "a", false);
        t(r#""""a""""#, "a", true);
        t(r#""\t""#, "\t", false);
        t(r#""\u0000""#, "\0", false);
        t(r#""\U00000000""#, "\0", false);
        t(r#""\U000A0000""#, "\u{A0000}", false);
        t(r#""\\t""#, "\\t", false);
        t("\"\t\"", "\t", false);
        t("\"\"\"\n\t\"\"\"", "\t", true);
        t("\"\"\"\\\n\"\"\"", "", true);
        t(
            "\"\"\"\\\n     \t   \t  \\\r\n  \t \n  \t \r\n\"\"\"",
            "",
            true,
        );
        t(r#""\r""#, "\r", false);
        t(r#""\n""#, "\n", false);
        t(r#""\b""#, "\u{8}", false);
        t(r#""a\fa""#, "a\u{c}a", false);
        t(r#""\"a""#, "\"a", false);
        t("\"\"\"\na\"\"\"", "a", true);
        t("\"\"\"\n\"\"\"", "", true);
        t(r#""""a\"""b""""#, "a\"\"\"b", true);
        err(r#""\a"#, Error::InvalidEscape(2, 'a'));
        err("\"\\\n", Error::InvalidEscape(2, '\n'));
        err("\"\\\r\n", Error::InvalidEscape(2, '\n'));
        err("\"\\", Error::UnterminatedString(0));
        err("\"\u{0}", Error::InvalidCharInString(1, '\u{0}'));
        err(r#""\U00""#, Error::InvalidHexEscape(5, '"'));
        err(r#""\U00"#, Error::UnterminatedString(0));
        err(r#""\uD800"#, Error::InvalidEscapeValue(2, 0xd800));
        err(r#""\UFFFFFFFF"#, Error::InvalidEscapeValue(2, 0xffff_ffff));
    }

    #[test]
    fn keylike() {
        fn t(input: &str) {
            let mut t = Tokenizer::new(input);
            let (_, token) = t.next().unwrap().unwrap();
            assert_eq!(token, Token::Keylike(input));
            assert!(t.next().unwrap().is_none());
        }
        t("foo");
        t("0bar");
        t("bar0");
        t("1234");
        t("a-b");
        t("a_B");
        t("-_-");
        t("___");
    }

    #[test]
    fn all() {
        fn t(input: &str, expected: &[((usize, usize), Token<'_>, &str)]) {
            let mut tokens = Tokenizer::new(input);
            let mut actual: Vec<((usize, usize), Token<'_>, &str)> = Vec::new();
            while let Some((span, token)) = tokens.next().unwrap() {
                actual.push((span.into(), token, &input[span.start..span.end]));
            }
            for (a, b) in actual.iter().zip(expected) {
                assert_eq!(a, b);
            }
            assert_eq!(actual.len(), expected.len());
        }

        t(
            " a ",
            &[
                ((0, 1), Token::Whitespace(" "), " "),
                ((1, 2), Token::Keylike("a"), "a"),
                ((2, 3), Token::Whitespace(" "), " "),
            ],
        );

        t(
            " a\t [[]] \t [] {} , . =\n# foo \r\n#foo \n ",
            &[
                ((0, 1), Token::Whitespace(" "), " "),
                ((1, 2), Token::Keylike("a"), "a"),
                ((2, 4), Token::Whitespace("\t "), "\t "),
                ((4, 5), Token::LeftBracket, "["),
                ((5, 6), Token::LeftBracket, "["),
                ((6, 7), Token::RightBracket, "]"),
                ((7, 8), Token::RightBracket, "]"),
                ((8, 11), Token::Whitespace(" \t "), " \t "),
                ((11, 12), Token::LeftBracket, "["),
                ((12, 13), Token::RightBracket, "]"),
                ((13, 14), Token::Whitespace(" "), " "),
                ((14, 15), Token::LeftBrace, "{"),
                ((15, 16), Token::RightBrace, "}"),
                ((16, 17), Token::Whitespace(" "), " "),
                ((17, 18), Token::Comma, ","),
                ((18, 19), Token::Whitespace(" "), " "),
                ((19, 20), Token::Period, "."),
                ((20, 21), Token::Whitespace(" "), " "),
                ((21, 22), Token::Equals, "="),
                ((22, 23), Token::Newline, "\n"),
                ((23, 29), Token::Comment("# foo "), "# foo "),
                ((29, 31), Token::Newline, "\r\n"),
                ((31, 36), Token::Comment("#foo "), "#foo "),
                ((36, 37), Token::Newline, "\n"),
                ((37, 38), Token::Whitespace(" "), " "),
            ],
        );
    }

    #[test]
    fn bare_cr_bad() {
        err("\r", Error::Unexpected(0, '\r'));
        err("'\n", Error::NewlineInString(1));
        err("'\u{0}", Error::InvalidCharInString(1, '\u{0}'));
        err("'", Error::UnterminatedString(0));
        err("\u{0}", Error::Unexpected(0, '\u{0}'));
    }

    #[test]
    fn bad_comment() {
        let mut t = Tokenizer::new("#\u{0}");
        t.next().unwrap().unwrap();
        assert_eq!(t.next(), Err(Error::Unexpected(1, '\u{0}')));
        assert!(t.next().unwrap().is_none());
    }
}

Coverage Report

Created: 2023-04-25 07:07

Line	Count	Source (jump to first uncovered line)
1		use std::borrow::Cow;
2		use std::char;
3		use std::str;
4		use std::string;
5		use std::string::String as StdString;
6
7		use self::Token::*;
8
9		/// A span, designating a range of bytes where a token is located.
10	0	#[derive(Eq, PartialEq, Debug, Clone, Copy)] Unexecuted instantiation: <toml::tokens::Span as core::clone::Clone>::clone Unexecuted instantiation: <toml::tokens::Span as core::clone::Clone>::clone Unexecuted instantiation: <toml::tokens::Span as core::clone::Clone>::clone
11		pub struct Span {
12		/// The start of the range.
13		pub start: usize,
14		/// The end of the range (exclusive).
15		pub end: usize,
16		}
17
18		impl From<Span> for (usize, usize) {
19	0	fn from(Span { start, end }: Span) -> (usize, usize) {
20	0	(start, end)
21	0	}
22		}
23
24	0	#[derive(Eq, PartialEq, Debug)]
25		pub enum Token<'a> {
26		Whitespace(&'a str),
27		Newline,
28		Comment(&'a str),
29
30		Equals,
31		Period,
32		Comma,
33		Colon,
34		Plus,
35		LeftBrace,
36		RightBrace,
37		LeftBracket,
38		RightBracket,
39
40		Keylike(&'a str),
41		String {
42		src: &'a str,
43		val: Cow<'a, str>,
44		multiline: bool,
45		},
46		}
47
48	0	#[derive(Eq, PartialEq, Debug)]
49		pub enum Error {
50		InvalidCharInString(usize, char),
51		InvalidEscape(usize, char),
52		InvalidHexEscape(usize, char),
53		InvalidEscapeValue(usize, u32),
54		NewlineInString(usize),
55		Unexpected(usize, char),
56		UnterminatedString(usize),
57		NewlineInTableKey(usize),
58		MultilineStringKey(usize),
59		Wanted {
60		at: usize,
61		expected: &'static str,
62		found: &'static str,
63		},
64		}
65
66	0	#[derive(Clone)]
67		pub struct Tokenizer<'a> {
68		input: &'a str,
69		chars: CrlfFold<'a>,
70		}
71
72	0	#[derive(Clone)]
73		struct CrlfFold<'a> {
74		chars: str::CharIndices<'a>,
75		}
76
77	0	#[derive(Debug)]
78		enum MaybeString {
79		NotEscaped(usize),
80		Owned(string::String),
81		}
82
83		impl<'a> Tokenizer<'a> {
84	0	pub fn new(input: &'a str) -> Tokenizer<'a> {
85	0	let mut t = Tokenizer {
86	0	input,
87	0	chars: CrlfFold {
88	0	chars: input.char_indices(),
89	0	},
90	0	};
91	0	// Eat utf-8 BOM
92	0	t.eatc('\u{feff}');
93	0	t
94	0	}
95
96	0	pub fn next(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> {
97	0	let (start, token) = match self.one() {
98	0	Some((start, '\n')) => (start, Newline),
99	0	Some((start, ' ')) => (start, self.whitespace_token(start)),
100	0	Some((start, '\t')) => (start, self.whitespace_token(start)),
101	0	Some((start, '#')) => (start, self.comment_token(start)),
102	0	Some((start, '=')) => (start, Equals),
103	0	Some((start, '.')) => (start, Period),
104	0	Some((start, ',')) => (start, Comma),
105	0	Some((start, ':')) => (start, Colon),
106	0	Some((start, '+')) => (start, Plus),
107	0	Some((start, '{')) => (start, LeftBrace),
108	0	Some((start, '}')) => (start, RightBrace),
109	0	Some((start, '[')) => (start, LeftBracket),
110	0	Some((start, ']')) => (start, RightBracket),
111	0	Some((start, '\'')) => {
112	0	return self
113	0	.literal_string(start)
114	0	.map(\|t\| Some((self.step_span(start), t)))
115		}
116	0	Some((start, '"')) => {
117	0	return self
118	0	.basic_string(start)
119	0	.map(\|t\| Some((self.step_span(start), t)))
120		}
121	0	Some((start, ch)) if is_keylike(ch) => (start, self.keylike(start)),
122
123	0	Some((start, ch)) => return Err(Error::Unexpected(start, ch)),
124	0	None => return Ok(None),
125		};
126
127	0	let span = self.step_span(start);
128	0	Ok(Some((span, token)))
129	0	}
130
131	0	pub fn peek(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> {
132	0	self.clone().next()
133	0	}
134
135	0	pub fn eat(&mut self, expected: Token<'a>) -> Result<bool, Error> {
136	0	self.eat_spanned(expected).map(\|s\| s.is_some())
137	0	}
138
139		/// Eat a value, returning it's span if it was consumed.
140	0	pub fn eat_spanned(&mut self, expected: Token<'a>) -> Result<Option<Span>, Error> {
141	0	let span = match self.peek()? {
142	0	Some((span, ref found)) if expected == *found => span,
143	0	Some(_) => return Ok(None),
144	0	None => return Ok(None),
145		};
146
147	0	drop(self.next());
148	0	Ok(Some(span))
149	0	}
150
151		pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> {
152		// ignore span
153	0	let _ = self.expect_spanned(expected)?;
154	0	Ok(())
155	0	}
156
157		/// Expect the given token returning its span.
158	0	pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error> {
159	0	let current = self.current();
160	0	match self.next()? {
161	0	Some((span, found)) => {
162	0	if expected == found {
163	0	Ok(span)
164		} else {
165	0	Err(Error::Wanted {
166	0	at: current,
167	0	expected: expected.describe(),
168	0	found: found.describe(),
169	0	})
170		}
171		}
172	0	None => Err(Error::Wanted {
173	0	at: self.input.len(),
174	0	expected: expected.describe(),
175	0	found: "eof",
176	0	}),
177		}
178	0	}
179
180	0	pub fn table_key(&mut self) -> Result<(Span, Cow<'a, str>), Error> {
181	0	let current = self.current();
182	0	match self.next()? {
183	0	Some((span, Token::Keylike(k))) => Ok((span, k.into())),
184		Some((
185	0	span,
186	0	Token::String {
187	0	src,
188	0	val,
189	0	multiline,
190	0	},
191	0	)) => {
192	0	let offset = self.substr_offset(src);
193	0	if multiline {
194	0	return Err(Error::MultilineStringKey(offset));
195	0	}
196	0	match src.find('\n') {
197	0	None => Ok((span, val)),
198	0	Some(i) => Err(Error::NewlineInTableKey(offset + i)),
199		}
200		}
201	0	Some((_, other)) => Err(Error::Wanted {
202	0	at: current,
203	0	expected: "a table key",
204	0	found: other.describe(),
205	0	}),
206	0	None => Err(Error::Wanted {
207	0	at: self.input.len(),
208	0	expected: "a table key",
209	0	found: "eof",
210	0	}),
211		}
212	0	}
213
214	0	pub fn eat_whitespace(&mut self) -> Result<(), Error> {
215	0	while self.eatc(' ') \|\| self.eatc('\t') {
216	0	// ...
217	0	}
218	0	Ok(())
219	0	}
220
221	0	pub fn eat_comment(&mut self) -> Result<bool, Error> {
222	0	if !self.eatc('#') {
223	0	return Ok(false);
224	0	}
225	0	drop(self.comment_token(0));
226	0	self.eat_newline_or_eof().map(\|()\| true)
227	0	}
228
229	0	pub fn eat_newline_or_eof(&mut self) -> Result<(), Error> {
230	0	let current = self.current();
231	0	match self.next()? {
232	0	None \| Some((_, Token::Newline)) => Ok(()),
233	0	Some((_, other)) => Err(Error::Wanted {
234	0	at: current,
235	0	expected: "newline",
236	0	found: other.describe(),
237	0	}),
238		}
239	0	}
240
241	0	pub fn skip_to_newline(&mut self) {
242	0	loop {
243	0	match self.one() {
244	0	Some((_, '\n')) \| None => break,
245	0	_ => {}
246		}
247		}
248	0	}
249
250		fn eatc(&mut self, ch: char) -> bool {
251	0	match self.chars.clone().next() {
252	0	Some((_, ch2)) if ch == ch2 => {
253	0	self.one();
254	0	true
255		}
256	0	_ => false,
257		}
258	0	}
259
260	0	pub fn current(&mut self) -> usize {
261	0	self.chars
262	0	.clone()
263	0	.next()
264	0	.map(\|i\| i.0)
265	0	.unwrap_or_else(\|\| self.input.len())
266	0	}
267
268	0	pub fn input(&self) -> &'a str {
269	0	self.input
270	0	}
271
272	0	fn whitespace_token(&mut self, start: usize) -> Token<'a> {
273	0	while self.eatc(' ') \|\| self.eatc('\t') {
274	0	// ...
275	0	}
276	0	Whitespace(&self.input[start..self.current()])
277	0	}
278
279	0	fn comment_token(&mut self, start: usize) -> Token<'a> {
280	0	while let Some((_, ch)) = self.chars.clone().next() {
281	0	if ch != '\t' && (ch < '\u{20}' \|\| ch > '\u{10ffff}') {
282	0	break;
283	0	}
284	0	self.one();
285		}
286	0	Comment(&self.input[start..self.current()])
287	0	}
288
289	0	fn read_string(
290	0	&mut self,
291	0	delim: char,
292	0	start: usize,
293	0	new_ch: &mut dyn FnMut(
294	0	&mut Tokenizer<'_>,
295	0	&mut MaybeString,
296	0	bool,
297	0	usize,
298	0	char,
299	0	) -> Result<(), Error>,
300	0	) -> Result<Token<'a>, Error> {
301	0	let mut multiline = false;
302	0	if self.eatc(delim) {
303	0	if self.eatc(delim) {
304	0	multiline = true;
305	0	} else {
306	0	return Ok(String {
307	0	src: &self.input[start..start + 2],
308	0	val: Cow::Borrowed(""),
309	0	multiline: false,
310	0	});
311		}
312	0	}
313	0	let mut val = MaybeString::NotEscaped(self.current());
314	0	let mut n = 0;
315	0	'outer: loop {
316	0	n += 1;
317	0	match self.one() {
318	0	Some((i, '\n')) => {
319	0	if multiline {
320	0	if self.input.as_bytes()[i] == b'\r' {
321	0	val.to_owned(&self.input[..i]);
322	0	}
323	0	if n == 1 {
324	0	val = MaybeString::NotEscaped(self.current());
325	0	} else {
326	0	val.push('\n');
327	0	}
328	0	continue;
329		} else {
330	0	return Err(Error::NewlineInString(i));
331		}
332		}
333	0	Some((mut i, ch)) if ch == delim => {
334	0	if multiline {
335	0	if !self.eatc(delim) {
336	0	val.push(delim);
337	0	continue 'outer;
338	0	}
339	0	if !self.eatc(delim) {
340	0	val.push(delim);
341	0	val.push(delim);
342	0	continue 'outer;
343	0	}
344	0	if self.eatc(delim) {
345	0	val.push(delim);
346	0	i += 1;
347	0	}
348	0	if self.eatc(delim) {
349	0	val.push(delim);
350	0	i += 1;
351	0	}
352	0	}
353	0	return Ok(String {
354	0	src: &self.input[start..self.current()],
355	0	val: val.into_cow(&self.input[..i]),
356	0	multiline,
357	0	});
358		}
359	0	Some((i, c)) => new_ch(self, &mut val, multiline, i, c)?,
360	0	None => return Err(Error::UnterminatedString(start)),
361		}
362		}
363	0	}
364
365	0	fn literal_string(&mut self, start: usize) -> Result<Token<'a>, Error> {
366	0	self.read_string('\'', start, &mut \|_me, val, _multi, i, ch\| {
367	0	if ch == '\u{09}' \|\| ('\u{20}' <= ch && ch <= '\u{10ffff}' && ch != '\u{7f}') {
368	0	val.push(ch);
369	0	Ok(())
370		} else {
371	0	Err(Error::InvalidCharInString(i, ch))
372		}
373	0	})
374	0	}
375
376	0	fn basic_string(&mut self, start: usize) -> Result<Token<'a>, Error> {
377	0	self.read_string('"', start, &mut \|me, val, multi, i, ch\| match ch {
378		'\\' => {
379	0	val.to_owned(&me.input[..i]);
380	0	match me.chars.next() {
381	0	Some((_, '"')) => val.push('"'),
382	0	Some((_, '\\')) => val.push('\\'),
383	0	Some((_, 'b')) => val.push('\u{8}'),
384	0	Some((_, 'f')) => val.push('\u{c}'),
385	0	Some((_, 'n')) => val.push('\n'),
386	0	Some((_, 'r')) => val.push('\r'),
387	0	Some((_, 't')) => val.push('\t'),
388	0	Some((i, c @ 'u')) \| Some((i, c @ 'U')) => {
389	0	let len = if c == 'u' { 4 } else { 8 };
390	0	val.push(me.hex(start, i, len)?);
391		}
392	0	Some((i, c @ ' ')) \| Some((i, c @ '\t')) \| Some((i, c @ '\n')) if multi => {
393	0	if c != '\n' {
394	0	while let Some((_, ch)) = me.chars.clone().next() {
395	0	match ch {
396		' ' \| '\t' => {
397	0	me.chars.next();
398	0	continue;
399		}
400		'\n' => {
401	0	me.chars.next();
402	0	break;
403		}
404	0	_ => return Err(Error::InvalidEscape(i, c)),
405		}
406		}
407	0	}
408	0	while let Some((_, ch)) = me.chars.clone().next() {
409	0	match ch {
410	0	' ' \| '\t' \| '\n' => {
411	0	me.chars.next();
412	0	}
413	0	_ => break,
414		}
415		}
416		}
417	0	Some((i, c)) => return Err(Error::InvalidEscape(i, c)),
418	0	None => return Err(Error::UnterminatedString(start)),
419		}
420	0	Ok(())
421		}
422	0	ch if ch == '\u{09}' \|\| ('\u{20}' <= ch && ch <= '\u{10ffff}' && ch != '\u{7f}') => {
423	0	val.push(ch);
424	0	Ok(())
425		}
426	0	_ => Err(Error::InvalidCharInString(i, ch)),
427	0	})
428	0	}
429
430	0	fn hex(&mut self, start: usize, i: usize, len: usize) -> Result<char, Error> {
431	0	let mut buf = StdString::with_capacity(len);
432	0	for _ in 0..len {
433	0	match self.one() {
434	0	Some((_, ch)) if ch as u32 <= 0x7F && ch.is_digit(16) => buf.push(ch),
435	0	Some((i, ch)) => return Err(Error::InvalidHexEscape(i, ch)),
436	0	None => return Err(Error::UnterminatedString(start)),
437		}
438		}
439	0	let val = u32::from_str_radix(&buf, 16).unwrap();
440	0	match char::from_u32(val) {
441	0	Some(ch) => Ok(ch),
442	0	None => Err(Error::InvalidEscapeValue(i, val)),
443		}
444	0	}
445
446	0	fn keylike(&mut self, start: usize) -> Token<'a> {
447	0	while let Some((_, ch)) = self.peek_one() {
448	0	if !is_keylike(ch) {
449	0	break;
450	0	}
451	0	self.one();
452		}
453	0	Keylike(&self.input[start..self.current()])
454	0	}
455
456	0	pub fn substr_offset(&self, s: &'a str) -> usize {
457	0	assert!(s.len() <= self.input.len());
458	0	let a = self.input.as_ptr() as usize;
459	0	let b = s.as_ptr() as usize;
460	0	assert!(a <= b);
461	0	b - a
462	0	}
463
464		/// Calculate the span of a single character.
465	0	fn step_span(&mut self, start: usize) -> Span {
466	0	let end = self
467	0	.peek_one()
468	0	.map(\|t\| t.0)
469	0	.unwrap_or_else(\|\| self.input.len());
470	0	Span { start, end }
471	0	}
472
473		/// Peek one char without consuming it.
474	0	fn peek_one(&mut self) -> Option<(usize, char)> {
475	0	self.chars.clone().next()
476	0	}
477
478		/// Take one char.
479	0	pub fn one(&mut self) -> Option<(usize, char)> {
480	0	self.chars.next()
481	0	}
482		}
483
484		impl<'a> Iterator for CrlfFold<'a> {
485		type Item = (usize, char);
486
487	0	fn next(&mut self) -> Option<(usize, char)> {
488	0	self.chars.next().map(\|(i, c)\| {
489	0	if c == '\r' {
490	0	let mut attempt = self.chars.clone();
491	0	if let Some((_, '\n')) = attempt.next() {
492	0	self.chars = attempt;
493	0	return (i, '\n');
494	0	}
495	0	}
496	0	(i, c)
497	0	})
498	0	}
499		}
500
501		impl MaybeString {
502	0	fn push(&mut self, ch: char) {
503	0	match *self {
504	0	MaybeString::NotEscaped(..) => {}
505	0	MaybeString::Owned(ref mut s) => s.push(ch),
506		}
507	0	}
508
509	0	fn to_owned(&mut self, input: &str) {
510	0	match *self {
511	0	MaybeString::NotEscaped(start) => {
512	0	*self = MaybeString::Owned(input[start..].to_owned());
513	0	}
514	0	MaybeString::Owned(..) => {}
515		}
516	0	}
517
518	0	fn into_cow(self, input: &str) -> Cow<'_, str> {
519	0	match self {
520	0	MaybeString::NotEscaped(start) => Cow::Borrowed(&input[start..]),
521	0	MaybeString::Owned(s) => Cow::Owned(s),
522		}
523	0	}
524		}
525
526	0	fn is_keylike(ch: char) -> bool {
527	0	('A' <= ch && ch <= 'Z')
528	0	\|\| ('a' <= ch && ch <= 'z')
529	0	\|\| ('0' <= ch && ch <= '9')
530	0	\|\| ch == '-'
531	0	\|\| ch == '_'
532	0	}
533
534		impl<'a> Token<'a> {
535	0	pub fn describe(&self) -> &'static str {
536	0	match *self {
537	0	Token::Keylike(_) => "an identifier",
538	0	Token::Equals => "an equals",
539	0	Token::Period => "a period",
540	0	Token::Comment(_) => "a comment",
541	0	Token::Newline => "a newline",
542	0	Token::Whitespace(_) => "whitespace",
543	0	Token::Comma => "a comma",
544	0	Token::RightBrace => "a right brace",
545	0	Token::LeftBrace => "a left brace",
546	0	Token::RightBracket => "a right bracket",
547	0	Token::LeftBracket => "a left bracket",
548	0	Token::String { multiline, .. } => {
549	0	if multiline {
550	0	"a multiline string"
551		} else {
552	0	"a string"
553		}
554		}
555	0	Token::Colon => "a colon",
556	0	Token::Plus => "a plus",
557		}
558	0	}
559		}
560
561		#[cfg(test)]
562		mod tests {
563		use super::{Error, Token, Tokenizer};
564		use std::borrow::Cow;
565
566		fn err(input: &str, err: Error) {
567		let mut t = Tokenizer::new(input);
568		let token = t.next().unwrap_err();
569		assert_eq!(token, err);
570		assert!(t.next().unwrap().is_none());
571		}
572
573		#[test]
574		fn literal_strings() {
575		fn t(input: &str, val: &str, multiline: bool) {
576		let mut t = Tokenizer::new(input);
577		let (_, token) = t.next().unwrap().unwrap();
578		assert_eq!(
579		token,
580		Token::String {
581		src: input,
582		val: Cow::Borrowed(val),
583		multiline: multiline,
584		}
585		);
586		assert!(t.next().unwrap().is_none());
587		}
588
589		t("''", "", false);
590		t("''''''", "", true);
591		t("'''\n'''", "", true);
592		t("'a'", "a", false);
593		t("'\"a'", "\"a", false);
594		t("''''a'''", "'a", true);
595		t("'''\n'a\n'''", "'a\n", true);
596		t("'''a\n'a\r\n'''", "a\n'a\n", true);
597		}
598
599		#[test]
600		fn basic_strings() {
601		fn t(input: &str, val: &str, multiline: bool) {
602		let mut t = Tokenizer::new(input);
603		let (_, token) = t.next().unwrap().unwrap();
604		assert_eq!(
605		token,
606		Token::String {
607		src: input,
608		val: Cow::Borrowed(val),
609		multiline: multiline,
610		}
611		);
612		assert!(t.next().unwrap().is_none());
613		}
614
615		t(r#""""#, "", false);
616		t(r#""""""""#, "", true);
617		t(r#""a""#, "a", false);
618		t(r#""""a""""#, "a", true);
619		t(r#""\t""#, "\t", false);
620		t(r#""\u0000""#, "\0", false);
621		t(r#""\U00000000""#, "\0", false);
622		t(r#""\U000A0000""#, "\u{A0000}", false);
623		t(r#""\\t""#, "\\t", false);
624		t("\"\t\"", "\t", false);
625		t("\"\"\"\n\t\"\"\"", "\t", true);
626		t("\"\"\"\\\n\"\"\"", "", true);
627		t(
628		"\"\"\"\\\n \t \t \\\r\n \t \n \t \r\n\"\"\"",
629		"",
630		true,
631		);
632		t(r#""\r""#, "\r", false);
633		t(r#""\n""#, "\n", false);
634		t(r#""\b""#, "\u{8}", false);
635		t(r#""a\fa""#, "a\u{c}a", false);
636		t(r#""\"a""#, "\"a", false);
637		t("\"\"\"\na\"\"\"", "a", true);
638		t("\"\"\"\n\"\"\"", "", true);
639		t(r#""""a\"""b""""#, "a\"\"\"b", true);
640		err(r#""\a"#, Error::InvalidEscape(2, 'a'));
641		err("\"\\\n", Error::InvalidEscape(2, '\n'));
642		err("\"\\\r\n", Error::InvalidEscape(2, '\n'));
643		err("\"\\", Error::UnterminatedString(0));
644		err("\"\u{0}", Error::InvalidCharInString(1, '\u{0}'));
645		err(r#""\U00""#, Error::InvalidHexEscape(5, '"'));
646		err(r#""\U00"#, Error::UnterminatedString(0));
647		err(r#""\uD800"#, Error::InvalidEscapeValue(2, 0xd800));
648		err(r#""\UFFFFFFFF"#, Error::InvalidEscapeValue(2, 0xffff_ffff));
649		}
650
651		#[test]
652		fn keylike() {
653		fn t(input: &str) {
654		let mut t = Tokenizer::new(input);
655		let (_, token) = t.next().unwrap().unwrap();
656		assert_eq!(token, Token::Keylike(input));
657		assert!(t.next().unwrap().is_none());
658		}
659		t("foo");
660		t("0bar");
661		t("bar0");
662		t("1234");
663		t("a-b");
664		t("a_B");
665		t("-_-");
666		t("___");
667		}
668
669		#[test]
670		fn all() {
671		fn t(input: &str, expected: &[((usize, usize), Token<'_>, &str)]) {
672		let mut tokens = Tokenizer::new(input);
673		let mut actual: Vec<((usize, usize), Token<'_>, &str)> = Vec::new();
674		while let Some((span, token)) = tokens.next().unwrap() {
675		actual.push((span.into(), token, &input[span.start..span.end]));
676		}
677		for (a, b) in actual.iter().zip(expected) {
678		assert_eq!(a, b);
679		}
680		assert_eq!(actual.len(), expected.len());
681		}
682
683		t(
684		" a ",
685		&[
686		((0, 1), Token::Whitespace(" "), " "),
687		((1, 2), Token::Keylike("a"), "a"),
688		((2, 3), Token::Whitespace(" "), " "),
689		],
690		);
691
692		t(
693		" a\t [[]] \t [] {} , . =\n# foo \r\n#foo \n ",
694		&[
695		((0, 1), Token::Whitespace(" "), " "),
696		((1, 2), Token::Keylike("a"), "a"),
697		((2, 4), Token::Whitespace("\t "), "\t "),
698		((4, 5), Token::LeftBracket, "["),
699		((5, 6), Token::LeftBracket, "["),
700		((6, 7), Token::RightBracket, "]"),
701		((7, 8), Token::RightBracket, "]"),
702		((8, 11), Token::Whitespace(" \t "), " \t "),
703		((11, 12), Token::LeftBracket, "["),
704		((12, 13), Token::RightBracket, "]"),
705		((13, 14), Token::Whitespace(" "), " "),
706		((14, 15), Token::LeftBrace, "{"),
707		((15, 16), Token::RightBrace, "}"),
708		((16, 17), Token::Whitespace(" "), " "),
709		((17, 18), Token::Comma, ","),
710		((18, 19), Token::Whitespace(" "), " "),
711		((19, 20), Token::Period, "."),
712		((20, 21), Token::Whitespace(" "), " "),
713		((21, 22), Token::Equals, "="),
714		((22, 23), Token::Newline, "\n"),
715		((23, 29), Token::Comment("# foo "), "# foo "),
716		((29, 31), Token::Newline, "\r\n"),
717		((31, 36), Token::Comment("#foo "), "#foo "),
718		((36, 37), Token::Newline, "\n"),
719		((37, 38), Token::Whitespace(" "), " "),
720		],
721		);
722		}
723
724		#[test]
725		fn bare_cr_bad() {
726		err("\r", Error::Unexpected(0, '\r'));
727		err("'\n", Error::NewlineInString(1));
728		err("'\u{0}", Error::InvalidCharInString(1, '\u{0}'));
729		err("'", Error::UnterminatedString(0));
730		err("\u{0}", Error::Unexpected(0, '\u{0}'));
731		}
732
733		#[test]
734		fn bad_comment() {
735		let mut t = Tokenizer::new("#\u{0}");
736		t.next().unwrap().unwrap();
737		assert_eq!(t.next(), Err(Error::Unexpected(1, '\u{0}')));
738		assert!(t.next().unwrap().is_none());
739		}
740		}