Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 28 additions & 12 deletions crates/sploosh-lexer/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,13 @@ pub fn is_contextual_keyword(text: &str) -> bool {
)
}

/// §16.1 numeric suffixes, longest-first so prefix scanning never matches a
/// shorter suffix inside a longer one. Shared by suffix scanning and
/// separator validation.
const NUMERIC_SUFFIXES: [&str; 13] = [
"i128", "u128", "u256", "i64", "u64", "f64", "i32", "u32", "f32", "i16", "u16", "i8", "u8",
];

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LexError {
pub message: String,
Expand Down Expand Up @@ -339,28 +346,20 @@ impl Lexer<'_> {
}
}

fn numeric_suffix(&mut self) -> Option<String> {
fn numeric_suffix(&mut self) -> Option<&'static str> {
let rest = &self.source[self.pos..];
let suffixes = [
"i128", "u128", "u256", "i64", "u64", "f64", "i32", "u32", "f32", "i16", "u16", "i8",
"u8",
];
for suffix in suffixes {
for suffix in NUMERIC_SUFFIXES {
if rest.starts_with(suffix) {
self.pos += suffix.len();
return Some(suffix.to_string());
return Some(suffix);
}
}
None
}

fn validate_numeric_body(&mut self, start: usize, base: u8) {
let text = &self.source[start..self.pos];
let suffixes = [
"i128", "u128", "u256", "i64", "u64", "f64", "i32", "u32", "f32", "i16", "u16", "i8",
"u8",
];
let body = suffixes
let body = NUMERIC_SUFFIXES
.iter()
.find_map(|suffix| text.strip_suffix(suffix))
.unwrap_or(text);
Expand Down Expand Up @@ -628,6 +627,23 @@ mod tests {
assert_eq!(tokens[4].kind, TokenKind::IntLit);
}

#[test]
fn every_numeric_suffix_lexes() {
for suffix in NUMERIC_SUFFIXES {
let source = format!("1{suffix}");
let tokens = lex(&source).unwrap();
assert_eq!(tokens.len(), 1, "{source}");
let expected = if suffix.starts_with('f') {
TokenKind::FloatLit
} else {
TokenKind::IntLit
};
assert_eq!(tokens[0].kind, expected, "{source}");
assert_eq!(tokens[0].lexeme, source);
assert_eq!(tokens[0].span, Span::new(0, source.len()));
}
}

#[test]
fn rejects_bad_numeric_separators() {
let err = lex("1__2").unwrap_err();
Expand Down