diff --git a/src/main/kotlin/lumen/lexer/Lexer.kt b/src/main/kotlin/lumen/lexer/Lexer.kt new file mode 100644 index 0000000..b3c36c6 --- /dev/null +++ b/src/main/kotlin/lumen/lexer/Lexer.kt @@ -0,0 +1,200 @@ +package lumen.lexer + +import lumen.LexError + +private val keywords = mapOf( + "let" to TokenType.Let, + "var" to TokenType.Var, + "fn" to TokenType.Fn, + "return" to TokenType.Return, + "if" to TokenType.If, + "else" to TokenType.Else, + "while" to TokenType.While, + "for" to TokenType.For, + "in" to TokenType.In, + "true" to TokenType.True, + "false" to TokenType.False, + "nil" to TokenType.Nil, + "and" to TokenType.And, + "or" to TokenType.Or, + "not" to TokenType.Not, + "import" to TokenType.Import +) + +class Lexer(private val source: String) { + private val tokens = mutableListOf() + private var start = 0 + private var current = 0 + private var line = 1 + private var lineStart = 0 + private var groupingDepth = 0 + + fun scan(): List { + while (!atEnd()) { + start = current + scanToken() + } + tokens.add(Token(TokenType.Eof, "", line, current - lineStart + 1)) + return tokens + } + + private fun scanToken() { + val c = advance() + when (c) { + ' ', '\t', '\r' -> {} + '\n' -> { + if (groupingDepth == 0) add(TokenType.Newline) + line++ + lineStart = current + } + '(' -> { + groupingDepth++ + add(TokenType.LeftParen) + } + ')' -> { + if (groupingDepth > 0) groupingDepth-- + add(TokenType.RightParen) + } + '[' -> { + groupingDepth++ + add(TokenType.LeftBracket) + } + ']' -> { + if (groupingDepth > 0) groupingDepth-- + add(TokenType.RightBracket) + } + '{' -> add(TokenType.LeftBrace) + '}' -> add(TokenType.RightBrace) + ',' -> add(TokenType.Comma) + ':' -> add(TokenType.Colon) + ';' -> add(TokenType.Semicolon) + '.' -> add(TokenType.Dot) + '+' -> add(TokenType.Plus) + '-' -> add(TokenType.Minus) + '*' -> add(TokenType.Star) + '%' -> add(TokenType.Percent) + '/' -> when { + match('/') -> lineComment() + match('*') -> blockComment() + else -> add(TokenType.Slash) + } + '=' -> add(if (match('=')) TokenType.EqualEqual else TokenType.Assign) + '!' -> { + if (match('=')) add(TokenType.BangEqual) + else throw LexError("unexpected character '!'", line, columnOf(start)) + } + '<' -> add(if (match('=')) TokenType.LessEqual else TokenType.Less) + '>' -> add(if (match('=')) TokenType.GreaterEqual else TokenType.Greater) + '"' -> string() + else -> when { + c.isDigit() -> number() + isIdentStart(c) -> identifier() + else -> throw LexError("unexpected character '$c'", line, columnOf(start)) + } + } + } + + private fun lineComment() { + while (!atEnd() && peek() != '\n') advance() + } + + private fun blockComment() { + var depth = 1 + while (!atEnd() && depth > 0) { + val c = advance() + when { + c == '\n' -> { + line++ + lineStart = current + } + c == '/' && peek() == '*' -> { + advance() + depth++ + } + c == '*' && peek() == '/' -> { + advance() + depth-- + } + } + } + if (depth > 0) throw LexError("unterminated block comment", line, columnOf(start)) + } + + private fun string() { + val builder = StringBuilder() + while (!atEnd() && peek() != '"') { + val c = advance() + when { + c == '\n' -> { + builder.append('\n') + line++ + lineStart = current + } + c == '\\' -> builder.append(escape()) + else -> builder.append(c) + } + } + if (atEnd()) throw LexError("unterminated string", line, columnOf(start)) + advance() + tokens.add(Token(TokenType.StringLiteral, builder.toString(), line, columnOf(start), stringValue = builder.toString())) + } + + private fun escape(): Char { + if (atEnd()) throw LexError("unterminated escape sequence", line, columnOf(start)) + return when (val e = advance()) { + 'n' -> '\n' + 't' -> '\t' + 'r' -> '\r' + '"' -> '"' + '\\' -> '\\' + '0' -> ' ' + else -> throw LexError("invalid escape '\\$e'", line, columnOf(current - 2)) + } + } + + private fun number() { + while (peek().isDigit()) advance() + var isDouble = false + if (peek() == '.' && peekNext().isDigit()) { + isDouble = true + advance() + while (peek().isDigit()) advance() + } + val text = source.substring(start, current) + if (isDouble) { + tokens.add(Token(TokenType.DoubleLiteral, text, line, columnOf(start), doubleValue = text.toDouble())) + } else { + tokens.add(Token(TokenType.IntLiteral, text, line, columnOf(start), intValue = text.toLong())) + } + } + + private fun identifier() { + while (isIdentPart(peek())) advance() + val text = source.substring(start, current) + add(keywords[text] ?: TokenType.Identifier) + } + + private fun isIdentStart(c: Char) = c.isLetter() || c == '_' + + private fun isIdentPart(c: Char) = c.isLetterOrDigit() || c == '_' + + private fun add(type: TokenType) { + tokens.add(Token(type, source.substring(start, current), line, columnOf(start))) + } + + private fun columnOf(index: Int): Int = index - lineStart + 1 + + private fun advance(): Char = source[current++] + + private fun match(expected: Char): Boolean { + if (atEnd() || source[current] != expected) return false + current++ + return true + } + + private fun peek(): Char = if (atEnd()) ' ' else source[current] + + private fun peekNext(): Char = if (current + 1 >= source.length) ' ' else source[current + 1] + + private fun atEnd(): Boolean = current >= source.length +}