From 2ae19e0ee02e7d86d21977d1e9182681a9c994ec Mon Sep 17 00:00:00 2001 From: DualFroz Date: Mon, 24 Aug 2026 13:53:05 +0200 Subject: [PATCH] feat(core/asm): add assembler lexer --- src/Chip8.Core/Assembly/Lexer.cs | 243 +++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 src/Chip8.Core/Assembly/Lexer.cs diff --git a/src/Chip8.Core/Assembly/Lexer.cs b/src/Chip8.Core/Assembly/Lexer.cs new file mode 100644 index 0000000..ef814c0 --- /dev/null +++ b/src/Chip8.Core/Assembly/Lexer.cs @@ -0,0 +1,243 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace Chip8.Core.Assembly; + +public sealed class Lexer +{ + readonly string source; + int offset; + int line = 1; + int lineStart; + + public Lexer(string source) + { + this.source = source; + } + + public List tokenize() + { + var tokens = new List(); + + while (offset < source.Length) + { + char c = source[offset]; + + if (c == '\n') + { + tokens.Add(makeToken(TokenKind.EndOfLine, "\\n")); + advanceLine(); + continue; + } + + if (c == '\r') + { + offset++; + continue; + } + + if (c == ' ' || c == '\t') + { + offset++; + continue; + } + + if (c == ';' || (c == '/' && peek(1) == '/')) + { + skipToLineEnd(); + continue; + } + + tokens.Add(scanToken()); + } + + tokens.Add(makeToken(TokenKind.EndOfLine, "\\n")); + tokens.Add(makeToken(TokenKind.EndOfFile, "")); + return tokens; + } + + Token scanToken() + { + char c = source[offset]; + + if (isIdentifierStart(c)) + { + return scanIdentifier(); + } + + if (char.IsDigit(c) || c == '$' || c == '%') + { + return scanNumber(); + } + + return scanPunctuation(); + } + + Token scanIdentifier() + { + int column = currentColumn(); + int start = offset; + while (offset < source.Length && isIdentifierPart(source[offset])) + { + offset++; + } + string text = source.Substring(start, offset - start); + return new Token(TokenKind.Identifier, text, 0, new SourcePosition(line, column)); + } + + Token scanNumber() + { + int column = currentColumn(); + int start = offset; + char c = source[offset]; + + if (c == '$') + { + offset++; + return finishRadixNumber(start, column, 16, "hex"); + } + + if (c == '%') + { + offset++; + return finishRadixNumber(start, column, 2, "binary"); + } + + if (c == '0' && (peek(1) == 'x' || peek(1) == 'X')) + { + offset += 2; + return finishRadixNumber(start, column, 16, "hex"); + } + + if (c == '0' && (peek(1) == 'b' || peek(1) == 'B')) + { + offset += 2; + return finishRadixNumber(start, column, 2, "binary"); + } + + while (offset < source.Length && char.IsDigit(source[offset])) + { + offset++; + } + string decimalText = source.Substring(start, offset - start); + int decimalValue = int.Parse(decimalText, CultureInfo.InvariantCulture); + return new Token(TokenKind.Number, decimalText, decimalValue, new SourcePosition(line, column)); + } + + Token finishRadixNumber(int start, int column, int radix, string label) + { + int digitsStart = offset; + while (offset < source.Length && isRadixDigit(source[offset], radix)) + { + offset++; + } + + if (offset == digitsStart) + { + throw new AsmException("malformed " + label + " literal", new SourcePosition(line, column)); + } + + string digits = source.Substring(digitsStart, offset - digitsStart); + int value = 0; + foreach (char digit in digits) + { + value = value * radix + digitValue(digit); + } + + string text = source.Substring(start, offset - start); + return new Token(TokenKind.Number, text, value, new SourcePosition(line, column)); + } + + Token scanPunctuation() + { + int column = currentColumn(); + char c = source[offset]; + char next = peek(1); + + if (c == '<' && next == '<') + { + offset += 2; + return new Token(TokenKind.ShiftLeft, "<<", 0, new SourcePosition(line, column)); + } + + if (c == '>' && next == '>') + { + offset += 2; + return new Token(TokenKind.ShiftRight, ">>", 0, new SourcePosition(line, column)); + } + + TokenKind kind = c switch + { + ',' => TokenKind.Comma, + ':' => TokenKind.Colon, + '-' => TokenKind.Minus, + '+' => TokenKind.Plus, + '*' => TokenKind.Star, + '/' => TokenKind.Slash, + '&' => TokenKind.Ampersand, + '|' => TokenKind.Pipe, + '^' => TokenKind.Caret, + '~' => TokenKind.Tilde, + '(' => TokenKind.LeftParen, + ')' => TokenKind.RightParen, + '[' => TokenKind.LeftBracket, + ']' => TokenKind.RightBracket, + _ => throw new AsmException("unexpected character '" + c + "'", new SourcePosition(line, column)), + }; + + offset++; + return new Token(kind, c.ToString(), 0, new SourcePosition(line, column)); + } + + void skipToLineEnd() + { + while (offset < source.Length && source[offset] != '\n') + { + offset++; + } + } + + void advanceLine() + { + offset++; + line++; + lineStart = offset; + } + + Token makeToken(TokenKind kind, string text) => + new(kind, text, 0, new SourcePosition(line, currentColumn())); + + int currentColumn() => offset - lineStart + 1; + + char peek(int ahead) + { + int index = offset + ahead; + return index < source.Length ? source[index] : '\0'; + } + + static bool isIdentifierStart(char c) => char.IsLetter(c) || c == '_' || c == '.'; + + static bool isIdentifierPart(char c) => char.IsLetterOrDigit(c) || c == '_' || c == '.'; + + static bool isRadixDigit(char c, int radix) => radix switch + { + 2 => c == '0' || c == '1', + 16 => Uri.IsHexDigit(c), + _ => char.IsDigit(c), + }; + + static int digitValue(char c) + { + if (c >= '0' && c <= '9') + { + return c - '0'; + } + if (c >= 'a' && c <= 'f') + { + return c - 'a' + 10; + } + return c - 'A' + 10; + } +}