Add server-side tokenizer #185

Merged
lars.winzer merged 16 commits from feat/tokenizer into main 2026-03-15 13:52:51 +01:00
2 changed files with 5 additions and 7 deletions
Showing only changes of commit ca11c135f1 - Show all commits
@@ -1,8 +1,4 @@
package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer;
public record RawToken(
RawTokenType type,
String value,
int line,
int column
) {}
/** Represents a raw (unclassified) token in the tokenizer. */
public record RawToken(RawTokenType type, String value, int line, int column) {}
@@ -24,7 +24,9 @@ class State {
}
char peek() {
if (pos + 1 >= input.length()) return '\0';
if (pos + 1 >= input.length()) {
return '\0';
}
return input.charAt(pos + 1);
}