Add: Tokenizer state and token representation

src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java

@@ -0,0 +1,39 @@
+package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** Internal state for the tokenizer. */
+class State {
+    final String input;
+    int pos;
+    int line;
+    int column;
+    final List<Token> tokens;
+
+    State(String input) {
+        this.input = input;
+        this.pos = 0;
+        this.line = 1;
+        this.column = 1;
+        this.tokens = new ArrayList<>();
+    }
+
+    char current() {
+        return input.charAt(pos);
+    }
+
+    char peek() {
+        if (pos + 1 >= input.length()) return '\0';
+        return input.charAt(pos + 1);
+    }
+
+    void advance() {
+        pos++;
+        column++;
+    }
+
+    boolean isEof() {
+        return pos >= input.length();
+    }
+}

src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java

@@ -0,0 +1,4 @@
+package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer;
+
+/** Represents a token in the tokenizer. */
+public record Token(TokenType type, String value, int line, int column) {}

src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java

@@ -0,0 +1,11 @@
+package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer;
+
+/** Enumeration of token types used in the tokenizer. */
+public enum TokenType {
+    COMMAND,
+    KEY,
+    VALUE,
+    SEPARATOR,
+    NEWLINE,
+    EOF
+}