From fa54fd75c16c22599302bb1de1f1f4e1030cf329 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sat, 14 Mar 2026 15:32:40 +0100 Subject: [PATCH 01/15] Add: Tokenizer state and token representation --- .../cs108/casono/common/tokenizer/State.java | 39 +++++++++++++++++++ .../cs108/casono/common/tokenizer/Token.java | 4 ++ .../casono/common/tokenizer/TokenType.java | 11 ++++++ 3 files changed, 54 insertions(+) create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java new file mode 100644 index 0000000..e99754f --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java @@ -0,0 +1,39 @@ +package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; + +import java.util.ArrayList; +import java.util.List; + +/** Internal state for the tokenizer. */ +class State { + final String input; + int pos; + int line; + int column; + final List tokens; + + State(String input) { + this.input = input; + this.pos = 0; + this.line = 1; + this.column = 1; + this.tokens = new ArrayList<>(); + } + + char current() { + return input.charAt(pos); + } + + char peek() { + if (pos + 1 >= input.length()) return '\0'; + return input.charAt(pos + 1); + } + + void advance() { + pos++; + column++; + } + + boolean isEof() { + return pos >= input.length(); + } +} diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java new file mode 100644 index 0000000..5ee8d4b --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java @@ -0,0 +1,4 @@ +package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; + +/** Represents a token in the tokenizer. */ +public record Token(TokenType type, String value, int line, int column) {} diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java new file mode 100644 index 0000000..54c36ff --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java @@ -0,0 +1,11 @@ +package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; + +/** Enumeration of token types used in the tokenizer. */ +public enum TokenType { + COMMAND, + KEY, + VALUE, + SEPARATOR, + NEWLINE, + EOF +} -- 2.52.0 From d40390834162bc44287992353ae29ad23b281090 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sat, 14 Mar 2026 16:12:50 +0100 Subject: [PATCH 02/15] Add: TokenizerException thrown during tokenization --- .../common/tokenizer/TokenizerException.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerException.java diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerException.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerException.java new file mode 100644 index 0000000..da03eae --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerException.java @@ -0,0 +1,21 @@ +package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; + +/** Exception thrown during tokenization. */ +public class TokenizerException extends RuntimeException { + private final int line; + private final int column; + + public TokenizerException(String message, int line, int column) { + super(message); + this.line = line; + this.column = column; + } + + public int getLine() { + return line; + } + + public int getColumn() { + return column; + } +} -- 2.52.0 From ef2a00268f10b57da82582cb1c63c31d8be66ed7 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sat, 14 Mar 2026 16:13:35 +0100 Subject: [PATCH 03/15] Add: Tokenizer to tokenize provided string into tokens and perform syntax check --- .../casono/common/tokenizer/Tokenizer.java | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Tokenizer.java diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Tokenizer.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Tokenizer.java new file mode 100644 index 0000000..667c101 --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Tokenizer.java @@ -0,0 +1,175 @@ +package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; + +import java.util.List; + +/** + * A static utility class for tokenizing input strings into a sequence of tokens. + * + *

Strings can be enclosed in single quotes and may contain escaped single quotes + * using a backslash. + */ +public class Tokenizer { + /** + * Tokenizes the given input string into a list of tokens. + * + * @param input the string to tokenize + * @return a list of tokens representing the input + * @throws TokenizerException if the input contains unexpected characters, + * unterminated strings, or other syntax errors + */ + public static List tokenize(String input) { + State state = new State(input); + + while (!state.isEof()) { + char c = state.current(); + + if (c == ' ' || c == '\t') { + state.advance(); + + } else if (c == '\n') { + readNewline(state); + + } else if (c == '=') { + readSeparator(state); + + } else if (c == '\'') { + readString(state); + + } else if (Character.isLetterOrDigit(c) || c == '_') { + readWord(state); + + } else { + throw new TokenizerException( + "Unexpected character '" + c + "'", state.line, state.column); + } + } + + state.tokens.add(new Token(TokenType.EOF, null, state.line, state.column)); + return state.tokens; + } + + /** + * Reads a newline character and creates a NEWLINE token. + * + *

Updates the line counter and resets the column to 1. + * + * @param state the current parsing state + */ + private static void readNewline(State state) { + state.tokens.add(new Token(TokenType.NEWLINE, "", state.line, state.column)); + state.advance(); + state.line++; + state.column = 1; + } + + /** + * Reads a separator character ('=') and creates a SEPARATOR token. + * + * @param state the current parsing state + */ + private static void readSeparator(State state) { + state.tokens.add(new Token(TokenType.SEPARATOR, "=", state.line, state.column)); + state.advance(); + } + + /** + * Reads a word (sequence of alphanumeric characters and underscores) and creates + * an appropriate token (COMMAND, KEY, or VALUE) based on the parsing context. + * + *

The token type is determined by the {@link #resolveWordType(State)} method. + * + * @param state the current parsing state + */ + private static void readWord(State state) { + int startColumn = state.column; + StringBuilder sb = new StringBuilder(); + + while ( + !state.isEof() && + (Character.isLetterOrDigit(state.current()) || state.current() == '_') + ) { + sb.append(state.current()); + state.advance(); + } + + String word = sb.toString(); + TokenType type = resolveWordType(state); + state.tokens.add(new Token(type, word, state.line, startColumn)); + } + + /** + * Reads a string literal enclosed in single quotes and creates a VALUE token. + * + *

Supports escaped single quotes using backslash notation (\'). Newlines within + * strings are properly tracked for line counting. + * + * @param state the current parsing state + * @throws TokenizerException if the string literal is not terminated before end of input + */ + private static void readString(State state) { + int startColumn = state.column; + state.advance(); + StringBuilder sb = new StringBuilder(); + + while (true) { + if (state.isEof()) { + throw new TokenizerException( + "Unterminated string literal", state.line, startColumn); + } + + char c = state.current(); + + if (c == '\\' && state.peek() == '\'') { + sb.append('\''); + state.advance(); + state.advance(); + + } else if (c == '\'') { + state.advance(); + break; + + } else { + if (c == '\n') { + state.line++; + state.column = 1; + } + sb.append(c); + state.advance(); + } + } + + state.tokens.add(new Token(TokenType.VALUE, sb.toString(), state.line, startColumn)); + } + + /** + * Determines the token type for a word based on the parsing context. + * + *

The type is resolved according to these rules: + *

    + *
  • If no tokens have been added yet, it's a COMMAND
    • + *
  • If the previous token is a SEPARATOR, it's a VALUE
    • + *
  • If the previous token is a NEWLINE, it's a COMMAND
    • + *
  • Otherwise, it's a KEY
    • + *
+ * + * @param state the current parsing state + * @return the determined token type + */ + private static TokenType resolveWordType(State state) { + if (state.tokens.isEmpty()) { + return TokenType.COMMAND; + } + + Token last = state.tokens.get(state.tokens.size() - 1); + + if (last.type() == TokenType.SEPARATOR) { + return TokenType.VALUE; + } + + if (last.type() == TokenType.NEWLINE) { + return TokenType.COMMAND; + } + + return TokenType.KEY; + } +} -- 2.52.0 From 208a284204980a28f5dc4b31f8a131c981b0844d Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sat, 14 Mar 2026 16:33:34 +0100 Subject: [PATCH 04/15] Test: Tokenizer state --- .../casono/common/tokenizer/StateTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/StateTest.java diff --git a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/StateTest.java b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/StateTest.java new file mode 100644 index 0000000..db0b4fe --- /dev/null +++ b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/StateTest.java @@ -0,0 +1,60 @@ +package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class StateTest { + @Test + void testConstructor() { + String input = "hello"; + State state = new State(input); + assertEquals(input, state.input); + assertEquals(0, state.pos); + assertEquals(1, state.line); + assertEquals(1, state.column); + assertTrue(state.tokens.isEmpty()); + } + + @Test + void testCurrent() { + String input = "abc"; + State state = new State(input); + assertEquals('a', state.current()); + state.advance(); + assertEquals('b', state.current()); + } + + @Test + void testPeek() { + String input = "abc"; + State state = new State(input); + assertEquals('b', state.peek()); + state.advance(); + assertEquals('c', state.peek()); + state.advance(); + assertEquals('\0', state.peek()); // End of input + } + + @Test + void testAdvance() { + String input = "abc"; + State state = new State(input); + assertEquals(0, state.pos); + assertEquals(1, state.column); + state.advance(); + assertEquals(1, state.pos); + assertEquals(2, state.column); + } + + @Test + void testIsEof() { + String input = "a"; + State state = new State(input); + assertFalse(state.isEof()); + state.advance(); + assertTrue(state.isEof()); + } +} -- 2.52.0 From 14c18df0eb1b3c110681340e7c1b9136551a3b46 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sat, 14 Mar 2026 16:37:11 +0100 Subject: [PATCH 05/15] Test: Tokenizer exception --- .../tokenizer/TokenizerExceptionTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerExceptionTest.java diff --git a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerExceptionTest.java b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerExceptionTest.java new file mode 100644 index 0000000..1a56f49 --- /dev/null +++ b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerExceptionTest.java @@ -0,0 +1,18 @@ +package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class TokenizerExceptionTest { + @Test + void testConstructorAndGetters() { + String message = "Test error"; + int line = 5; + int column = 10; + TokenizerException e = new TokenizerException(message, line, column); + assertEquals(message, e.getMessage()); + assertEquals(line, e.getLine()); + assertEquals(column, e.getColumn()); + } +} -- 2.52.0 From 2af3fe7c35d3e7b22a53105ce4c387e3a7c435ad Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sat, 14 Mar 2026 17:13:18 +0100 Subject: [PATCH 06/15] Move: Tokenizer related files into server directory --- .../cs108/casono/{common => server}/tokenizer/State.java | 2 +- .../dmi/dbis/cs108/casono/server/tokenizer/Test.java | 7 +++++++ .../cs108/casono/{common => server}/tokenizer/Token.java | 2 +- .../casono/{common => server}/tokenizer/TokenType.java | 2 +- .../casono/{common => server}/tokenizer/Tokenizer.java | 2 +- .../{common => server}/tokenizer/TokenizerException.java | 2 +- 6 files changed, 12 insertions(+), 5 deletions(-) rename src/main/java/ch/unibas/dmi/dbis/cs108/casono/{common => server}/tokenizer/State.java (92%) create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java rename src/main/java/ch/unibas/dmi/dbis/cs108/casono/{common => server}/tokenizer/Token.java (67%) rename src/main/java/ch/unibas/dmi/dbis/cs108/casono/{common => server}/tokenizer/TokenType.java (72%) rename src/main/java/ch/unibas/dmi/dbis/cs108/casono/{common => server}/tokenizer/Tokenizer.java (98%) rename src/main/java/ch/unibas/dmi/dbis/cs108/casono/{common => server}/tokenizer/TokenizerException.java (88%) diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java similarity index 92% rename from src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java rename to src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java index e99754f..d9fe798 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/State.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java @@ -1,4 +1,4 @@ -package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java new file mode 100644 index 0000000..4c1944c --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java @@ -0,0 +1,7 @@ +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; + +public class Test { + public static void main(String[] args) { + System.out.println(Tokenizer.tokenize("ARG=12")); + } +} diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Token.java similarity index 67% rename from src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java rename to src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Token.java index 5ee8d4b..93b9026 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Token.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Token.java @@ -1,4 +1,4 @@ -package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; /** Represents a token in the tokenizer. */ public record Token(TokenType type, String value, int line, int column) {} diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java similarity index 72% rename from src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java rename to src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java index 54c36ff..b3a4e85 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenType.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java @@ -1,4 +1,4 @@ -package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; /** Enumeration of token types used in the tokenizer. */ public enum TokenType { diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Tokenizer.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java similarity index 98% rename from src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Tokenizer.java rename to src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java index 667c101..8daa411 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/Tokenizer.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java @@ -1,4 +1,4 @@ -package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; import java.util.List; diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerException.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerException.java similarity index 88% rename from src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerException.java rename to src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerException.java index da03eae..0975b1d 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerException.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerException.java @@ -1,4 +1,4 @@ -package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; /** Exception thrown during tokenization. */ public class TokenizerException extends RuntimeException { -- 2.52.0 From f3c34dfa7fa5bbc2011b18878362c3a00a60a062 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sat, 14 Mar 2026 17:14:37 +0100 Subject: [PATCH 07/15] Move: Tokenizer related tests into server server directory --- .../cs108/casono/{common => server}/tokenizer/StateTest.java | 2 +- .../{common => server}/tokenizer/TokenizerExceptionTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/test/java/ch/unibas/dmi/dbis/cs108/casono/{common => server}/tokenizer/StateTest.java (96%) rename src/test/java/ch/unibas/dmi/dbis/cs108/casono/{common => server}/tokenizer/TokenizerExceptionTest.java (89%) diff --git a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/StateTest.java b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/StateTest.java similarity index 96% rename from src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/StateTest.java rename to src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/StateTest.java index db0b4fe..fd1717b 100644 --- a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/StateTest.java +++ b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/StateTest.java @@ -1,4 +1,4 @@ -package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerExceptionTest.java b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerExceptionTest.java similarity index 89% rename from src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerExceptionTest.java rename to src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerExceptionTest.java index 1a56f49..d161b9b 100644 --- a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/common/tokenizer/TokenizerExceptionTest.java +++ b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerExceptionTest.java @@ -1,4 +1,4 @@ -package ch.unibas.dmi.dbis.cs108.casono.common.tokenizer; +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; import static org.junit.jupiter.api.Assertions.assertEquals; -- 2.52.0 From 45fd696fec1b601fb397b427a5387c33a08d7dc9 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 12:08:32 +0100 Subject: [PATCH 08/15] Add: RawToken and RawTokenType for token representation --- .../dmi/dbis/cs108/casono/server/tokenizer/RawToken.java | 8 ++++++++ .../dbis/cs108/casono/server/tokenizer/RawTokenType.java | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawTokenType.java diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java new file mode 100644 index 0000000..2093775 --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java @@ -0,0 +1,8 @@ +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; + +public record RawToken( + RawTokenType type, + String value, + int line, + int column +) {} diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawTokenType.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawTokenType.java new file mode 100644 index 0000000..b37d430 --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawTokenType.java @@ -0,0 +1,9 @@ +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; + +public enum RawTokenType { + WORD, + STRING, + SEPARATOR, + NEWLINE, + EOF +} -- 2.52.0 From 3e138ab84e4f1fb399c5e40555c251ceca0a94d7 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 12:11:59 +0100 Subject: [PATCH 09/15] Add: State to use RawToken --- .../ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java index d9fe798..6a32249 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java @@ -9,7 +9,7 @@ class State { int pos; int line; int column; - final List tokens; + final List tokens; State(String input) { this.input = input; -- 2.52.0 From 4c0e6aeb3614223abd05485836dd8c41afdbd1c1 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 12:20:37 +0100 Subject: [PATCH 10/15] Delete: Falsly added file for rudimentary quick testing --- .../dmi/dbis/cs108/casono/server/tokenizer/Test.java | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java deleted file mode 100644 index 4c1944c..0000000 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Test.java +++ /dev/null @@ -1,7 +0,0 @@ -package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; - -public class Test { - public static void main(String[] args) { - System.out.println(Tokenizer.tokenize("ARG=12")); - } -} -- 2.52.0 From 3509a967a6a3fac824618bfff28dcb15cd53b7a7 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 12:21:56 +0100 Subject: [PATCH 11/15] Add: Tokenizer does no longer classify tokens and instead returns 'RawToken's --- .../casono/server/tokenizer/Tokenizer.java | 74 +++++-------------- 1 file changed, 19 insertions(+), 55 deletions(-) diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java index 8daa411..805d05b 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java @@ -4,9 +4,9 @@ import java.util.List; /** * A static utility class for tokenizing input strings into a sequence of tokens. - * - *

Strings can be enclosed in single quotes and may contain escaped single quotes - * using a backslash. + * + *

Strings can be enclosed in single quotes and may contain escaped single quotes using a + * backslash. */ public class Tokenizer { /** @@ -14,10 +14,10 @@ public class Tokenizer { * * @param input the string to tokenize * @return a list of tokens representing the input - * @throws TokenizerException if the input contains unexpected characters, - * unterminated strings, or other syntax errors + * @throws TokenizerException if the input contains unexpected characters, unterminated strings, + * or other syntax errors */ - public static List tokenize(String input) { + public static List tokenize(String input) { State state = new State(input); while (!state.isEof()) { @@ -40,11 +40,11 @@ public class Tokenizer { } else { throw new TokenizerException( - "Unexpected character '" + c + "'", state.line, state.column); + "Unexpected character '" + c + "'", state.line, state.column); } } - state.tokens.add(new Token(TokenType.EOF, null, state.line, state.column)); + state.tokens.add(new RawToken(RawTokenType.EOF, null, state.line, state.column)); return state.tokens; } @@ -56,7 +56,7 @@ public class Tokenizer { * @param state the current parsing state */ private static void readNewline(State state) { - state.tokens.add(new Token(TokenType.NEWLINE, "", state.line, state.column)); + state.tokens.add(new RawToken(RawTokenType.NEWLINE, "", state.line, state.column)); state.advance(); state.line++; state.column = 1; @@ -68,13 +68,13 @@ public class Tokenizer { * @param state the current parsing state */ private static void readSeparator(State state) { - state.tokens.add(new Token(TokenType.SEPARATOR, "=", state.line, state.column)); + state.tokens.add(new RawToken(RawTokenType.SEPARATOR, "=", state.line, state.column)); state.advance(); } /** - * Reads a word (sequence of alphanumeric characters and underscores) and creates - * an appropriate token (COMMAND, KEY, or VALUE) based on the parsing context. + * Reads a word (sequence of alphanumeric characters and underscores) and creates an appropriate + * token (COMMAND, KEY, or VALUE) based on the parsing context. * *

The token type is determined by the {@link #resolveWordType(State)} method. * @@ -84,24 +84,20 @@ public class Tokenizer { int startColumn = state.column; StringBuilder sb = new StringBuilder(); - while ( - !state.isEof() && - (Character.isLetterOrDigit(state.current()) || state.current() == '_') - ) { + while (!state.isEof() + && (Character.isLetterOrDigit(state.current()) || state.current() == '_')) { sb.append(state.current()); state.advance(); } - String word = sb.toString(); - TokenType type = resolveWordType(state); - state.tokens.add(new Token(type, word, state.line, startColumn)); + state.tokens.add(new RawToken(RawTokenType.WORD, sb.toString(), state.line, startColumn)); } /** * Reads a string literal enclosed in single quotes and creates a VALUE token. * - *

Supports escaped single quotes using backslash notation (\'). Newlines within - * strings are properly tracked for line counting. + *

Supports escaped single quotes using backslash notation (\'). Newlines within strings are + * properly tracked for line counting. * * @param state the current parsing state * @throws TokenizerException if the string literal is not terminated before end of input @@ -114,7 +110,7 @@ public class Tokenizer { while (true) { if (state.isEof()) { throw new TokenizerException( - "Unterminated string literal", state.line, startColumn); + "Unterminated string literal", state.line, startColumn); } char c = state.current(); @@ -138,38 +134,6 @@ public class Tokenizer { } } - state.tokens.add(new Token(TokenType.VALUE, sb.toString(), state.line, startColumn)); - } - - /** - * Determines the token type for a word based on the parsing context. - * - *

The type is resolved according to these rules: - *

    - *
  • If no tokens have been added yet, it's a COMMAND
    • - *
  • If the previous token is a SEPARATOR, it's a VALUE
    • - *
  • If the previous token is a NEWLINE, it's a COMMAND
    • - *
  • Otherwise, it's a KEY
    • - *
- * - * @param state the current parsing state - * @return the determined token type - */ - private static TokenType resolveWordType(State state) { - if (state.tokens.isEmpty()) { - return TokenType.COMMAND; - } - - Token last = state.tokens.get(state.tokens.size() - 1); - - if (last.type() == TokenType.SEPARATOR) { - return TokenType.VALUE; - } - - if (last.type() == TokenType.NEWLINE) { - return TokenType.COMMAND; - } - - return TokenType.KEY; + state.tokens.add(new RawToken(RawTokenType.STRING, sb.toString(), state.line, startColumn)); } } -- 2.52.0 From 1e794337b59fb1d9a1d7b3b3dd5e502f1b34a079 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 13:08:23 +0100 Subject: [PATCH 12/15] Remove: 'NEWLINE' member from TokenType enum as commands dont span over multiple lines --- .../unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java index b3a4e85..af52b02 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenType.java @@ -6,6 +6,5 @@ public enum TokenType { KEY, VALUE, SEPARATOR, - NEWLINE, EOF } -- 2.52.0 From 60f59e03d421a3f127bf8f19c80fb18d62245571 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 13:26:59 +0100 Subject: [PATCH 13/15] Add: TokenClassifier to perform more extensive syntax verification and classify tokens --- .../server/tokenizer/TokenClassifier.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenClassifier.java diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenClassifier.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenClassifier.java new file mode 100644 index 0000000..cb19021 --- /dev/null +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenClassifier.java @@ -0,0 +1,103 @@ +package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; + +import java.util.ArrayList; +import java.util.List; + +public class TokenClassifier { + public static List classify(List rawTokens) { + validateFirstToken(rawTokens); + + List tokens = new ArrayList<>(); + + for (int i = 0; i < rawTokens.size(); i++) { + RawToken raw = rawTokens.get(i); + + switch (raw.type()) { + case SEPARATOR -> { + validateSeparator(rawTokens, i); + tokens.add( + new Token(TokenType.SEPARATOR, raw.value(), raw.line(), raw.column())); + } + case WORD -> { + TokenType type = resolveWordType(rawTokens, i); + tokens.add(new Token(type, raw.value(), raw.line(), raw.column())); + } + case STRING -> { + validateString(rawTokens, i); + tokens.add(new Token(TokenType.VALUE, raw.value(), raw.line(), raw.column())); + } + + case NEWLINE -> { + throw new TokenizerException( + "Unexpected newline in request", raw.line(), raw.column()); + } + + case EOF -> { + tokens.add(new Token(TokenType.EOF, raw.value(), raw.line(), raw.column())); + } + } + } + + return tokens; + } + + private static void validateFirstToken(List rawTokens) { + if (rawTokens.isEmpty() || rawTokens.get(0).type() != RawTokenType.WORD) { + throw new TokenizerException("Expected COMMAND as first token", 1, 1); + } + } + + private static void validateSeparator(List rawTokens, int index) { + boolean missingKey = index == 0 || rawTokens.get(index - 1).type() != RawTokenType.WORD; + + boolean missingValue = + index + 1 >= rawTokens.size() + || (rawTokens.get(index + 1).type() != RawTokenType.WORD + && rawTokens.get(index + 1).type() != RawTokenType.STRING); + + boolean nextWordIsKey = + !missingValue + && rawTokens.get(index + 1).type() == RawTokenType.WORD + && index + 2 < rawTokens.size() + && rawTokens.get(index + 2).type() == RawTokenType.SEPARATOR; + + RawToken separator = rawTokens.get(index); + + if (missingKey) { + throw new TokenizerException( + "Expected KEY before '='", separator.line(), separator.column()); + } + + if (missingValue || nextWordIsKey) { + throw new TokenizerException( + "Expected VALUE after '='", separator.line(), separator.column()); + } + } + + private static void validateString(List rawTokens, int index) { + boolean afterSeparator = + index > 0 && rawTokens.get(index - 1).type() == RawTokenType.SEPARATOR; + + if (!afterSeparator) { + RawToken token = rawTokens.get(index); + throw new TokenizerException("Unexpected string literal", token.line(), token.column()); + } + } + + private static TokenType resolveWordType(List rawTokens, int index) { + boolean isFirst = index == 0; + boolean afterNewline = index > 0 && rawTokens.get(index - 1).type() == RawTokenType.NEWLINE; + boolean afterSeparator = + index > 0 && rawTokens.get(index - 1).type() == RawTokenType.SEPARATOR; + + if (isFirst || afterNewline) { + return TokenType.COMMAND; + } + + if (afterSeparator) { + return TokenType.VALUE; + } + + return TokenType.KEY; + } +} -- 2.52.0 From ca11c135f1691844072e2e738028d655b3736ce7 Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 13:28:06 +0100 Subject: [PATCH 14/15] Style: Bracket after 'if' and record in one-line --- .../dmi/dbis/cs108/casono/server/tokenizer/RawToken.java | 8 ++------ .../dmi/dbis/cs108/casono/server/tokenizer/State.java | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java index 2093775..c8bc6bb 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/RawToken.java @@ -1,8 +1,4 @@ package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; -public record RawToken( - RawTokenType type, - String value, - int line, - int column -) {} +/** Represents a raw (unclassified) token in the tokenizer. */ +public record RawToken(RawTokenType type, String value, int line, int column) {} diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java index 6a32249..d130a52 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/State.java @@ -24,7 +24,9 @@ class State { } char peek() { - if (pos + 1 >= input.length()) return '\0'; + if (pos + 1 >= input.length()) { + return '\0'; + } return input.charAt(pos + 1); } -- 2.52.0 From 08dc94b4b09fc5552f5b9b6f47720aaa7dc636bf Mon Sep 17 00:00:00 2001 From: Lars Simon Winzer Date: Sun, 15 Mar 2026 13:38:59 +0100 Subject: [PATCH 15/15] Style: Comply with Spotless Created issue #2 regarding this issue --- .../dmi/dbis/cs108/casono/server/tokenizer/StateTest.java | 3 +-- .../cs108/casono/server/tokenizer/TokenizerExceptionTest.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/StateTest.java b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/StateTest.java index fd1717b..0f498d7 100644 --- a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/StateTest.java +++ b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/StateTest.java @@ -1,11 +1,10 @@ package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; +import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Test; - class StateTest { @Test void testConstructor() { diff --git a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerExceptionTest.java b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerExceptionTest.java index d161b9b..0ed20d3 100644 --- a/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerExceptionTest.java +++ b/src/test/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/TokenizerExceptionTest.java @@ -1,8 +1,7 @@ package ch.unibas.dmi.dbis.cs108.casono.server.tokenizer; -import static org.junit.jupiter.api.Assertions.assertEquals; - import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; class TokenizerExceptionTest { @Test -- 2.52.0