diff --git a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java index 8daa411..805d05b 100644 --- a/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java +++ b/src/main/java/ch/unibas/dmi/dbis/cs108/casono/server/tokenizer/Tokenizer.java @@ -4,9 +4,9 @@ import java.util.List; /** * A static utility class for tokenizing input strings into a sequence of tokens. - * - *
Strings can be enclosed in single quotes and may contain escaped single quotes - * using a backslash. + * + *
Strings can be enclosed in single quotes and may contain escaped single quotes using a
+ * backslash.
*/
public class Tokenizer {
/**
@@ -14,10 +14,10 @@ public class Tokenizer {
*
* @param input the string to tokenize
* @return a list of tokens representing the input
- * @throws TokenizerException if the input contains unexpected characters,
- * unterminated strings, or other syntax errors
+ * @throws TokenizerException if the input contains unexpected characters, unterminated strings,
+ * or other syntax errors
*/
- public static List The token type is determined by the {@link #resolveWordType(State)} method.
*
@@ -84,24 +84,20 @@ public class Tokenizer {
int startColumn = state.column;
StringBuilder sb = new StringBuilder();
- while (
- !state.isEof() &&
- (Character.isLetterOrDigit(state.current()) || state.current() == '_')
- ) {
+ while (!state.isEof()
+ && (Character.isLetterOrDigit(state.current()) || state.current() == '_')) {
sb.append(state.current());
state.advance();
}
- String word = sb.toString();
- TokenType type = resolveWordType(state);
- state.tokens.add(new Token(type, word, state.line, startColumn));
+ state.tokens.add(new RawToken(RawTokenType.WORD, sb.toString(), state.line, startColumn));
}
/**
* Reads a string literal enclosed in single quotes and creates a VALUE token.
*
- * Supports escaped single quotes using backslash notation (\'). Newlines within
- * strings are properly tracked for line counting.
+ * Supports escaped single quotes using backslash notation (\'). Newlines within strings are
+ * properly tracked for line counting.
*
* @param state the current parsing state
* @throws TokenizerException if the string literal is not terminated before end of input
@@ -114,7 +110,7 @@ public class Tokenizer {
while (true) {
if (state.isEof()) {
throw new TokenizerException(
- "Unterminated string literal", state.line, startColumn);
+ "Unterminated string literal", state.line, startColumn);
}
char c = state.current();
@@ -138,38 +134,6 @@ public class Tokenizer {
}
}
- state.tokens.add(new Token(TokenType.VALUE, sb.toString(), state.line, startColumn));
- }
-
- /**
- * Determines the token type for a word based on the parsing context.
- *
- * The type is resolved according to these rules:
- *
- *
- *
- * @param state the current parsing state
- * @return the determined token type
- */
- private static TokenType resolveWordType(State state) {
- if (state.tokens.isEmpty()) {
- return TokenType.COMMAND;
- }
-
- Token last = state.tokens.get(state.tokens.size() - 1);
-
- if (last.type() == TokenType.SEPARATOR) {
- return TokenType.VALUE;
- }
-
- if (last.type() == TokenType.NEWLINE) {
- return TokenType.COMMAND;
- }
-
- return TokenType.KEY;
+ state.tokens.add(new RawToken(RawTokenType.STRING, sb.toString(), state.line, startColumn));
}
}