import java.util.List; import java.util.ArrayList; /* * Lexical analyzer for arithmetic expressions which include variable names * and positive integer literals * e.g. (sum + 47) / total */ public class Lexer { List tokens; int index = 0; Lexer(String s) { tokens = tokenize(s); index = 0; } public List tokenize(String input) { List result = new ArrayList(); for(int i = 0; i < input.length(); ) { switch(input.charAt(i)) { case '(': result.add(new Token(TokenTypes.LPAREN, "(")); i++; break; case ')': result.add(new Token(TokenTypes.RPAREN, ")")); i++; break; case '+': result.add(new Token(TokenTypes.ADD_OP, "+")); i++; break; case '-': result.add(new Token(TokenTypes.SUB_OP, "-")); i++; break; case '*': result.add(new Token(TokenTypes.MULT_OP, "*")); i++; break; case '/': result.add(new Token(TokenTypes.DIV_OP, "/")); i++; break; default: if(Character.isWhitespace(input.charAt(i))) i++; else if (Character.isDigit(input.charAt(i))) { int j = i; while ((j < input.length()) && Character.isDigit(input.charAt(j))) j++; result.add(new Token(TokenTypes.INT_LIT, input.substring(i,j))); i = j; } else if (Character.isLetter(input.charAt(i))){ int j = i; while ((j < input.length()) && (Character.isDigit(input.charAt(j)) || Character.isLetter(input.charAt(j)))) j++; result.add(new Token(TokenTypes.IDENT, input.substring(i,j))); i = j; } else { System.out.println("UNEXPECTED CHARACTER ENCOUNTERED"); System.exit(-1); } break; } } result.add(new Token(TokenTypes.EOF, "-1")); return result; } public Token lex() { Token t = null; if (index < tokens.size()) { t = tokens.get(index); index++; } System.out.println("Next Token is: "+t.getTokenID()+", Next lexeme is "+t); return t; } }