package ij.macro;
import ij.*;
import java.io.*;

/** This class converts an imageJ macro file file into a token stream. */
public class Tokenizer implements MacroConstants {

    private StreamTokenizer st;
    private int token;
    private String tokenString;
    private double tokenValue;
    private Program pgm;
    private int lineNumber;

    /** Uses a StreamTokenizer to convert an ImageJ macro file into a token stream. */
    public Program tokenize(String program) {
        if (program.contains("/*") && program.contains("*/"))
            program = addSpacesToEmptyLines(program);
        st = new StreamTokenizer(new StringReader(program));
        st.ordinaryChar('-');
        st.ordinaryChar('/');
        st.ordinaryChar('.');
        st.wordChars('_', '_');
        st.whitespaceChars(128, 255);
        st.slashStarComments(true);
        st.slashSlashComments(true);
        pgm = new Program();
        do {
            getToken();
            addToken();
        } while (token!=EOF);
        if (pgm.hasFunctions)
        	addUserFunctions();
		//IJ.log(program.length()+" "+pgm.getSize()+" "+IJ.d2s((double)program.length()/pgm.getSize(),1)+" "+program.length()/10);
        return pgm;
    }

    final void getToken() {
        try {
            token = st.nextToken();
            lineNumber = st.lineno();
            String ret = null;
            int nextToken;
            switch (st.ttype) {
				case StreamTokenizer.TT_EOF:
					ret = "EOF";
					token = EOF;
					break;
				case StreamTokenizer.TT_WORD:
					ret = st.sval;
					token = WORD;
					break;
				case StreamTokenizer.TT_NUMBER:
					ret = ""+st.nval;
					tokenValue = st.nval;
					if (tokenValue==0.0)
						tokenValue = getHexConstant();
					else if (tryScientificNotation())
						ret += st.sval;
					token = NUMBER;
					break;
				case '"': case '\'':
					ret = ""+st.sval;
					token = STRING_CONSTANT;
					break;
				case '+':
					nextToken = st.nextToken();
					if (nextToken=='+')
						token = PLUS_PLUS;
					else if (nextToken=='=')
						token = PLUS_EQUAL;
					else
						st.pushBack();
					break;
				case '-':
					nextToken = st.nextToken();
					if (nextToken=='-')
						token = MINUS_MINUS;
					else if (nextToken=='=')
						token = MINUS_EQUAL;
					else
						st.pushBack();
					break;
				case '*':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = MUL_EQUAL;
					else
						st.pushBack();
					break;
				case '/':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = DIV_EQUAL;
					else
						st.pushBack();
					break;
				case '=':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = EQ;
					else
						st.pushBack();
					break;
				case '!':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = NEQ;
					else
						st.pushBack();
					break;
				case '>':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = GTE;
					else if (nextToken=='>')
						token = SHIFT_RIGHT;
					else {
						st.pushBack();
						token = GT;
					}
					break;
				case '<':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = LTE;
					else if (nextToken=='<')
						token = SHIFT_LEFT;
					else {
						st.pushBack();
						token = LT;
					}
					break;
				case '&':
					nextToken = st.nextToken();
					if (nextToken=='&')
						token = LOGICAL_AND;
					else
						st.pushBack();
					break;
				case '|':
					nextToken = st.nextToken();
					if (nextToken=='|')
						token = LOGICAL_OR;
					else
						st.pushBack();
					break;
				default:
            }
            tokenString = ret;
        } catch (Exception e) {
            return;
        }
    }

    final void addToken() {
        	int tok = token;
        	switch (token) {
        		case WORD:
				Symbol symbol = pgm.lookupWord(tokenString);
				if (symbol!=null) {
					int type = symbol.getFunctionType();
					if (type==0) {
						tok = symbol.type;
						switch (tok) {
							case FUNCTION: pgm.hasFunctions=true; break;
							case VAR: pgm.hasVars=true; break;
							case MACRO: pgm.macroCount++; break;
						}
					} else
						tok = type;
					tok += pgm.symTabLoc<<TOK_SHIFT;
				} else {
					pgm.addSymbol(new Symbol(token, tokenString));
					tok += pgm.stLoc<<TOK_SHIFT;
				}
				break;
			case STRING_CONSTANT:
				pgm.addSymbol(new Symbol(token, tokenString));
				tok += pgm.stLoc<<TOK_SHIFT;
				break;
			case NUMBER:
				pgm.addSymbol(new Symbol(tokenValue));
				tok += pgm.stLoc<<TOK_SHIFT;
				break;
			default:
				break;
        }
        pgm.addToken(tok, lineNumber);
    }

    double getHexConstant() {
        try {
            token = st.nextToken();
        } catch (Exception e) {
            return 0.0;
        }
        if (st.ttype != StreamTokenizer.TT_WORD) {
            st.pushBack();
            return 0.0;
        }
        if (!st.sval.startsWith("x")) {
            st.pushBack();
            return 0.0;
        }
        String s = st.sval.substring(1, st.sval.length());
        double n = 0.0;
        try {
            n = Long.parseLong(s, 16);
        } catch (NumberFormatException e) {
            st.pushBack();
            n = 0.0;
        }
        return n;
    }

	boolean tryScientificNotation() {
		String sval = "" + tokenValue;
		try {
			int next = st.nextToken();
			String sval2 = st.sval;
			if (st.ttype == st.TT_WORD && (sval2.startsWith("e")||sval2.startsWith("E"))) {
				// Usually we would just append the sval, but "-" is special...
				if (sval2.equalsIgnoreCase("e")) {
					//if (st.nextToken() != st.TT_WORD || !st.sval.equals("-"))
					next = st.nextToken();
					if (next == '-')
						sval2 += "-";
					else if (next != '+')
						throw new Exception();
					if (st.nextToken() != st.TT_NUMBER)
						throw new Exception();
					sval2 += st.nval;
				}
				if (sval2.endsWith(".0"))
					sval2 = sval2.substring(0, sval2.length() - 2);
				tokenValue = Double.parseDouble(sval + sval2);
				return true;
			}
			st.pushBack();
		} catch(Exception e) {}
		return false;
    }

	/** Adds user-defined functions to the symbol table. */
	void addUserFunctions() {
		int[] code = pgm.getCode();
		int nextToken, address, address2;
		for (int i=0; i<code.length; i++) {
			token = code[i]&TOK_MASK;
			if (token==FUNCTION) {
				nextToken = code[i+1]&TOK_MASK;
				if (nextToken==WORD || (nextToken>=PREDEFINED_FUNCTION&&nextToken<=ARRAY_FUNCTION)) {
					address = address2 = code[i+1]>>TOK_SHIFT;
					if (nextToken!=WORD) { // override built in function
                		pgm.addSymbol(new Symbol(WORD, pgm.getSymbolTable()[address].str));
                		address2 = pgm.stLoc;
                		code[i+1] = WORD + (address2<<TOK_SHIFT);
					}
					Symbol sym = pgm.getSymbolTable()[address2];
					sym.type = USER_FUNCTION;
					sym.value = i+1;  //address of function
					for (int j=0; j<code.length; j++) {
						token = code[j]&TOK_MASK;
						if ((token==WORD || (token>=PREDEFINED_FUNCTION&&token<=ARRAY_FUNCTION))
						&& (code[j]>>TOK_SHIFT)==address && (j==0||(code[j-1]&0xfff)!=FUNCTION)) {
							code[j] = USER_FUNCTION;
							code[j] += address2<<TOK_SHIFT;
							//IJ.log((code[j]&TOK_MASK)+" "+(code[j]>>TOK_SHIFT)+" "+USER_FUNCTION+" "+address);
						} else if (token==EOF)
							break;
					}
					//IJ.log(i+"  "+pgm.decodeToken(nextToken, address));
				}					
			} else if (token==EOF)
				break;
		}
	}
	
	private String addSpacesToEmptyLines(String pgm) {
		StringBuilder sb = new StringBuilder();
		int len = pgm.length();
		for (int jj=0; jj<len-1; jj++) {//insert a space if line is empty
			char c = pgm.charAt(jj);
			sb.append(c);
			if (jj<len-1 && c=='\n' && pgm.charAt(jj+1)=='\n')
				sb.append(" ");
		}
		sb.append(pgm.charAt(len-1)); //add last char
		return sb.toString(); //program no longer contain empty lines
	}

}