/*
 * libtu/tokenizer.c
 *
 * Copyright (c) Tuomo Valkonen 1999-2002.
 *
 * You may distribute and modify this library under the terms of either
 * the Clarified Artistic License or the GNU LGPL, version 2.1 or later.
 */

#include <errno.h>
#include <stdio.h>
#include <ctype.h>
#include <limits.h>
#include <assert.h>
#include <math.h>
#include <string.h>

#include "tokenizer.h"
#include "misc.h"
#include "output.h"
#include "private.h"


static const char *errors[]={
    DUMMY_TR("(no error)"),
    DUMMY_TR("Unexpected end of file"),                /* E_TOKZ_UNEXPECTED_EOF */
    DUMMY_TR("Unexpected end of line"),                /* E_TOKZ_UNEXPECTED_EOL */
    DUMMY_TR("End of line expected"),                /* E_TOKZ_EOL_EXPECTED */
    DUMMY_TR("Invalid character"),                    /* E_TOKZ_INVALID_CHAR*/
    DUMMY_TR("Numeric constant too big"),            /* E_TOKZ_TOOBIG */
    DUMMY_TR("Invalid numberic format"),            /* E_TOKZ_NUMFMT */
    DUMMY_TR("Junk after numeric constant"),        /* E_TOKZ_NUM_JUNK */
    DUMMY_TR("Not an integer"),                        /* E_TOKZ_NOTINT */
    DUMMY_TR("Numeric constant out of range"),        /* E_TOKZ_RANGE */
    DUMMY_TR("Multi-character character constant"),    /* E_TOKZ_MULTICHAR */
    DUMMY_TR("Token/statement limit reached"),        /* E_TOKZ_TOKEN_LIMIT */
    DUMMY_TR("Unknown option"),                        /* E_TOKZ_UNKONWN_OPTION */
    DUMMY_TR("Syntax error"),                        /* E_TOKZ_SYNTAX */
    DUMMY_TR("Invalid argument"),                    /* E_TOKZ_INVALID_ARGUMENT */
    DUMMY_TR("End of statement expected"),            /* E_TOKZ_EOS_EXPECTED */
    DUMMY_TR("Too few arguments"),                    /* E_TOKZ_TOO_FEW_ARGS */
    DUMMY_TR("Too many arguments"),                    /* E_TOKZ_TOO_MANY_ARGS */
    DUMMY_TR("Maximum section nestin level exceeded"), /* E_TOK_Z_MAX_NEST */
    DUMMY_TR("Identifier expected"),                /* E_TOKZ_IDENTIFIER_EXPECTED */
    DUMMY_TR("Starting brace ('{') expected"),        /* E_TOKZ_LBRACE_EXPECTED */
};


/* */

#define STRBLEN 32

#define STRING_DECL(X) int err=0; char* X=NULL; char X##_tmp[STRBLEN]; int X##_tmpl=0; ((void)0)
#define STRING_DECL_P(X, P) int err=0; char* X=NULL; char X##_tmp[STRBLEN]=P; int X##_tmpl=sizeof(P)-1; ((void)0)
#define STRING_APPEND(X, C) {if(!_string_append(&X, X##_tmp, &X##_tmpl, c)) err=-ENOMEM;} ((void)0)
#define STRING_FREE(X) if(X!=NULL) free(X)
#define STRING_FINISH(X) {if(err!=0) return err; if(!_string_finish(&X, X##_tmp, X##_tmpl)) err=-ENOMEM;} ((void)0)


static bool _string_append(char **p, char *tmp, int *tmplen, char c)
{
    char *tmp2;

    if(*tmplen==STRBLEN-1){
        tmp[STRBLEN-1]='\0';
        if(*p!=NULL){
            tmp2=scat(*p, tmp);
            free(*p);
            *p=tmp2;
        }else{
            *p=scopy(tmp);
        }
        *tmplen=1;
        tmp[0]=c;
        return *p!=NULL;
    }else{
        tmp[(*tmplen)++]=c;
        return TRUE;
    }
}


static bool _string_finish(char **p, char *tmp, int tmplen)
{
    char *tmp2;

    if(tmplen==0){
        if(*p==NULL)
            *p=scopy("");
    }else{
        tmp[tmplen]='\0';
        if(*p!=NULL){
            tmp2=scat(*p, tmp);
            free(*p);
            *p=tmp2;
        }else{
            *p=scopy(tmp);
        }
    }
    return *p!=NULL;
}


/* */


#define INC_LINE() tokz->line++
#define GETCH() _getch(tokz)
#define UNGETCH(C) _ungetch(tokz, C)

static int _getch(Tokenizer *tokz)
{
    int c;

    if(tokz->ungetc!=-1){
        c=tokz->ungetc;
        tokz->ungetc=-1;
    }else if (tokz->flags&TOKZ_READ_FROM_BUFFER) {
        assert(tokz->buffer.data!=NULL);
        if (tokz->buffer.pos==tokz->buffer.len)
            c=EOF;
        else
            c=tokz->buffer.data[tokz->buffer.pos++];
    }else{
        c=getc(tokz->file);
    }

    return c;
}


static void _ungetch(Tokenizer *tokz, int c)
{
    tokz->ungetc=c;
}


/* */


static int scan_line_comment(Token *tok, Tokenizer *tokz)
{
    STRING_DECL_P(s, "#");
    int c;

    c=GETCH();

    while(c!='\n' && c!=EOF){
        STRING_APPEND(s, c);
        c=GETCH();
    }

    UNGETCH(c);

    STRING_FINISH(s);

    TOK_SET_COMMENT(tok, s);

    return 0;
}


static int skip_line_comment(Tokenizer *tokz)
{
    int c;

    do{
        c=GETCH();
    }while(c!='\n' && c!=EOF);

    UNGETCH(c);

    return 0;
}


/* */


static int scan_c_comment(Token *tok, Tokenizer *tokz)
{
    STRING_DECL_P(s, "/*");
    int c;
    int st=0;

    while(1){
        c=GETCH();

        if(c==EOF){
            STRING_FREE(s);
            return E_TOKZ_UNEXPECTED_EOF;
        }

        STRING_APPEND(s, c);

        if(c=='\n'){
            INC_LINE();
        }else if(st==0 && c=='*'){
            st=1;
        }else if(st==1){
            if(c=='/')
                break;
            st=0;
        }
    }

    STRING_FINISH(s);

    TOK_SET_COMMENT(tok, s);

    return 0;
}


static int skip_c_comment(Tokenizer *tokz)
{
    int c;
    int st=0;

    while(1){
        c=GETCH();

        if(c==EOF)
            return E_TOKZ_UNEXPECTED_EOF;

        if(c=='\n')
            INC_LINE();
        else if(st==0 && c=='*')
            st=1;
        else if(st==1){
            if(c=='/')
                break;
            st=0;
        }
    }

    return 0;
}


/* */


static int scan_char_escape(Tokenizer *tokz)
{
    static char* special_chars="nrtbae";
    static char* specials="\n\r\t\b\a\033";
    int base, max;
    int i ,c;

    c=GETCH();

    for(i=0;special_chars[i];i++){
        if(special_chars[i]==c)
            return specials[c];
    }

    if(c=='x' || c=='X'){
        base=16;max=2;i=0;
    }else if(c=='d' || c=='D'){
        base=10;max=3;i=0;
    }else if(c=='8' || c=='9'){
        base=10;max=2;i=c-'0';
    }else if('0'<=c && c<='7'){
        base=8;max=2;i=c-'0';
    }else if(c=='\n'){
        UNGETCH(c);
        return -2;
    }else{
        return c;
    }


    while(--max>=0){
        c=GETCH();

        if(c==EOF)
            return EOF;

        if(c=='\n'){
            UNGETCH(c);
            return -2;
        }

        if(base==16){
            if(!isxdigit(c))
                break;

            i<<=4;

            if(isdigit(c))
                i+=c-'0';
            else if(i>='a')
                i+=0xa+c-'a';
            else
                i+=0xa+c-'a';

        }else if(base==10){
            if(!isdigit(c))
                break;
            i*=10;
            i+=c-'0';
        }else{
            if(c<'0' || c>'7')
                break;
            i<<=3;
            i+=c-'0';
        }
    }

    if(max>=0)
        UNGETCH(c);

    return i;
}


/* */


static int scan_string(Token *tok, Tokenizer *tokz, bool escapes)
{
    STRING_DECL(s);
    int c;

    while(1){
        c=GETCH();

        if(c=='"')
            break;

        if(c=='\n'){
            UNGETCH(c);
            STRING_FREE(s);
            return E_TOKZ_UNEXPECTED_EOL;
        }

        if(c=='\\' && escapes){
            c=scan_char_escape(tokz);
            if(c==-2){
                STRING_FREE(s);
                return E_TOKZ_UNEXPECTED_EOL;
            }
        }

        if(c==EOF){
            STRING_FREE(s);
            return E_TOKZ_UNEXPECTED_EOF;
        }

        STRING_APPEND(s, c);
    }

    STRING_FINISH(s);

    TOK_SET_STRING(tok, s);

    return 0;
}


/* */


static int scan_char(Token *tok, Tokenizer *tokz)
{
    int c, c2;

    c=GETCH();

    if(c==EOF)
        return E_TOKZ_UNEXPECTED_EOF;

    if(c=='\n')
        return E_TOKZ_UNEXPECTED_EOL;

    if(c=='\\'){
        c=scan_char_escape(tokz);

        if(c==EOF)
            return E_TOKZ_UNEXPECTED_EOF;

        if(c==-2)
            return E_TOKZ_UNEXPECTED_EOL;
    }

    c2=GETCH();

    if(c2!='\'')
        return E_TOKZ_MULTICHAR;

    TOK_SET_CHAR(tok, c);

    return 0;
}


/* */


#define START_IDENT(X) (isalpha(X) || X=='_' || X=='$')


static int scan_identifier(Token *tok, Tokenizer *tokz, int c)
{
    STRING_DECL(s);

    do{
        STRING_APPEND(s, c);
        c=GETCH();
    }while(isalnum(c) || c=='_' || c=='$');

    UNGETCH(c);

    STRING_FINISH(s);

    TOK_SET_IDENT(tok, s);

    return 0;
}

#define NP_SIMPLE_IMPL
#include "np/numparser2.h"
#include "np/np-conv.h"


static int scan_number(Token *tok, Tokenizer *tokz, int c)
{
    NPNum num=NUM_INIT;
    int e;

    if((e=parse_number(&num, tokz, c)))
        return e;

    if(num.type==NPNUM_INT){
        long l;
        if((e=num_to_long(&l, &num, TRUE)))
            return e;

        TOK_SET_LONG(tok, l);
    }else if(num.type==NPNUM_FLOAT){
          double d;
          if((e=num_to_double(&d, &num)))
              return e;

        TOK_SET_DOUBLE(tok, d);
    }else{
        return E_TOKZ_NUMFMT;
    }

    return 0;
}


/* */


static uchar op_map[]={
    0x00,        /* ________ 0-7     */
    0x00,        /* ________ 8-15    */
    0x00,        /* ________ 16-23   */
    0x00,        /* ________ 24-31   */
    0x62,        /* _!___%&_ 32-39   */
    0xff,        /* ()*+,-./ 40-47   */
    0x00,        /* ________ 48-55   */
    0xfc,        /* __:;<=>? 56-63   */
    0x01,        /* @_______ 64-71   */
    0x00,        /* ________ 72-79   */
    0x00,        /* ________ 80-87   */
    0x78,        /* ___[_]^_ 88-95   */
    0x00,        /* ________ 96-103  */
    0x00,        /* ________ 104-111 */
    0x00,        /* ________ 112-119 */
    0x38        /* ___{|}__ 120-127 */
};


static bool map_isset(uchar *map, uint ch)
{
    if(ch>127)
        return FALSE;

    return map[ch>>3]&(1<<(ch&7));
}


static bool is_opch(uint ch)
{
    return map_isset(op_map, ch);
}


static int scan_op(Token *tok, Tokenizer *tokz,  int c)
{
    int c2;
    int op=-1;

    /* Quickly check it is an operator character */
    if(!is_opch(c))
        return E_TOKZ_INVALID_CHAR;

    switch(c){
    case '+':
    case '-':
    case '*':
/*    case '/':     Checked elsewhere */
    case '%':
    case '^':
    case '!':
    case '=':
    case '<':
    case '>':
        c2=GETCH();
        if(c2=='='){
            op=c|(c2<<8);
        }else if(c2==c && (c2!='%' && c2!='!' && c2!='*')){
            if(c=='<' || c=='>'){
                int c3=GETCH();
                if(c3=='='){
                    op=c|(c2<<8)|(c3<<16);
                }else{
                    UNGETCH(c3);
                    op=c|(c2<<8);
                }
            }else{
                op=c|(c2<<8);
            }
        }else{
            UNGETCH(c2);
            op=c;
        }
        break;

    /* It is already known that it is a operator so these are not needed
    case ':':
    case '~':
    case '?':
    case '.':
    case ';';
    case '{':
    case '}':
    case '@':
    case '|':
    case '&':
    */
    default:
        op=c;
    }

    TOK_SET_OP(tok, op);

    return 0;
}


/* */


void tokz_warn(const Tokenizer *tokz, int line, const char *fmt, ...)
{
    va_list args;

    va_start(args, fmt);

    if(tokz!=NULL)
        warn_obj_line_v(tokz->name, line, fmt, args);
    else
        warn(fmt, args);

    va_end(args);
}


void tokz_warn_error(const Tokenizer *tokz, int line, int e)
{
    if(e==E_TOKZ_UNEXPECTED_EOF)
        line=0;

    if(e<0)
        tokz_warn(tokz, line, "%s", strerror(-e));
    else
        tokz_warn(tokz, line, "%s", TR(errors[e]));
}


bool tokz_get_token(Tokenizer *tokz, Token *tok)
{
    int c, c2, e;

    if (!(tokz->flags&TOKZ_READ_FROM_BUFFER))
    assert(tokz->file!=NULL);

    tok_free(tok);

    if(!TOK_IS_INVALID(&(tokz->ungettok))){
        *tok=tokz->ungettok;
        tokz->ungettok.type=TOK_INVALID;
        return TRUE;
    }

    while(1){

        e=0;

        do{
            c=GETCH();
        }while(c!='\n' && c!=EOF && isspace(c));

        tok->line=tokz->line;

        switch(c){
        case EOF:
            TOK_SET_OP(tok, OP_EOF);
            return TRUE;

        case '\n':
            INC_LINE();

            if(tokz->flags&TOKZ_IGNORE_NEXTLINE)
                continue;

            TOK_SET_OP(tok, OP_NEXTLINE);

            return TRUE;

        case '\\':
            do{
                c=GETCH();
                if(c==EOF){
                    TOK_SET_OP(tok, OP_EOF);
                    return FALSE;
                }
                if(!isspace(c) && e==0){
                    e=E_TOKZ_EOL_EXPECTED;
                    tokz_warn_error(tokz, tokz->line, e);
                    if(!(tokz->flags&TOKZ_ERROR_TOLERANT))
                        return FALSE;
                }
            }while(c!='\n');

            INC_LINE();
            continue;

        case '#':
            if(tokz->flags&TOKZ_READ_COMMENTS){
                e=scan_line_comment(tok, tokz);
                break;
            }else if((e=skip_line_comment(tokz))){
                break;
            }

            continue;

        case '/':
            c2=GETCH();

            if(c2=='='){
                TOK_SET_OP(tok, OP_AS_DIV);
                return TRUE;
            }

            if(c2!='*'){
                UNGETCH(c2);
                TOK_SET_OP(tok, OP_DIV);
                return TRUE;
            }

            if(tokz->flags&TOKZ_READ_COMMENTS){
                e=scan_c_comment(tok, tokz);
                break;
            }else if((e=skip_c_comment(tokz))){
                break;
            }

            continue;

        case '\"':
            e=scan_string(tok, tokz, TRUE);
            break;

        case '\'':
            e=scan_char(tok, tokz);
            break;

        default:
            if(('0'<=c && c<='9') || c=='-' || c=='+'){
                e=scan_number(tok, tokz, c);
                break;
            }

             if(START_IDENT(c))
                e=scan_identifier(tok, tokz, c);
            else
                e=scan_op(tok, tokz, c);
        }

        if(!e)
            return TRUE;

        tokz_warn_error(tokz, tokz->line, e);
        return FALSE;
    }
}


void tokz_unget_token(Tokenizer *tokz, Token *tok)
{
    tok_free(&(tokz->ungettok));
    tokz->ungettok=*tok;
    tok->type=TOK_INVALID;
}


/*
 * File open
 */

static bool do_tokz_pushf(Tokenizer *tokz)
{
    Tokenizer_FInfo *finfo;

    finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo,
                    tokz->filestack_n, tokz->filestack_n+1);

    if(finfo==NULL)
        return FALSE;

    tokz->filestack=finfo;
    finfo=&(finfo[tokz->filestack_n++]);

    finfo->file=tokz->file;
    finfo->name=tokz->name;
    finfo->line=tokz->line;
    finfo->ungetc=tokz->ungetc;
    finfo->ungettok=tokz->ungettok;

    return TRUE;
}


bool tokz_pushf_file(Tokenizer *tokz, FILE *file, const char *fname)
{
    char *fname_copy=NULL;

    if(file==NULL)
        return FALSE;

    if(fname!=NULL){
        fname_copy=scopy(fname);
        if(fname_copy==NULL){
            warn_err();
            return FALSE;
        }
    }

    if(tokz->file!=NULL){
        if(!do_tokz_pushf(tokz)){
            warn_err();
            if(fname_copy!=NULL)
                free(fname_copy);
            return FALSE;
        }
    }

    tokz->file=file;
    tokz->name=fname_copy;
    tokz->line=1;
    tokz->ungetc=-1;
    tokz->ungettok.type=TOK_INVALID;

    return TRUE;
}


bool tokz_pushf(Tokenizer *tokz, const char *fname)
{
    FILE *file;

    file=fopen(fname, "r");

    if(file==NULL){
        warn_err_obj(fname);
        return FALSE;
    }

    if(!tokz_pushf_file(tokz, file, fname)){
        fclose(file);
        return FALSE;
    }

    return TRUE;
}



static Tokenizer *tokz_create()
{
    Tokenizer*tokz;

    tokz=ALLOC(Tokenizer);

    if(tokz==NULL){
        warn_err();
        return NULL;
    }

    tokz->file=NULL;
    tokz->name=NULL;
    tokz->line=1;
    tokz->ungetc=-1;
    tokz->ungettok.type=TOK_INVALID;
    tokz->flags=0;
    tokz->optstack=NULL;
    tokz->nest_lvl=0;
    tokz->filestack_n=0;
    tokz->filestack=NULL;
    tokz->buffer.data=0;
    tokz->buffer.len=0;
    tokz->buffer.pos=0;

    return tokz;
}


Tokenizer *tokz_open(const char *fname)
{
    Tokenizer *tokz;

    tokz=tokz_create();

    if(!tokz_pushf(tokz, fname)){
        free(tokz);
        return NULL;
    }

    return tokz;
}


Tokenizer *tokz_open_file(FILE *file, const char *fname)
{
    Tokenizer *tokz;

    tokz=tokz_create();

    if(!tokz_pushf_file(tokz, file, fname)){
        free(tokz);
        return NULL;
    }

    return tokz;
}

Tokenizer *tokz_prepare_buffer(char *buffer, int len)
{
    Tokenizer *tokz;
    char old=0;

    tokz=tokz_create();
    if(len>0){
        old=buffer[len-1];
        buffer[len-1]='\0';
    }

    tokz->flags|=TOKZ_READ_FROM_BUFFER;
    tokz->buffer.data=scopy(buffer);
    tokz->buffer.len=(len>0 ? (uint)len : strlen(tokz->buffer.data));
    tokz->buffer.pos=0;

    if(old>0)
        buffer[len-1]=old;

    return tokz;
}

/*
 * File close
 */

static bool do_tokz_popf(Tokenizer *tokz, bool shrink)
{
    Tokenizer_FInfo *finfo;

    if(tokz->filestack_n<=0)
        return FALSE;

    if(tokz->file!=NULL)
        fclose(tokz->file);
    if(tokz->name!=NULL)
        free(tokz->name);

    finfo=&(tokz->filestack[--tokz->filestack_n]);

    tokz->file=finfo->file;
    tokz->name=finfo->name;
    tokz->line=finfo->line;
    tokz->ungetc=finfo->ungetc;
    tokz->ungettok=finfo->ungettok;

    if(tokz->filestack_n==0){
        free(tokz->filestack);
        tokz->filestack=NULL;
    }else if(shrink){
        finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo,
                        tokz->filestack_n+1, tokz->filestack_n);
        if(finfo==NULL)
            warn_err();
        else
            tokz->filestack=finfo;
    }

    return TRUE;
}


bool tokz_popf(Tokenizer *tokz)
{
    return do_tokz_popf(tokz, TRUE);
}


void tokz_close(Tokenizer *tokz)
{
    while(tokz->filestack_n>0)
        do_tokz_popf(tokz, FALSE);

    if(tokz->file!=NULL)
        fclose(tokz->file);
    if(tokz->name!=NULL)
        free(tokz->name);
    tok_free(&(tokz->ungettok));

    free(tokz);
}



/* */


void tok_free(Token *tok)
{
    if(TOK_IS_STRING(tok) || TOK_IS_IDENT(tok) || TOK_IS_COMMENT(tok)){
        if(TOK_STRING_VAL(tok)!=NULL)
            free(TOK_STRING_VAL(tok));
    }

    tok->type=TOK_INVALID;
}


void tok_init(Token *tok)
{
    static Token dummy=TOK_INIT;

    memcpy(tok, &dummy, sizeof(*tok));
}