%{
/*
    mmorph, MULTEXT morphology tool
    Version 2.3, October 1995
    Copyright (c) 1994,1995 ISSCO/SUISSETRA, Geneva, Switzerland
    Dominique Petitpierre, <petitp@divsun.unige.ch>
*/
/*
     tokenizer for user syntax
     Dominique Petitpierre, ISSCO, July 1994

     TODO: better handling of strings: a la lex, without necessity of
	   quoting when possible.
*/

#include "user.h"
#include "parse.h"
#include "y.tab.h"

#define NUL '\0'

//int	yylineno = 1;
int	linepos = 0;
static char	buf_str[MAXSTRLNG];
static char   *pbuf_str;
int	augmenting = FALSE;
int	allow_string_letters = TRUE; /* true until alphabets are declared */

#ifdef yywrap
#undef yywrap
#endif

#define MAX_INCLUDE_DEPTH 10
static FILE    *file_stack[MAX_INCLUDE_DEPTH];
static int	 line_no_stack[MAX_INCLUDE_DEPTH];
char	*file_name_stack[MAX_INCLUDE_DEPTH];
short	stack_index = -1;
static YY_BUFFER_STATE buffer_state_stack[MAX_INCLUDE_DEPTH];

static int	lexstr();
void	initfile();

#ifdef TEST
#define fatal_error \
    (void) printf("file=%s,line=%d\n",file_name_stack[stack_index],yylineno);\
    (void) printf
#endif

%}

isoalphanum	([a-zA-Z0-9_\241-\377.])
blank           ([ \t\240])
sep	        ([ \t\n\240])
notsep	        ([^ \t\n\240])
comment		(\;.*\n)
		
%x QSTRING INCLUDE ASTRING

%%


\?          { linepos += yyleng; return(ANY);}
\|          { linepos += yyleng; return(BAR);}
"<=>"	    { linepos += yyleng; return(BIARROW);}
\*	    { linepos += yyleng; return(CONCATBOUNDARY);}
\+          { linepos += yyleng; return(MORPHEMEBOUNDARY);}
\-          { linepos += yyleng; return(CONTEXTBOUNDARY);}
\~          { linepos += yyleng; return(WORDBOUNDARY);}
\$          { linepos += yyleng; return(DOLLAR);}
\=          { linepos += yyleng; return(EQUAL);}
\<	    { linepos += yyleng; return(LANGLE);}
"<-"	    { linepos += yyleng; return(LARROW);}
"<="	    { linepos += yyleng; return(COERCEARROW);}
\[          { linepos += yyleng; return(LBRA);}
"!="        { linepos += yyleng; return(NOTEQUAL);}
\>	    { linepos += yyleng; return(RANGLE);}
"=>"	    { linepos += yyleng; return(RARROW);}
\]	    { linepos += yyleng; return(RBRA);}
\:          { linepos += yyleng; return(COLON);}
\/          { linepos += yyleng; return(SLASH);}

\@{blank}*Alphabets  { linepos += yyleng; return(ALPHABETS);}
\@{blank}*Attributes { linepos += yyleng; return(ATTRIBUTES);}
\@{blank}*Classes    { linepos += yyleng; return(CLASSES);}
\@{blank}*Grammar    { linepos += yyleng; return(GRAMMAR);}
\@{blank}*Lexicon    { linepos += yyleng; return(LEXICON);}
\@{blank}*Pairs      { linepos += yyleng; return(PAIRS);}
\@{blank}*Spelling   { linepos += yyleng; return(SPELLING);}
\@{blank}*Types      { linepos += yyleng; return(TYPES);}

{isoalphanum}+/({sep}*{comment})*{sep}*: {
    /* declaration identifier name */
    linepos += yyleng;
    return(lexstr(DNAME,yytext));
    }

\'{notsep}+/{sep}({sep}*{comment})*{sep}*: {
    /* declaration identifier name */
    linepos += yyleng;
    return(lexstr(DNAME,yytext+1));
    }

{isoalphanum}+/({sep}*{comment})*{sep}*\[ {
    /* declaration identifier name */
    linepos += yyleng;
    return(lexstr(TNAME,yytext));
    }

\'{notsep}+/{sep}({sep}*{comment})*{sep}*\[ {
    /* declaration identifier name */
    linepos += yyleng;
    return(lexstr(TNAME,yytext+1));
    }

{isoalphanum}+ { /* identifier name */
    linepos += yyleng;
    return(lexstr(NAME,yytext));
    }

\'{notsep}+ { /* quoted identifier name */
    linepos += yyleng;
    return(lexstr(NAME,yytext+1));
    }


\"  { /* start string */
    linepos += yyleng;
    pbuf_str=buf_str;
    if (allow_string_letters 
        || current_letter_map == NULL) /* alphabets not yet declared */
	BEGIN(ASTRING);
    else
	BEGIN(QSTRING);
    }

<QSTRING>\\\n { /* ignore escaped newline */
    yylineno++;
    linepos=0;
    }

<INCLUDE,ASTRING>\\(0[0-7][0-7]*|.) { /* escape notation */

    unsigned int escaped_char;

    linepos += yyleng;
    if ((pbuf_str - buf_str) >= (MAXSTRLNG - 1)) {
        *pbuf_str=NUL;
        fatal_error("string too long: \"%s\"",buf_str);
        }
    switch (yytext[1]) {
        case 'n':
	    *pbuf_str++='\n';
	    break;
        case 't':
	    *pbuf_str++='\t';
	    break;
	case '0':
	    if (sscanf(yytext+1, "%o", &escaped_char) != 1
		|| escaped_char > MAXUCHAR
		|| escaped_char == 0)
		    fatal_error("%s value \"%s\" is not allowed",
		    "error while reading octal code:",
		    yytext);
	    *pbuf_str++ = (unsigned char) escaped_char;
	    break;
        default: 
	    *pbuf_str++=yytext[1];
	    break;
    }
    }

<QSTRING>\\(0[0-7][0-7]*|.) { /* escape notation */

     /* temp storage, because yytext does not like to be changed with \n */
    unsigned int   escaped_char;

    linepos += yyleng;
    if ((pbuf_str - buf_str) >= (MAXSTRLNG - 1)) {
	fatal_error("string too long"); /* TODO: print the string */
	}
    switch (yytext[1]) {
	case 'n':
	    escaped_char='\n';
	    break;
	case 't':
	    escaped_char='\t';
	    break;
	case '0':
	    if (sscanf(yytext+1, "%o", &escaped_char) != 1
		|| escaped_char > MAXUCHAR
		|| escaped_char == 0)
		    fatal_error("%s value \"%s\" is not allowed",
		    "error while reading octal code:",
		    yytext);
	    break;
	default:
	    escaped_char=yytext[1];
	    break;
    }
    if ((*pbuf_str++ = current_letter_map[escaped_char]) == 0 )
	fatal_error("symbol \"%c\" is not a %s",
		    escaped_char,
		    symbol_kind_str[current_letter_kind]);
    }

<ASTRING>\&{isoalphanum}+\; { /* long letter name a la SGML */
    linepos += yyleng;
    fatal_error("Sorry, %s \"%s\"",
		"cannot declare an alphabet symbol with such a name:",
		yytext);
    }

<QSTRING>\&{isoalphanum}+\; { /* long letter name a la SGML */
    t_str letter_name;

    linepos += yyleng;
    if ((pbuf_str - buf_str) >= (MAXSTRLNG - 1)) {
	*pbuf_str=NUL;
	fatal_error("string too long"); /* TODO: print string */
	}
	MY_STRDUP(letter_name, yytext + 1); /* find_letter will free it */
	letter_name[yyleng - 2] = '\0';
	*pbuf_str++ = *find_letter(letter_name,current_letter_kind);
    }

<QSTRING,INCLUDE,ASTRING>\n {
    fatal_error("newline not allowed in string; use \\n if you mean one");
    }

<ASTRING>\" {
    linepos += yyleng;
    *pbuf_str=NUL;
    BEGIN(INITIAL);
    return(lexstr(STRING,buf_str));
    }

<QSTRING>\" {
    linepos += yyleng;
    *pbuf_str=NUL;
    BEGIN(INITIAL);
    return(lexstr(LSTRING,buf_str));
    }

<INCLUDE>\"{blank}*/\n {
    linepos += yyleng;
    *pbuf_str=NUL;
    BEGIN(INITIAL);
    initfile(buf_str);
    }

<INCLUDE,ASTRING>. {
    linepos += yyleng;
    if ((pbuf_str - buf_str) >= (MAXSTRLNG - 1)) {
	*pbuf_str=NUL;
	fatal_error("string too long: \"%s\"",buf_str);
	}
    *pbuf_str++ = *yytext;
    }

<QSTRING>. {
    linepos += yyleng;
    if ((pbuf_str - buf_str) >= (MAXSTRLNG - 1)) {
	fatal_error("string too long"); /* TODO print string */
	}
    if ((*pbuf_str++ = current_letter_map[(unsigned char) *yytext]) == 0 )
	fatal_error("symbol \"%c\" is not a %s",
		    *yytext,
		    symbol_kind_str[current_letter_kind]);
    }

{blank} { /* separator blank */
    linepos += yyleng;
    }

\n  { /* separator newline */
    yylineno++;
    linepos=0;
    }

{comment}    { /* comment */
    yylineno++;
    linepos=0;
    }

\#{blank}*include{blank}+\" {
    linepos += yyleng;
    pbuf_str=buf_str;
    BEGIN(INCLUDE);
    }


.           { /* error */
    linepos += yyleng;
    fatal_error("unexpected character: '%s'",yytext);
    }

<<EOF>>     {
    /*
   if ( ! end_parse)  {
	fatal_error("unexpected end of file");
    */
	yyterminate();
    }

%%


void
initfile(file_name)
char *file_name;

{
    char **including;

    if (stack_index >= MAX_INCLUDE_DEPTH - 1) {
       fatal_error(
	   "cannot include file \"%s\": too many nested include (maximum %u)",
	   file_name,MAX_INCLUDE_DEPTH);
    }
    stack_index++;
    if (file_name == NULL) {
	file_stack[stack_index] = stdin;
	file_name = "standard input";
	file_name_stack[stack_index] = file_name;
    } else
	if ((file_stack[stack_index] = fopen(file_name,"r")) == NULL) {
	    fatal_error("Cannot open file %s",file_name);
	}
	else {
	    MY_STRDUP(file_name_stack[stack_index],file_name);
	}
    for (including = file_name_stack;
         strcmp(file_name,*including) != 0;
	 including++);
    if (including != &file_name_stack[stack_index])
	fatal_error("#include recursion with file \"%s\"",file_name);
    yyin= file_stack[stack_index];
    if (stack_index) {
	line_no_stack[stack_index-1] = yylineno;
	buffer_state_stack[stack_index-1] = YY_CURRENT_BUFFER;
        yy_switch_to_buffer(yy_create_buffer(file_stack[stack_index],
	                                     YY_BUF_SIZE));
    }
    yylineno = 1;
    linepos = 0;
}

/* yywrap is called when an end of file is reached */
int
yywrap()

{
    if (augmenting
	&& strcmp(file_name_stack[stack_index], augment_file_name) == 0) {
	/* end of augment file */
	while(stack_index > 0 ) { /* close all including files */
	    (void) fclose(file_stack[stack_index]);
	    if (file_stack[stack_index] != stdin )
		MY_FREE(file_name_stack[stack_index]);
	    yy_delete_buffer(YY_CURRENT_BUFFER);
	    stack_index--;
	    yy_switch_to_buffer( buffer_state_stack[stack_index] );
	}
    }
    (void) fclose(file_stack[stack_index]);
    if (file_stack[stack_index] != stdin )
	MY_FREE(file_name_stack[stack_index]);
    stack_index--;
    if (stack_index < 0)
	return(1); /* no more files to read from, triggers <<EOF>> */
    else {
	yy_delete_buffer(YY_CURRENT_BUFFER);
	yy_switch_to_buffer( buffer_state_stack[stack_index] );
	yylineno = line_no_stack[stack_index];
	linepos=0;
	return(0); /* continue in enclosing file */
    }
}


#ifdef TEST
main( argc, argv )
int argc;
char **argv;

{
    int r;

    ++argv, --argc;  /* skip over program name */
    if ( argc > 0 )
            initfile(argv[0]);
    else
            initfile(NULL);

    while (r=yylex())
	(void) printf("%d\n",r);
}
#endif

static int
lexstr(token,string)
    int token;
    char *string;
{
#ifdef TEST
    (void) printf("%s\n",string);
#else
    if (token == STRING) {
	MY_STRDUP(yylval.string,string);
    }
    else {
	MY_STRDUP(yylval.letter_string,string);
    }
#endif
    return(token);
}
/* other procedures are in file "main.c" */