/*
 *  Lexer defines.
 */

#if !defined(DUK_LEXER_H_INCLUDED)
#define DUK_LEXER_H_INCLUDED

typedef void (*duk_re_range_callback)(void *user, duk_codepoint_t r1, duk_codepoint_t r2, duk_bool_t direct);

/*
 *  A token is interpreted as any possible production of InputElementDiv
 *  and InputElementRegExp, see E5 Section 7 in its entirety.  Note that
 *  the E5 "Token" production does not cover all actual tokens of the
 *  language (which is explicitly stated in the specification, Section 7.5).
 *  Null and boolean literals are defined as part of both ReservedWord
 *  (E5 Section 7.6.1) and Literal (E5 Section 7.8) productions.  Here,
 *  null and boolean values have literal tokens, and are not reserved
 *  words.
 *
 *  Decimal literal negative/positive sign is -not- part of DUK_TOK_NUMBER.
 *  The number tokens always have a non-negative value.  The unary minus
 *  operator in "-1.0" is optimized during compilation to yield a single
 *  negative constant.
 *
 *  Token numbering is free except that reserved words are required to be
 *  in a continuous range and in a particular order.  See genstrings.py.
 */

#define DUK_LEXER_INITCTX(ctx) duk_lexer_initctx((ctx))

#define DUK_LEXER_SETPOINT(ctx, pt) duk_lexer_setpoint((ctx), (pt))

#define DUK_LEXER_GETPOINT(ctx, pt) duk_lexer_getpoint((ctx), (pt))

/* Currently 6 characters of lookup are actually needed (duk_lexer.c). */
#define DUK_LEXER_WINDOW_SIZE 6
#if defined(DUK_USE_LEXER_SLIDING_WINDOW)
#define DUK_LEXER_BUFFER_SIZE 64
#endif

#define DUK_TOK_MINVAL 0

/* returned after EOF (infinite amount) */
#define DUK_TOK_EOF 0

/* identifier names (E5 Section 7.6) */
#define DUK_TOK_IDENTIFIER 1

/* reserved words: keywords */
#define DUK_TOK_START_RESERVED 2
#define DUK_TOK_BREAK          2
#define DUK_TOK_CASE           3
#define DUK_TOK_CATCH          4
#define DUK_TOK_CONTINUE       5
#define DUK_TOK_DEBUGGER       6
#define DUK_TOK_DEFAULT        7
#define DUK_TOK_DELETE         8
#define DUK_TOK_DO             9
#define DUK_TOK_ELSE           10
#define DUK_TOK_FINALLY        11
#define DUK_TOK_FOR            12
#define DUK_TOK_FUNCTION       13
#define DUK_TOK_IF             14
#define DUK_TOK_IN             15
#define DUK_TOK_INSTANCEOF     16
#define DUK_TOK_NEW            17
#define DUK_TOK_RETURN         18
#define DUK_TOK_SWITCH         19
#define DUK_TOK_THIS           20
#define DUK_TOK_THROW          21
#define DUK_TOK_TRY            22
#define DUK_TOK_TYPEOF         23
#define DUK_TOK_VAR            24
#define DUK_TOK_CONST          25
#define DUK_TOK_VOID           26
#define DUK_TOK_WHILE          27
#define DUK_TOK_WITH           28

/* reserved words: future reserved words */
#define DUK_TOK_CLASS   29
#define DUK_TOK_ENUM    30
#define DUK_TOK_EXPORT  31
#define DUK_TOK_EXTENDS 32
#define DUK_TOK_IMPORT  33
#define DUK_TOK_SUPER   34

/* "null", "true", and "false" are always reserved words.
 * Note that "get" and "set" are not!
 */
#define DUK_TOK_NULL  35
#define DUK_TOK_TRUE  36
#define DUK_TOK_FALSE 37

/* reserved words: additional future reserved words in strict mode */
#define DUK_TOK_START_STRICT_RESERVED 38 /* inclusive */
#define DUK_TOK_IMPLEMENTS            38
#define DUK_TOK_INTERFACE             39
#define DUK_TOK_LET                   40
#define DUK_TOK_PACKAGE               41
#define DUK_TOK_PRIVATE               42
#define DUK_TOK_PROTECTED             43
#define DUK_TOK_PUBLIC                44
#define DUK_TOK_STATIC                45
#define DUK_TOK_YIELD                 46

#define DUK_TOK_END_RESERVED 47 /* exclusive */

/* "get" and "set" are tokens but NOT ReservedWords.  They are currently
 * parsed and identifiers and these defines are actually now unused.
 */
#define DUK_TOK_GET 47
#define DUK_TOK_SET 48

/* punctuators (unlike the spec, also includes "/" and "/=") */
#define DUK_TOK_LCURLY     49
#define DUK_TOK_RCURLY     50
#define DUK_TOK_LBRACKET   51
#define DUK_TOK_RBRACKET   52
#define DUK_TOK_LPAREN     53
#define DUK_TOK_RPAREN     54
#define DUK_TOK_PERIOD     55
#define DUK_TOK_SEMICOLON  56
#define DUK_TOK_COMMA      57
#define DUK_TOK_LT         58
#define DUK_TOK_GT         59
#define DUK_TOK_LE         60
#define DUK_TOK_GE         61
#define DUK_TOK_EQ         62
#define DUK_TOK_NEQ        63
#define DUK_TOK_SEQ        64
#define DUK_TOK_SNEQ       65
#define DUK_TOK_ADD        66
#define DUK_TOK_SUB        67
#define DUK_TOK_MUL        68
#define DUK_TOK_DIV        69
#define DUK_TOK_MOD        70
#define DUK_TOK_EXP        71
#define DUK_TOK_INCREMENT  72
#define DUK_TOK_DECREMENT  73
#define DUK_TOK_ALSHIFT    74 /* named "arithmetic" because result is signed */
#define DUK_TOK_ARSHIFT    75
#define DUK_TOK_RSHIFT     76
#define DUK_TOK_BAND       77
#define DUK_TOK_BOR        78
#define DUK_TOK_BXOR       79
#define DUK_TOK_LNOT       80
#define DUK_TOK_BNOT       81
#define DUK_TOK_LAND       82
#define DUK_TOK_LOR        83
#define DUK_TOK_QUESTION   84
#define DUK_TOK_COLON      85
#define DUK_TOK_EQUALSIGN  86
#define DUK_TOK_ADD_EQ     87
#define DUK_TOK_SUB_EQ     88
#define DUK_TOK_MUL_EQ     89
#define DUK_TOK_DIV_EQ     90
#define DUK_TOK_MOD_EQ     91
#define DUK_TOK_EXP_EQ     92
#define DUK_TOK_ALSHIFT_EQ 93
#define DUK_TOK_ARSHIFT_EQ 94
#define DUK_TOK_RSHIFT_EQ  95
#define DUK_TOK_BAND_EQ    96
#define DUK_TOK_BOR_EQ     97
#define DUK_TOK_BXOR_EQ    98

/* literals (E5 Section 7.8), except null, true, false, which are treated
 * like reserved words (above).
 */
#define DUK_TOK_NUMBER 99
#define DUK_TOK_STRING 100
#define DUK_TOK_REGEXP 101

#define DUK_TOK_MAXVAL 101 /* inclusive */

#define DUK_TOK_INVALID DUK_SMALL_UINT_MAX

/* Convert heap string index to a token (reserved words) */
#define DUK_STRIDX_TO_TOK(x) ((x) -DUK_STRIDX_START_RESERVED + DUK_TOK_START_RESERVED)

/* Sanity check */
#if (DUK_TOK_MAXVAL > 255)
#error DUK_TOK_MAXVAL too large, code assumes it fits into 8 bits
#endif

/* Sanity checks for string and token defines */
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_BREAK) != DUK_TOK_BREAK)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_CASE) != DUK_TOK_CASE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_CATCH) != DUK_TOK_CATCH)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_CONTINUE) != DUK_TOK_CONTINUE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_DEBUGGER) != DUK_TOK_DEBUGGER)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_DEFAULT) != DUK_TOK_DEFAULT)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_DELETE) != DUK_TOK_DELETE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_DO) != DUK_TOK_DO)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_ELSE) != DUK_TOK_ELSE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_FINALLY) != DUK_TOK_FINALLY)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_FOR) != DUK_TOK_FOR)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_LC_FUNCTION) != DUK_TOK_FUNCTION)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_IF) != DUK_TOK_IF)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_IN) != DUK_TOK_IN)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_INSTANCEOF) != DUK_TOK_INSTANCEOF)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_NEW) != DUK_TOK_NEW)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_RETURN) != DUK_TOK_RETURN)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_SWITCH) != DUK_TOK_SWITCH)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_THIS) != DUK_TOK_THIS)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_THROW) != DUK_TOK_THROW)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_TRY) != DUK_TOK_TRY)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_TYPEOF) != DUK_TOK_TYPEOF)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_VAR) != DUK_TOK_VAR)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_VOID) != DUK_TOK_VOID)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_WHILE) != DUK_TOK_WHILE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_WITH) != DUK_TOK_WITH)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_CLASS) != DUK_TOK_CLASS)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_CONST) != DUK_TOK_CONST)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_ENUM) != DUK_TOK_ENUM)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_EXPORT) != DUK_TOK_EXPORT)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_EXTENDS) != DUK_TOK_EXTENDS)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_IMPORT) != DUK_TOK_IMPORT)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_SUPER) != DUK_TOK_SUPER)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_LC_NULL) != DUK_TOK_NULL)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_TRUE) != DUK_TOK_TRUE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_FALSE) != DUK_TOK_FALSE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_IMPLEMENTS) != DUK_TOK_IMPLEMENTS)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_INTERFACE) != DUK_TOK_INTERFACE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_LET) != DUK_TOK_LET)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_PACKAGE) != DUK_TOK_PACKAGE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_PRIVATE) != DUK_TOK_PRIVATE)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_PROTECTED) != DUK_TOK_PROTECTED)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_PUBLIC) != DUK_TOK_PUBLIC)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_STATIC) != DUK_TOK_STATIC)
#error mismatch in token defines
#endif
#if (DUK_STRIDX_TO_TOK(DUK_STRIDX_YIELD) != DUK_TOK_YIELD)
#error mismatch in token defines
#endif

/* Regexp tokens */
#define DUK_RETOK_EOF                           0
#define DUK_RETOK_DISJUNCTION                   1
#define DUK_RETOK_QUANTIFIER                    2
#define DUK_RETOK_ASSERT_START                  3
#define DUK_RETOK_ASSERT_END                    4
#define DUK_RETOK_ASSERT_WORD_BOUNDARY          5
#define DUK_RETOK_ASSERT_NOT_WORD_BOUNDARY      6
#define DUK_RETOK_ASSERT_START_POS_LOOKAHEAD    7
#define DUK_RETOK_ASSERT_START_NEG_LOOKAHEAD    8
#define DUK_RETOK_ATOM_PERIOD                   9
#define DUK_RETOK_ATOM_CHAR                     10
#define DUK_RETOK_ATOM_DIGIT                    11 /* assumptions in regexp compiler */
#define DUK_RETOK_ATOM_NOT_DIGIT                12 /* -""- */
#define DUK_RETOK_ATOM_WHITE                    13 /* -""- */
#define DUK_RETOK_ATOM_NOT_WHITE                14 /* -""- */
#define DUK_RETOK_ATOM_WORD_CHAR                15 /* -""- */
#define DUK_RETOK_ATOM_NOT_WORD_CHAR            16 /* -""- */
#define DUK_RETOK_ATOM_BACKREFERENCE            17
#define DUK_RETOK_ATOM_START_CAPTURE_GROUP      18
#define DUK_RETOK_ATOM_START_NONCAPTURE_GROUP   19
#define DUK_RETOK_ATOM_START_CHARCLASS          20
#define DUK_RETOK_ATOM_START_CHARCLASS_INVERTED 21
#define DUK_RETOK_ATOM_END_GROUP                22

/* Constants for duk_lexer_ctx.buf. */
#define DUK_LEXER_TEMP_BUF_LIMIT 256

/* A token value.  Can be memcpy()'d, but note that slot1/slot2 values are on the valstack.
 * Some fields (like num, str1, str2) are only valid for specific token types and may have
 * stale values otherwise.
 */
struct duk_token {
	duk_small_uint_t t; /* token type (with reserved word identification) */
	duk_small_uint_t t_nores; /* token type (with reserved words as DUK_TOK_IDENTIFER) */
	duk_double_t num; /* numeric value of token */
	duk_hstring *str1; /* string 1 of token (borrowed, stored to ctx->slot1_idx) */
	duk_hstring *str2; /* string 2 of token (borrowed, stored to ctx->slot2_idx) */
	duk_size_t start_offset; /* start byte offset of token in lexer input */
	duk_int_t start_line; /* start line of token (first char) */
	duk_int_t num_escapes; /* number of escapes and line continuations (for directive prologue) */
	duk_bool_t lineterm; /* token was preceded by a lineterm */
	duk_bool_t allow_auto_semi; /* token allows automatic semicolon insertion (eof or preceded by newline) */
};

#define DUK_RE_QUANTIFIER_INFINITE ((duk_uint32_t) 0xffffffffUL)

/* A regexp token value. */
struct duk_re_token {
	duk_small_uint_t t; /* token type */
	duk_small_uint_t greedy;
	duk_uint32_t num; /* numeric value (character, count) */
	duk_uint32_t qmin;
	duk_uint32_t qmax;
};

/* A structure for 'snapshotting' a point for rewinding */
struct duk_lexer_point {
	duk_size_t offset;
	duk_int_t line;
};

/* Lexer codepoint with additional info like offset/line number */
struct duk_lexer_codepoint {
	duk_codepoint_t codepoint;
	duk_size_t offset;
	duk_int_t line;
};

/* Lexer context.  Same context is used for ECMAScript and Regexp parsing. */
struct duk_lexer_ctx {
#if defined(DUK_USE_LEXER_SLIDING_WINDOW)
	duk_lexer_codepoint *window; /* unicode code points, window[0] is always next, points to 'buffer' */
	duk_lexer_codepoint buffer[DUK_LEXER_BUFFER_SIZE];
#else
	duk_lexer_codepoint window[DUK_LEXER_WINDOW_SIZE]; /* unicode code points, window[0] is always next */
#endif

	duk_hthread *thr; /* thread; minimizes argument passing */

	const duk_uint8_t *input; /* input string (may be a user pointer) */
	duk_size_t input_length; /* input byte length */
	duk_size_t input_offset; /* input offset for window leading edge (not window[0]) */
	duk_int_t input_line; /* input linenumber at input_offset (not window[0]), init to 1 */

	duk_idx_t slot1_idx; /* valstack slot for 1st token value */
	duk_idx_t slot2_idx; /* valstack slot for 2nd token value */
	duk_idx_t buf_idx; /* valstack slot for temp buffer */
	duk_hbuffer_dynamic *buf; /* temp accumulation buffer */
	duk_bufwriter_ctx bw; /* bufwriter for temp accumulation */

	duk_int_t token_count; /* number of tokens parsed */
	duk_int_t token_limit; /* maximum token count before error (sanity backstop) */

	duk_small_uint_t flags; /* lexer flags, use compiler flag defines for now */
};

/*
 *  Prototypes
 */

DUK_INTERNAL_DECL void duk_lexer_initctx(duk_lexer_ctx *lex_ctx);

DUK_INTERNAL_DECL void duk_lexer_getpoint(duk_lexer_ctx *lex_ctx, duk_lexer_point *pt);
DUK_INTERNAL_DECL void duk_lexer_setpoint(duk_lexer_ctx *lex_ctx, duk_lexer_point *pt);

DUK_INTERNAL_DECL
void duk_lexer_parse_js_input_element(duk_lexer_ctx *lex_ctx, duk_token *out_token, duk_bool_t strict_mode, duk_bool_t regexp_mode);
#if defined(DUK_USE_REGEXP_SUPPORT)
DUK_INTERNAL_DECL void duk_lexer_parse_re_token(duk_lexer_ctx *lex_ctx, duk_re_token *out_token);
DUK_INTERNAL_DECL void duk_lexer_parse_re_ranges(duk_lexer_ctx *lex_ctx, duk_re_range_callback gen_range, void *userdata);
#endif /* DUK_USE_REGEXP_SUPPORT */

#endif /* DUK_LEXER_H_INCLUDED */