1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
(* camlp5r *)
(* plexing.mli,v *)
(* Copyright (c) INRIA 2007-2017 *)
(** Lexing for Camlp5 grammars.
This module defines the Camlp5 lexer type to be used in extensible
grammars (see module [Grammar]). It also provides some useful functions
to create lexers. *)
type pattern = string * string;;
(* Type for values used by the generated code of the EXTEND
statement to represent terminals in entry rules.
- The first string is the constructor name (must start with
an uppercase character). When it is empty, the second string
is supposed to be a keyword.
- The second string is the constructor parameter. Empty if it
has no parameter (corresponding to the 'wildcard' pattern).
- The way tokens patterns are interpreted to parse tokens is done
by the lexer, function [tok_match] below. *)
exception Error of string;;
(** A lexing error exception to be used by lexers. *)
(** location array, used by lexer to find locations based on token-index *)
module Locations :
sig
type t = { locations : Ploc.t option array ref; overflow : bool ref };;
val create : unit -> t;;
val lookup : t -> int -> Ploc.t;;
val add : t -> int -> Ploc.t -> unit;;
end
;;
(** Lexer type *)
type 'te lexer =
{ tok_func : 'te lexer_func;
tok_using : pattern -> unit;
tok_removing : pattern -> unit;
mutable tok_match : pattern -> 'te -> string;
tok_text : pattern -> string;
mutable tok_comm : Ploc.t list option;
kwds : (string, string) Hashtbl.t }
and 'te lexer_func = char Stream.t -> 'te Stream.t * Locations.t
and location_function = int -> Ploc.t;;
(** The type of a function giving the location of a token in the
source from the token number in the stream (starting from zero). *)
val lexer_text : pattern -> string;;
(** A simple [tok_text] function. *)
val default_match : pattern -> string * string -> string;;
(** A simple [tok_match] function, appling to the token type
[(string * string)] *)
(** Lexers from parsers or ocamllex
The functions below create lexer functions either from a [char stream]
parser or for an [ocamllex] function. With the returned function [f],
it is possible to get a simple lexer (of the type [Plexing.glexer] above):
{[
{ Plexing.tok_func = f;
Plexing.tok_using = (fun _ -> ());
Plexing.tok_removing = (fun _ -> ());
Plexing.tok_match = Plexing.default_match;
Plexing.tok_text = Plexing.lexer_text;
Plexing.tok_comm = None }
]}
Note that a better [tok_using] function should check the used tokens
and raise [Plexing.Error] for incorrect ones. The other functions
[tok_removing], [tok_match] and [tok_text] may have other implementations
as well. *)
val lexer_func_of_parser :
(char Stream.t * int ref * int ref -> 'te * Ploc.t) -> 'te lexer_func;;
(** A lexer function from a lexer written as a char stream parser
returning the next token and its location. The two references
with the char stream contain the current line number and the
position of the beginning of the current line. *)
val lexer_func_of_ocamllex_located :
(Lexing.lexbuf -> 'te * Ploc.t) -> 'te lexer_func;;
(** A lexer function from a lexer created by [ocamllex],
but that returns camlp5 locations, too. This can be used to get comments properly
transmitted to the parser.
*)
val lexer_func_of_ocamllex : (Lexing.lexbuf -> 'te) -> 'te lexer_func;;
(** A lexer function from a lexer created by [ocamllex] *)
(** Function to build a stream and a location function *)
val make_stream_and_location :
(unit -> 'te * Ploc.t) -> 'te Stream.t * Locations.t;;
(** General function *)
(** Useful functions and values *)
val eval_char : string -> char;;
val eval_string : Ploc.t -> string -> string;;
(** Convert a char or a string token, where the backslashes had not
been interpreted into a real char or string; raise [Failure] if
bad backslash sequence found; [Plexing.eval_char (Char.escaped c)]
would return [c] and [Plexing.eval_string (String.escaped s)] would
return [s] *)
val restore_lexing_info : (int * int) option ref;;
val input_file : string ref;;
val line_nb : int ref ref;;
val bol_pos : int ref ref;;
(** Special variables used to reinitialize line numbers and position
of beginning of line with their correct current values when a parser
is called several times with the same character stream. Necessary
for directives (e.g. #load or #use) which interrupt the parsing.
Without usage of these variables, locations after the directives
can be wrong. *)
(** The lexing buffer used by streams lexers *)
module Lexbuf :
sig
type t;;
val empty : t;;
val add : char -> t -> t;;
val get : t -> string;;
end
;;
|
