File: cpp_token_c.ml

package info (click to toggle)
coccinelle 1.0.8.deb-5
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 26,148 kB
  • sloc: ml: 136,392; ansic: 23,594; sh: 2,189; makefile: 2,157; perl: 1,576; lisp: 840; python: 823; awk: 70; csh: 12
file content (661 lines) | stat: -rw-r--r-- 23,457 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
open Common

module TH = Token_helpers

open Parser_c
open Token_views_c

(*****************************************************************************)
(* Prelude *)
(*****************************************************************************)

(* cpp functions working at the token level. Cf cpp_ast_c for cpp functions
 * working at the AST level (which is very unusual but makes sense in
 * the coccinelle context for instance).
 *
 * Note that as I use a single lexer  to work both at the C and cpp level
 * there are some inconveniencies.
 * For instance 'for' is a valid name for a macro parameter and macro
 * body, but is interpreted in a special way by our single lexer, and
 * so at some places where I expect a TIdent I need also to
 * handle special cases and accept Tfor, Tif, etc at those places.
 *
 * There are multiple issues related to those keywords incorrect tokens.
 * Those keywords can be:
 *   - (1) in the name of the macro as  in  #define inline
 *   - (2) in a parameter of the macro as in #define foo(char)   char x;
 *   - (3) in an argument to a macro call as in   IDENT(if);
 * Case 1 is easy to fix in define_ident.
 * Case 2 is easy to fix in define_parse where detect such toks in
 * the parameter and then replace their occurrence in the body in a Tident.
 * Case 3 is only an issue when the expanded token is not really use
 * as usual but use for instance in concatenation as in  a ## if
 * when expanded. In the case the grammar this time will not be happy
 * so this is also easy to fix in cpp_engine.
 *
 *)

(*****************************************************************************)
(* Wrappers *)
(*****************************************************************************)
let pr2, pr2_once = Common.mk_pr2_wrappers Flag_parsing_c.verbose_parsing

(*****************************************************************************)
(* Types *)
(*****************************************************************************)

(* ------------------------------------------------------------------------- *)
(* mimic standard.h *)
(* ------------------------------------------------------------------------- *)

type define_def = string * define_param * define_body
 and define_param =
   | NoParam
   | Params of define_arg list
 and define_arg = FixedArg of string | VariadicArg of string
 and define_body =
   | DefineBody of Parser_c.token list
   | DefineHint of parsinghack_hint

   and parsinghack_hint =
     | HintIterator
     | HintDeclarator
     | HintMacroString
     | HintMacroStatement
     | HintAttribute
     | HintEndAttribute
     | HintMacroIdentBuilder


(*****************************************************************************)
(* Parsing and helpers of hints  *)
(*****************************************************************************)

(* cf also data/test.h *)
let assoc_hint_string = [
  "YACFE_ITERATOR"   , HintIterator;
  "YACFE_DECLARATOR" , HintDeclarator;
  "YACFE_STRING"     , HintMacroString;
  "YACFE_STATEMENT"  , HintMacroStatement;
  "YACFE_ATTRIBUTE"  , HintAttribute;
  "YACFE_END_ATTRIBUTE" , HintEndAttribute;
  "YACFE_IDENT_BUILDER" , HintMacroIdentBuilder;

  "MACROSTATEMENT"   , HintMacroStatement; (* backward compatibility *)
]


let (parsinghack_hint_of_string: string -> parsinghack_hint option) = fun s ->
  Common.assoc_option s assoc_hint_string
let (string_of_parsinghack_hint: parsinghack_hint -> string) = fun hint ->
  let assoc' = assoc_hint_string +> List.map (fun (a,b) -> (b,a) ) in
  Common.assoc hint assoc'



let (is_parsinghack_hint: string -> bool) = fun s ->
  parsinghack_hint_of_string s <> None

let (token_from_parsinghack_hint:
     (string * Ast_c.info) -> parsinghack_hint -> Parser_c.token) =
 fun (s,ii) hint ->
   match hint with
   | HintIterator ->
       Parser_c.TMacroIterator (s, ii)
   | HintDeclarator ->
       Parser_c.TMacroDecl (s, ii)
   | HintMacroString ->
       Parser_c.TMacroString (s, ii)
   | HintMacroStatement ->
       Parser_c.TMacroStmt (s, ii)
   | HintAttribute ->
       Parser_c.TMacroAttr (s, ii)
   | HintEndAttribute -> failwith "not supported"
   | HintMacroIdentBuilder ->
       Parser_c.TMacroIdentBuilder (s, ii)


(* used in extract_macros for example *)
let string_of_define_def (s, params, body) =

  let s1 =
    match params with
    | NoParam ->
        spf "#define %s " s
    | Params xs ->
	let xs = List.map (function FixedArg s -> s | VariadicArg s -> s) xs in
        spf "#define %s(%s) " s (String.concat "," xs)
  in
  let s2 =
    match body with
    | DefineHint hint ->
        string_of_parsinghack_hint hint
    | DefineBody xs ->
        String.concat " " (xs +> List.map Token_helpers.str_of_tok)
  in
  s1 ^ s2


(*****************************************************************************)
(* Expansion helpers *)
(*****************************************************************************)

(* In some cases we can have macros like IDENT(if) that expands to some
 * 'int xxx_if(void)', but as the lexer will currently generate a Tif for
 * the expanded code, that may not be accepted as a token after a ##
 * in the grammar. Hence this function to remap some tokens. This is because
 * we should not use a single lexer for both working at the C level and
 * cpp level.
 *
 * update: it can also rename some TypedefIdent into TIdent, possibly
 * because of bad interaction with add_typedef_root in parsing_hacks.
 *)
let rec remap_keyword_tokens xs =
  match xs with
  | [] -> []
  | [x] -> [x]
  | x::y::xs ->
      (match x, y with
      | Parser_c.TCppConcatOp _, Parser_c.TIdent _ ->
          x::y::remap_keyword_tokens xs
      | Parser_c.TIdent _, Parser_c.TCppConcatOp _ ->
          x::y::remap_keyword_tokens xs

      | Parser_c.TCppConcatOp (i1),   y ->
          let s = TH.str_of_tok y in
          let ii = TH.info_of_tok y in
          if s ==~ Common.regexp_alpha
          then begin
            pr2 (spf "remapping: %s to an ident in expanded code" s);
            x::(Parser_c.TIdent (s, ii))::remap_keyword_tokens xs
          end
          else
            x::y::remap_keyword_tokens xs

      | x, Parser_c.TCppConcatOp (i1) ->
          let s = TH.str_of_tok x in
          let ii = TH.info_of_tok x in
          if s ==~ Common.regexp_alpha
          then begin
            pr2 (spf "remapping: %s to an ident in expanded code" s);
            (Parser_c.TIdent (s, ii))::remap_keyword_tokens (y::xs)
          end
          else
            x::y::remap_keyword_tokens xs

      | _, _ ->
          x::remap_keyword_tokens (y::xs)
      )


(* works with agglomerate_concat_op_ident below *)
let rec get_ident_in_concat_op xs =
  match xs with
  | [] ->
      pr2_once "weird: ident after ## operator not found";
      "", []
  | [x] ->
      (match x with
      | Parser_c.TIdent (s, i1) -> s, []
      | _ ->
          pr2_once "weird: ident after ## operator not found";
          "", [x]
      )
  | x::y::xs ->
      (match x, y with
      | Parser_c.TIdent (s,i1), Parser_c.TCppConcatOp (i2) ->
          let (s2, rest) = get_ident_in_concat_op xs in
          s ^ s2, rest
      | Parser_c.TIdent (s, i1), _ ->
          s, (y::xs)
      | _ ->
          pr2_once "weird: ident after ## operator not found";
          "", x::y::xs
      )

(* must be run after the expansion has been done for the parameter so
 * that all idents are actually ident, not macro parameter names.
 *)
let rec agglomerate_concat_op_ident xs =
  match xs with
  | [] -> []
  | [x] -> [x]
  | x::y::xs ->
      (* can we have ## id, and so ## as first token ? yes
       * but the semantic is different as it represents variadic
       * names so this must be handled elsewhere.
       *)
      (match x, y with
      | Parser_c.TIdent (s,i1), Parser_c.TCppConcatOp (i2) ->
          let (all_str_ident, rest_toks) =
            get_ident_in_concat_op xs
          in
          let new_s = s ^ all_str_ident in
          let i1' = Ast_c.rewrap_str new_s i1 in
          Parser_c.TIdent (new_s, i1')::agglomerate_concat_op_ident rest_toks
      | Parser_c.TCppConcatOp _, _ ->
          pr2_once "weird, ## alone";
          x::agglomerate_concat_op_ident (y::xs)
      | _ ->
          x::agglomerate_concat_op_ident (y::xs)

      )



(* To expand the parameter of the macro. The env corresponds to the actual
 * code that is binded to the parameters of the macro.
 * Recurse ? fixpoint ? the expansion may also contain macro.
 * Or to macro expansion in a strict manner, that is process first
 * the parameters, expands macro in params, and then process enclosing
 * macro call.
 *
 * note: do the concatenation job of a##b here ?
 * normally this should be done in the grammar. Here just expand
 * tokens. The only thing we handle here is we may have to remap
 * some tokens.
 *
 * todo: handle stringification here ? if #n
 *
 * todo? but could parsing_hacks then pass over the remapped tokens,
 * for instance transform some of the back into some TypedefIdent
 * so cpp_engine may be fooled?
 *)
let (cpp_engine:
          ?evaluate_concatop:bool ->
          (string , Parser_c.token list) assoc ->
          Parser_c.token list -> Parser_c.token list) =
 fun ?(evaluate_concatop=true) env xs ->
  xs +> List.map (fun tok ->
    (* expand only TIdent ? no cos the parameter of the macro
     * can actually be some 'register' so may have to look for
     * any tokens candidates for the expansion.
     * Only subtelity is maybe don't expand the TDefineIdent.
     *
     * update: in fact now the caller (define_parse) will have done
     * the job right and already replaced the macro parameter with a TIdent.
     *)
    match tok with
    | TIdent (s,i1) when List.mem_assoc s env ->
	Common.assoc s env
    | x -> [x]
  )
  +> List.flatten
  +> remap_keyword_tokens
  +> (fun xs ->
       if evaluate_concatop
       then agglomerate_concat_op_ident xs
       else xs
  )



(* ------------------------------------------------------------------------- *)
(* apply macro, using standard.h or other defs *)
(* ------------------------------------------------------------------------- *)

(* Thanks to this function many stuff are not anymore hardcoded in ocaml code.
 * At some point there were hardcoded in a standard.h file but now I
 * can even generate them on the fly on demand when there is actually
 * a parsing problem.
 *
 * No need to take care to not substitute the macro name itself
 * that occurs in the macro definition because the macro name is
 * after fix_token_define a TDefineIdent, no more a TIdent.
 *)

let around parens (err_line,err_col) =
  match parens with (* can contain only one element, if open with no close *)
    lp::_ -> (* parens also contains commas *)
      let rp = List.hd (List.rev parens) in
      let lp_line = lp.Token_views_c.line in
      let lp_col = lp.Token_views_c.col in
      let rp_line = rp.Token_views_c.line in
      let rp_col = rp.Token_views_c.col in
      if (lp_line = rp_line && lp_col <= rp_col) || lp_line < rp_line
      then
	(lp_line < err_line || (lp_line = err_line && lp_col < err_col)) &&
	(rp_line > err_line || (rp_line = err_line && rp_col > err_col))
      else failwith "unexpected paren order"
  | _ -> false

let at tok (err_line,err_col) =
  let tok_line = tok.Token_views_c.line in
  let tok_col = tok.Token_views_c.col in
  tok_line = err_line && tok_col = err_col

(* This applies the macro in the current definition and all subsequent ones.
This is unfortunate when the macro itself introduces a parse error, but it
seems that we don't know where the current function ends. *)

let apply_macro_defs
 ~msg_apply_known_macro
 ~msg_apply_known_macro_hint
 ?evaluate_concatop
 ?(inplace_when_single=true)
 defs pos xs =

 let rec apply_macro_defs dynamic_macs pos xs =
  match xs with
  | [] -> ()

  (* old: "but could do more, could reuse same original token
   * so that have in the Ast a Dbg, not a MACROSTATEMENT"
   *
   *   | PToken ({tok = TIdent (s,i1)} as id)::xs
   *     when s = "MACROSTATEMENT" ->
   *
   *     msg_macro_statement_hint s;
   *     id.tok <- TMacroStmt(TH.info_of_tok id.tok);
   *     find_macro_paren xs
   *
   *  let msg_macro_statement_hint s =
   *    incr Stat.nMacroHint;
   *   ()
   *
   *)

  (* recognized macro of standard.h (or other) *)
  | PToken ({tok = TIdent (s,i1)} as id)::Parenthised (xxs,info_parens)::xs
      when Hashtbl.mem defs s ->

      msg_apply_known_macro s;
      let (s, params, body) = Hashtbl.find defs s in

      (match params with
      | NoParam ->
          pr2 ("WEIRD: macro without param used before parenthize: " ^ s);
          (* ex: PRINTP("NCR53C400 card%s detected\n" ANDP(((struct ... *)

          (match body with
          | DefineBody bodymacro ->
              set_as_comment (Token_c.CppMacro) id;
              id.new_tokens_before <- bodymacro;
          | DefineHint hint ->
              msg_apply_known_macro_hint s;
              id.tok <- token_from_parsinghack_hint (s,i1) hint;
          )
      | Params params ->
          (match body with
          | DefineBody bodymacro ->

              (* bugfix: better to put this that before the match body,
		 * cos our macrostatement hint can have variable number of
		 * arguments and so it's ok if it does not match exactly
		 * the number of arguments. *)
	      let build_binder params xxs =
		let rec loop = function
		    ([],[]) -> Some (function [] -> [] | _ -> failwith "bad")
		  | ([],[[]]) -> Some (function [[]] -> [] | _ -> failwith "bad")
		  | ([],l) -> None
		  | ([(VariadicArg s)],l) ->
		      Some (function l -> List.map (function a -> (s,a)) l)
		  | ((VariadicArg _)::_,l) -> None
		  | ((FixedArg _)::_,[]) -> None
		  | ((FixedArg s)::rest,x::xs) ->
		      (match loop (rest,xs) with
			Some k ->
			  Some (function l -> (s,(List.hd l)) :: k (List.tl l))
		      |	None -> None) in
		loop (params, xxs) in
	      (match build_binder params xxs with
		None ->
                  pr2_once
		    ("WEIRD: macro with wrong number of arguments: " ^ s);
                  (* old: id.new_tokens_before <- bodymacro; *)

                  (* update: if wrong number, then I just pass this macro *)
                  [Parenthised (xxs, info_parens)] +>
                  iter_token_paren (set_as_comment Token_c.CppMacro);
                  set_as_comment Token_c.CppMacro id
	      |	Some bind ->

                  let xxs' = xxs +> List.map (fun x ->
		    (tokens_of_paren_ordered x) +> List.map (fun x ->
		      TH.visitor_info_of_tok Ast_c.make_expanded x.tok
			)
		      ) in
                  id.new_tokens_before <-
                      (* !!! cpp expansion job here  !!! *)
		    cpp_engine ?evaluate_concatop
		      (bind xxs') bodymacro;

                      (* important to do that after have apply the macro,
			 otherwise will pass as argument to the macro some
			 tokens that are all TCommentCpp
                      *)
                  [Parenthised (xxs, info_parens)] +>
                  iter_token_paren (set_as_comment Token_c.CppMacro);
                  set_as_comment Token_c.CppMacro id)

          | DefineHint (HintMacroStatement as hint) ->
                (* important to do that after have apply the macro, otherwise
                   * will pass as argument to the macro some tokens that
                 * are all TCommentCpp
                 *
                 * note: such macrostatement can have a variable number of
                 * arguments but here we don't care, we just pass all the
                 * parameters.
                 *)

                (match xs with
                | PToken ({tok = TPtVirg _} as id2)::_ ->
                    pr2_once
                      ("macro stmt with trailing ';', passing also ';' for: "^
                       s);
                    (* sometimes still want pass its params ... as in
                     *  DEBUGPOLL(static unsigned int prev_mask = 0);
                     *)

                    msg_apply_known_macro_hint s;
                    id.tok <- token_from_parsinghack_hint (s,i1) hint;
                    [Parenthised (xxs, info_parens)] +>
                      iter_token_paren (set_as_comment Token_c.CppMacro);
                    set_as_comment Token_c.CppMacro id2;

                | _ ->
                    msg_apply_known_macro_hint s;
                    id.tok <- token_from_parsinghack_hint (s,i1) hint;
                    [Parenthised (xxs, info_parens)] +>
                      iter_token_paren (set_as_comment Token_c.CppMacro);
                )


            | DefineHint hint ->
                msg_apply_known_macro_hint s;
                id.tok <- token_from_parsinghack_hint (s,i1) hint;
            )
      );
      apply_macro_defs dynamic_macs pos xs

  | PToken {tok = TIdent (s,i1)}::Parenthised (xxs,info_parens)::xs
    when List.mem s dynamic_macs ||
         not(Hashtbl.mem defs s) && List.exists (around info_parens) pos ->
      (* drop out the argument list if it contains a parse error *)
      let commas =
	match info_parens with
	  _::rest ->
	    (match List.rev rest with
	      _::commas -> commas
	    | [] -> [])
	| [] -> [] in
      msg_apply_known_macro s;
      [Parenthised (xxs, commas)]
	+> iter_token_paren (set_as_comment Token_c.CppMacro);
      let dynamic_macs =
	if List.mem s dynamic_macs then dynamic_macs else s::dynamic_macs in
      apply_macro_defs dynamic_macs pos xs

  | ((PToken ({tok = TDefine i} as t1)) as start)::xs
    when List.exists (at t1) pos ->
      let (in_def,end_and_after) =
	Common.span (function PToken {tok = TDefEOL _} -> false | _ -> true)
	  xs in
      (match end_and_after with
	((PToken {tok = TDefEOL _}) as t2)::xs ->
	  (* not reusable, like macros, so only comments out one #define
	     per round *)
	  [start] +> iter_token_paren (set_as_comment Token_c.CppMacro);
	  in_def +> iter_token_paren (set_as_comment Token_c.CppMacro);
	  [t2] +> iter_token_paren (set_as_comment Token_c.CppMacro);
	  msg_apply_known_macro "#define";
	  let pos =
	    let mypos = (t1.Token_views_c.line,t1.Token_views_c.col) in
	    List.filter (function x -> not (x = mypos)) pos in
	  apply_macro_defs dynamic_macs pos xs
      | _ -> failwith "#define with no end")

  | PToken ({tok = TIdent (s,i1)} as id)::
    PToken ({tok = TPtVirg _} | {tok = TEq _})::xs
      when List.mem s !Flag.cocci_attribute_names ->
	id.tok <- TMacroEndAttr (s, i1);
	apply_macro_defs dynamic_macs pos xs

  | PToken ({tok = TIdent (s,i1)} as id)::xs
      when List.mem s !Flag.cocci_attribute_names ->
	id.tok <- TMacroAttr (s, i1);
	apply_macro_defs dynamic_macs pos xs

  | PToken ({tok = TIdent (s,i1)} as id)::xs
      when Hashtbl.mem defs s ->

      msg_apply_known_macro s;
      let (_s, params, body) = Hashtbl.find defs s in

      (match params with
      | Params _ ->
          pr2 ("WEIRD: macro with params but no parens found: " ^ s);
          (* don't apply the macro, perhaps a redefinition *)
          ()
      | NoParam ->
          (match body with
          (* bugfix: we prefer not using this special case when we come
           * from extract_macros context
           *)
          | DefineBody [newtok] when inplace_when_single ->
             (* special case when 1-1 substitution, we reuse the token *)
              id.tok <- (newtok +> TH.visitor_info_of_tok (fun _ ->
                TH.info_of_tok id.tok))
          | DefineBody bodymacro ->
              set_as_comment Token_c.CppMacro id;
              id.new_tokens_before <- bodymacro;
          | DefineHint hint ->
                msg_apply_known_macro_hint s;
                id.tok <- token_from_parsinghack_hint (s,i1) hint;
          )
      );
      apply_macro_defs dynamic_macs pos xs

  (* recurse *)
  | (PToken x)::xs -> apply_macro_defs dynamic_macs pos xs
  | (Parenthised (xxs, info_parens))::xs ->
      xxs +> List.iter (apply_macro_defs dynamic_macs pos);
      apply_macro_defs dynamic_macs pos xs
 in
 apply_macro_defs [] pos xs



(*****************************************************************************)
(* extracting define_def from a standard.h  *)
(*****************************************************************************)
(* was the cpp-builtin, standard.h, part 0 *)

let macro_body_to_maybe_hint body =
  match body with
  | [] -> DefineBody body
  | [TIdent (s,i1)] ->
      (match parsinghack_hint_of_string s with
      | Some hint -> DefineHint hint
      | None -> DefineBody body
      )
  | xs -> DefineBody body

exception Bad_param

let rec (define_parse: Parser_c.token list -> (string * define_def) list) =
 fun xs ->
  match xs with
  | [] -> []
  | TDefine i1::TIdentDefine (s,i2)::TOParDefine i3::xs ->
      (* note: the macro could be badly written and have no closing ')' for
       * its param, which would make us go too far away, but I don't think
       * it's important to handle such an error *)
      let def =
	try
	  let (tokparams, _, xs) =
            xs +> Common.split_when (function TCPar _ -> true | _ -> false) in
	  let (body, _, xs) =
            xs +> Common.split_when (function TDefEOL _ -> true | _ -> false) in
	  let params =
            tokparams +> Common.map_filter (function
              |	 TComma _ -> None
              |	 TIdent (s, _) -> Some (FixedArg s)

              (* TODO *)
              |	 TDefParamVariadic (s, _) -> Some (VariadicArg s)
              (* TODO *)
              |	 TEllipsis _ -> Some (VariadicArg "...")

              |	 x ->
              (* bugfix: param of macros can be tricky *)
                  let s = TH.str_of_tok x in
                  if s ==~ Common.regexp_alpha
                  then begin
                    pr2 (spf "remapping: %s to a macro parameter" s);
                    Some (FixedArg s)
                  end
                  else
                    begin
                      pr2 (spf "bad character %s in macro parameter list" s);
                      raise Bad_param
                    end) in
          (* bugfix: also substitute to ident in body so cpp_engine will
             * have an easy job.
          *)
	  let body = body +> List.map (fun tok ->
            match tok with
            | TIdent _ -> tok
            | _ ->
		let s = TH.str_of_tok tok in
		let ii = TH.info_of_tok tok in
		let params =
		  List.map
		    (function FixedArg s -> s | VariadicArg s -> s)
		    params in
		if s ==~ Common.regexp_alpha && List.mem s params
		then begin
		  pr2 (spf "remapping: %s to an ident in macro body" s);
		  TIdent (s, ii)
		end
		else tok) +>
	    List.map (TH.visitor_info_of_tok Ast_c.make_expanded) in
	  Some (s, (s, Params params, macro_body_to_maybe_hint body))
	with Bad_param -> None in
      (match def with
	Some def -> def::define_parse xs
      |	None -> define_parse xs)

  | TDefine i1::TIdentDefine (s,i2)::xs ->
      let (body, _, xs) =
        xs +> Common.split_when (function TDefEOL _ -> true | _ -> false) in
      let body = body +> List.map
        (TH.visitor_info_of_tok Ast_c.make_expanded) in
      let def = (s, (s, NoParam, macro_body_to_maybe_hint body)) in
      def::define_parse xs

  (* cf tests-bis/define_plus.c *)
  | TDefine i1::xs ->
      let line = Ast_c.line_of_info i1 in
      pr2 (spf "WEIRD: no ident in define at line %d" line);
      define_parse xs

  | x::xs -> define_parse xs



let extract_macros xs =
  let cleaner = xs +> List.filter (fun x ->
    not (TH.is_comment x)
  ) in
  define_parse cleaner