File: antlr3baserecognizer.c

package info (click to toggle)
libantlr3c 3.4+dfsg-3
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 2,604 kB
  • sloc: ansic: 11,828; sh: 10,382; makefile: 99; cpp: 17
file content (2235 lines) | stat: -rw-r--r-- 68,666 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
/** \file
 * Contains the base functions that all recognizers require.
 * Any function can be overridden by a lexer/parser/tree parser or by the
 * ANTLR3 programmer.
 * 
 * \addtogroup pANTLR3_BASE_RECOGNIZER
 * @{
 */
#include    <antlr3baserecognizer.h>

// [The "BSD licence"]
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
// http://www.temporal-wave.com
// http://www.linkedin.com/in/jimidle
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifdef	ANTLR3_WINDOWS
#pragma warning( disable : 4100 )
#endif

/* Interface functions -standard implementations cover parser and treeparser
 * almost completely but are overridden by the parser or tree parser as needed. Lexer overrides
 * most of these functions.
 */
static void					beginResync					(pANTLR3_BASE_RECOGNIZER recognizer);
static pANTLR3_BITSET		computeErrorRecoverySet	    (pANTLR3_BASE_RECOGNIZER recognizer);
static void					endResync					(pANTLR3_BASE_RECOGNIZER recognizer);
static void					beginBacktrack				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level);
static void					endBacktrack				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level, ANTLR3_BOOLEAN successful);

static void *				match						(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow);
static void					matchAny					(pANTLR3_BASE_RECOGNIZER recognizer);
static void					mismatch					(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow);
static ANTLR3_BOOLEAN		mismatchIsUnwantedToken		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, ANTLR3_UINT32 ttype);
static ANTLR3_BOOLEAN		mismatchIsMissingToken		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, pANTLR3_BITSET_LIST follow);
static void					reportError					(pANTLR3_BASE_RECOGNIZER recognizer);
static pANTLR3_BITSET		computeCSRuleFollow			(pANTLR3_BASE_RECOGNIZER recognizer);
static pANTLR3_BITSET		combineFollows				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_BOOLEAN exact);
static void					displayRecognitionError	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames);
static void					recover						(pANTLR3_BASE_RECOGNIZER recognizer);
static void	*				recoverFromMismatchedToken  (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow);
static void	*				recoverFromMismatchedSet    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST follow);
static ANTLR3_BOOLEAN		recoverFromMismatchedElement(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST follow);
static void					consumeUntil				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 tokenType);
static void					consumeUntilSet				(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET set);
static pANTLR3_STACK		getRuleInvocationStack	    (pANTLR3_BASE_RECOGNIZER recognizer);
static pANTLR3_STACK		getRuleInvocationStackNamed (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 name);
static pANTLR3_HASH_TABLE	toStrings					(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_HASH_TABLE);
static ANTLR3_MARKER		getRuleMemoization			(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_INTKEY ruleIndex, ANTLR3_MARKER ruleParseStart);
static ANTLR3_BOOLEAN		alreadyParsedRule			(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex);
static void					memoize						(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex, ANTLR3_MARKER ruleParseStart);
static ANTLR3_BOOLEAN		synpred						(pANTLR3_BASE_RECOGNIZER recognizer, void * ctx, void (*predicate)(void * ctx));
static void					reset						(pANTLR3_BASE_RECOGNIZER recognizer);
static void					freeBR						(pANTLR3_BASE_RECOGNIZER recognizer);
static void *				getCurrentInputSymbol		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream);
static void *				getMissingSymbol			(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM	istream, pANTLR3_EXCEPTION	e,
															ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow);
static ANTLR3_UINT32		getNumberOfSyntaxErrors		(pANTLR3_BASE_RECOGNIZER recognizer);

ANTLR3_API pANTLR3_BASE_RECOGNIZER
antlr3BaseRecognizerNew(ANTLR3_UINT32 type, ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state)
{
    pANTLR3_BASE_RECOGNIZER recognizer;

    // Allocate memory for the structure
    //
    recognizer	    = (pANTLR3_BASE_RECOGNIZER) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_BASE_RECOGNIZER));

    if	(recognizer == NULL)
    {
		// Allocation failed
		//
		return	NULL;
    }

	
	// If we have been supplied with a pre-existing recognizer state
	// then we just install it, otherwise we must create one from scratch
	//
	if	(state == NULL)
	{
		recognizer->state = (pANTLR3_RECOGNIZER_SHARED_STATE) ANTLR3_CALLOC(1, (size_t)sizeof(ANTLR3_RECOGNIZER_SHARED_STATE));

		if	(recognizer->state == NULL)
		{
			ANTLR3_FREE(recognizer);
			return	NULL;
		}

		// Initialize any new recognizer state
		//
		recognizer->state->errorRecovery	= ANTLR3_FALSE;
		recognizer->state->lastErrorIndex	= -1;
		recognizer->state->failed		= ANTLR3_FALSE;
		recognizer->state->errorCount		= 0;
		recognizer->state->backtracking		= 0;
		recognizer->state->following		= NULL;
		recognizer->state->ruleMemo		= NULL;
		recognizer->state->tokenNames		= NULL;
		recognizer->state->sizeHint             = sizeHint;
		recognizer->state->tokSource		= NULL;
                recognizer->state->tokFactory           = NULL;

		// Rather than check to see if we must initialize
		// the stack every time we are asked for an new rewrite stream
		// we just always create an empty stack and then just
		// free it when the base recognizer is freed.
		//
		recognizer->state->rStreams		= antlr3VectorNew(0);  // We don't know the size.

		if	(recognizer->state->rStreams == NULL)
		{
			// Out of memory
			//
			ANTLR3_FREE(recognizer->state);
			ANTLR3_FREE(recognizer);
			return	NULL;
		}
	}
	else
	{
		// Install the one we were given, and do not reset it here
		// as it will either already have been initialized or will
		// be in a state that needs to be preserved.
		//
		recognizer->state = state;
	}
		
    // Install the BR API
    //
    recognizer->alreadyParsedRule           = alreadyParsedRule;
    recognizer->beginResync                 = beginResync;
    recognizer->combineFollows              = combineFollows;
    recognizer->beginBacktrack              = beginBacktrack;
    recognizer->endBacktrack                = endBacktrack;
    recognizer->computeCSRuleFollow         = computeCSRuleFollow;
    recognizer->computeErrorRecoverySet     = computeErrorRecoverySet;
    recognizer->consumeUntil                = consumeUntil;
    recognizer->consumeUntilSet             = consumeUntilSet;
    recognizer->displayRecognitionError     = displayRecognitionError;
    recognizer->endResync                   = endResync;
    recognizer->exConstruct                 = antlr3MTExceptionNew;
    recognizer->getRuleInvocationStack      = getRuleInvocationStack;
    recognizer->getRuleInvocationStackNamed = getRuleInvocationStackNamed;
    recognizer->getRuleMemoization          = getRuleMemoization;
    recognizer->match                       = match;
    recognizer->matchAny                    = matchAny;
    recognizer->memoize                     = memoize;
    recognizer->mismatch                    = mismatch;
    recognizer->mismatchIsUnwantedToken     = mismatchIsUnwantedToken;
    recognizer->mismatchIsMissingToken      = mismatchIsMissingToken;
    recognizer->recover                     = recover;
    recognizer->recoverFromMismatchedElement= recoverFromMismatchedElement;
    recognizer->recoverFromMismatchedSet    = recoverFromMismatchedSet;
    recognizer->recoverFromMismatchedToken  = recoverFromMismatchedToken;
    recognizer->getNumberOfSyntaxErrors     = getNumberOfSyntaxErrors;
    recognizer->reportError                 = reportError;
    recognizer->reset                       = reset;
    recognizer->synpred                     = synpred;
    recognizer->toStrings                   = toStrings;
    recognizer->getCurrentInputSymbol       = getCurrentInputSymbol;
    recognizer->getMissingSymbol            = getMissingSymbol;
    recognizer->debugger                    = NULL;

    recognizer->free = freeBR;

    /* Initialize variables
     */
    recognizer->type			= type;


    return  recognizer;
}
static void	
freeBR	    (pANTLR3_BASE_RECOGNIZER recognizer)
{
    pANTLR3_EXCEPTION thisE;

	// Did we have a state allocated?
	//
	if	(recognizer->state != NULL)
	{
		// Free any rule memoization we set up
		//
		if	(recognizer->state->ruleMemo != NULL)
		{
			recognizer->state->ruleMemo->free(recognizer->state->ruleMemo);
			recognizer->state->ruleMemo = NULL;
		}

		// Free any exception space we have left around
		//
		thisE = recognizer->state->exception;
		if	(thisE != NULL)
		{
			thisE->freeEx(thisE);
		}

		// Free any rewrite streams we have allocated
		//
		if	(recognizer->state->rStreams != NULL)
		{
			recognizer->state->rStreams->free(recognizer->state->rStreams);
		}

		// Free up any token factory we created (error recovery for instance)
		//
		if	(recognizer->state->tokFactory != NULL)
		{
			recognizer->state->tokFactory->close(recognizer->state->tokFactory);
		}
		// Free the shared state memory
		//
		ANTLR3_FREE(recognizer->state);
	}

	// Free the actual recognizer space
	//
    ANTLR3_FREE(recognizer);
}

/**
 * Creates a new Mismatched Token Exception and inserts in the recognizer
 * exception stack.
 * 
 * \param recognizer
 * Context pointer for this recognizer
 * 
 */
ANTLR3_API	void
antlr3MTExceptionNew(pANTLR3_BASE_RECOGNIZER recognizer)
{
    /* Create a basic recognition exception structure
     */
    antlr3RecognitionExceptionNew(recognizer);

    /* Now update it to indicate this is a Mismatched token exception
     */
    recognizer->state->exception->name		= ANTLR3_MISMATCHED_EX_NAME;
    recognizer->state->exception->type		= ANTLR3_MISMATCHED_TOKEN_EXCEPTION;

    return;
}

ANTLR3_API	void
antlr3RecognitionExceptionNew(pANTLR3_BASE_RECOGNIZER recognizer)
{
	pANTLR3_EXCEPTION				ex;
	pANTLR3_LEXER					lexer;
	pANTLR3_PARSER					parser;
	pANTLR3_TREE_PARSER				tparser;

	pANTLR3_INPUT_STREAM			ins;
	pANTLR3_INT_STREAM				is;
	pANTLR3_COMMON_TOKEN_STREAM	    cts;
	pANTLR3_TREE_NODE_STREAM	    tns;

	ins	    = NULL;
	cts	    = NULL;
	tns	    = NULL;
	is	    = NULL;
	lexer   = NULL;
	parser  = NULL;
	tparser = NULL;

	switch	(recognizer->type)
	{
	case	ANTLR3_TYPE_LEXER:

		lexer	= (pANTLR3_LEXER) (recognizer->super);
		ins	= lexer->input;
		is	= ins->istream;

		break;

	case	ANTLR3_TYPE_PARSER:

		parser  = (pANTLR3_PARSER) (recognizer->super);
		cts	= (pANTLR3_COMMON_TOKEN_STREAM)(parser->tstream->super);
		is	= parser->tstream->istream;

		break;

	case	ANTLR3_TYPE_TREE_PARSER:

		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
		tns	= tparser->ctnstream->tnstream;
		is	= tns->istream;

		break;

	default:

		ANTLR3_FPRINTF(stderr, "Base recognizer function antlr3RecognitionExceptionNew called by unknown parser type - provide override for this function\n");
		return;

		break;
	}

	/* Create a basic exception structure
	 */
	ex = antlr3ExceptionNew(ANTLR3_RECOGNITION_EXCEPTION,
		(void *)ANTLR3_RECOGNITION_EX_NAME,
		NULL,
		ANTLR3_FALSE);

	/* Rest of information depends on the base type of the 
	 * input stream.
	 */
	switch  (is->type & ANTLR3_INPUT_MASK)
	{
	case    ANTLR3_CHARSTREAM:

		ex->c			= is->_LA		    	(is, 1);					/* Current input character			*/
		ex->line		= ins->getLine			(ins);						/* Line number comes from stream		*/
		ex->charPositionInLine	= ins->getCharPositionInLine	(ins);	    /* Line offset also comes from the stream   */
		ex->index		= is->index			(is);
		ex->streamName		= ins->fileName;
		ex->message		= "Unexpected character";
		break;

	case    ANTLR3_TOKENSTREAM:

		ex->token		= cts->tstream->_LT						(cts->tstream, 1);	    /* Current input token			    */
		ex->line		= ((pANTLR3_COMMON_TOKEN)(ex->token))->getLine			(ex->token);
		ex->charPositionInLine	= ((pANTLR3_COMMON_TOKEN)(ex->token))->getCharPositionInLine	(ex->token);
		ex->index		= cts->tstream->istream->index					(cts->tstream->istream);
		if	(((pANTLR3_COMMON_TOKEN)(ex->token))->type == ANTLR3_TOKEN_EOF)
		{
			ex->streamName		= NULL;
		}
		else
		{
			ex->streamName		= ((pANTLR3_COMMON_TOKEN)(ex->token))->input->fileName;
		}
		ex->message		= "Unexpected token";
		break;

	case    ANTLR3_COMMONTREENODE:

		ex->token		= tns->_LT						    (tns, 1);	    /* Current input tree node			    */
		ex->line		= ((pANTLR3_BASE_TREE)(ex->token))->getLine		    (ex->token);
		ex->charPositionInLine	= ((pANTLR3_BASE_TREE)(ex->token))->getCharPositionInLine   (ex->token);
		ex->index		= tns->istream->index					    (tns->istream);

		// Are you ready for this? Deep breath now...
		//
		{
			pANTLR3_COMMON_TREE tnode;

			tnode		= ((pANTLR3_COMMON_TREE)(((pANTLR3_BASE_TREE)(ex->token))->super));

			if	(tnode->token    == NULL)
			{
				ex->streamName = ((pANTLR3_BASE_TREE)(ex->token))->strFactory->newStr(((pANTLR3_BASE_TREE)(ex->token))->strFactory, (pANTLR3_UINT8)"-unknown source-");
			}
			else
			{
				if	(tnode->token->input == NULL)
				{
					ex->streamName		= NULL;
				}
				else
				{
					ex->streamName		= tnode->token->input->fileName;
				}
			}
			ex->message		= "Unexpected node";
		}
		break;
	}

	ex->input						= is;
	ex->nextException				= recognizer->state->exception;	/* So we don't leak the memory */
	recognizer->state->exception	= ex;
	recognizer->state->error	    = ANTLR3_TRUE;	    /* Exception is outstanding	*/

	return;
}


/// Match current input symbol against ttype.  Upon error, do one token
/// insertion or deletion if possible.  
/// To turn off single token insertion or deletion error
/// recovery, override mismatchRecover() and have it call
/// plain mismatch(), which does not recover.  Then any error
/// in a rule will cause an exception and immediate exit from
/// rule.  Rule would recover by resynchronizing to the set of
/// symbols that can follow rule ref.
///
static void *
match(	pANTLR3_BASE_RECOGNIZER recognizer,
		ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow)
{
    pANTLR3_PARSER			parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;
	void					* matchedSymbol;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			is	= parser->tstream->istream;

			break;

		case	ANTLR3_TYPE_TREE_PARSER:

			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
			parser	= NULL;
			is	= tparser->ctnstream->tnstream->istream;

			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function 'match' called by unknown parser type - provide override for this function\n");
			return ANTLR3_FALSE;

			break;
    }

	// Pick up the current input token/node for assignment to labels
	//
	matchedSymbol = recognizer->getCurrentInputSymbol(recognizer, is);

    if	(is->_LA(is, 1) == ttype)
    {
		// The token was the one we were told to expect
		//
		is->consume(is);									// Consume that token from the stream
		recognizer->state->errorRecovery	= ANTLR3_FALSE;	// Not in error recovery now (if we were)
		recognizer->state->failed			= ANTLR3_FALSE;	// The match was a success
		return matchedSymbol;								// We are done
    }

    // We did not find the expected token type, if we are backtracking then
    // we just set the failed flag and return.
    //
    if	(recognizer->state->backtracking > 0)
    {
		// Backtracking is going on
		//
		recognizer->state->failed  = ANTLR3_TRUE;
		return matchedSymbol;
	}

    // We did not find the expected token and there is no backtracking
    // going on, so we mismatch, which creates an exception in the recognizer exception
    // stack.
    //
	matchedSymbol = recognizer->recoverFromMismatchedToken(recognizer, ttype, follow);
    return matchedSymbol;
}

/// Consumes the next token, whatever it is, and resets the recognizer state
/// so that it is not in error.
///
/// \param recognizer
/// Recognizer context pointer
///
static void
matchAny(pANTLR3_BASE_RECOGNIZER recognizer)
{
    pANTLR3_PARSER	    parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			is	= parser->tstream->istream;

			break;

		case	ANTLR3_TYPE_TREE_PARSER:

			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
			parser	= NULL;
			is	= tparser->ctnstream->tnstream->istream;

			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function 'matchAny' called by unknown parser type - provide override for this function\n");
			return;

		break;
    }
    recognizer->state->errorRecovery	= ANTLR3_FALSE;
    recognizer->state->failed		    = ANTLR3_FALSE;
    is->consume(is);

    return;
}
///
///
static ANTLR3_BOOLEAN
mismatchIsUnwantedToken(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, ANTLR3_UINT32 ttype)
{
	ANTLR3_UINT32 nextt;

	nextt = is->_LA(is, 2);

	if	(nextt == ttype)
	{
		if	(recognizer->state->exception != NULL)
		{
			recognizer->state->exception->expecting = nextt;
		}
		return ANTLR3_TRUE;		// This token is unknown, but the next one is the one we wanted
	}
	else
	{
		return ANTLR3_FALSE;	// Neither this token, nor the one following is the one we wanted
	}
}

///
///
static ANTLR3_BOOLEAN
mismatchIsMissingToken(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, pANTLR3_BITSET_LIST follow)
{
	ANTLR3_BOOLEAN	retcode;
	pANTLR3_BITSET	followClone;
	pANTLR3_BITSET	viableTokensFollowingThisRule;

	if	(follow == NULL)
	{
		// There is no information about the tokens that can follow the last one
		// hence we must say that the current one we found is not a member of the 
		// follow set and does not indicate a missing token. We will just consume this
		// single token and see if the parser works it out from there.
		//
		return	ANTLR3_FALSE;
	}

	followClone						= NULL;
	viableTokensFollowingThisRule	= NULL;

	// The C bitset maps are laid down at compile time by the
	// C code generation. Hence we cannot remove things from them
	// and so on. So, in order to remove EOR (if we need to) then
	// we clone the static bitset.
	//
	followClone = antlr3BitsetLoad(follow);
	if	(followClone == NULL)
	{
		return ANTLR3_FALSE;
	}

	// Compute what can follow this grammar reference
	//
	if	(followClone->isMember(followClone, ANTLR3_EOR_TOKEN_TYPE))
	{
		// EOR can follow, but if we are not the start symbol, we
		// need to remove it.
		//
		if	(recognizer->state->following->vector->count >= 0)
		{
			followClone->remove(followClone, ANTLR3_EOR_TOKEN_TYPE);
		}

		// Now compute the visiable tokens that can follow this rule, according to context
		// and make them part of the follow set.
		//
		viableTokensFollowingThisRule = recognizer->computeCSRuleFollow(recognizer);
		followClone->borInPlace(followClone, viableTokensFollowingThisRule);
	}

	/// if current token is consistent with what could come after set
	/// then we know we're missing a token; error recovery is free to
	/// "insert" the missing token
	///
	/// BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR
	/// in follow set to indicate that the fall of the start symbol is
	/// in the set (EOF can follow).
	///
	if	(		followClone->isMember(followClone, is->_LA(is, 1))
			||	followClone->isMember(followClone, ANTLR3_EOR_TOKEN_TYPE)
		)
	{
		retcode = ANTLR3_TRUE;
	}
	else
	{
		retcode	= ANTLR3_FALSE;
	}

	if	(viableTokensFollowingThisRule != NULL)
	{
		viableTokensFollowingThisRule->free(viableTokensFollowingThisRule);
	}
	if	(followClone != NULL)
	{
		followClone->free(followClone);
	}

	return retcode;

}

/// Factor out what to do upon token mismatch so tree parsers can behave
/// differently.  Override and call mismatchRecover(input, ttype, follow)
/// to get single token insertion and deletion.  Use this to turn off
/// single token insertion and deletion. Override mismatchRecover
/// to call this instead.
///
/// \remark mismatch only works for parsers and must be overridden for anything else.
///
static	void
mismatch(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow)
{
    pANTLR3_PARSER	    parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    // Install a mismatched token exception in the exception stack
    //
    antlr3MTExceptionNew(recognizer);
    recognizer->state->exception->expecting    = ttype;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			is	= parser->tstream->istream;

			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function 'mismatch' called by unknown parser type - provide override for this function\n");
			return;

			break;
    }

	if	(mismatchIsUnwantedToken(recognizer, is, ttype))
	{
		// Create a basic recognition exception structure
		//
	    antlr3RecognitionExceptionNew(recognizer);
		
		// Now update it to indicate this is an unwanted token exception
		//
		recognizer->state->exception->name		= ANTLR3_UNWANTED_TOKEN_EXCEPTION_NAME;
		recognizer->state->exception->type		= ANTLR3_UNWANTED_TOKEN_EXCEPTION;

		return;
	}
	
	if	(mismatchIsMissingToken(recognizer, is, follow))
	{
		// Create a basic recognition exception structure
		//
	    antlr3RecognitionExceptionNew(recognizer);
		
		// Now update it to indicate this is an unwanted token exception
		//
		recognizer->state->exception->name		= ANTLR3_MISSING_TOKEN_EXCEPTION_NAME;
		recognizer->state->exception->type		= ANTLR3_MISSING_TOKEN_EXCEPTION;

		return;
	}

	// Just a mismatched token is all we can dtermine
	//
	antlr3MTExceptionNew(recognizer);

	return;
}
/// Report a recognition problem.
///
/// This method sets errorRecovery to indicate the parser is recovering
/// not parsing.  Once in recovery mode, no errors are generated.
/// To get out of recovery mode, the parser must successfully match
/// a token (after a resync).  So it will go:
///
///		1. error occurs
///		2. enter recovery mode, report error
///		3. consume until token found in resynch set
///		4. try to resume parsing
///		5. next match() will reset errorRecovery mode
///
/// If you override, make sure to update errorCount if you care about that.
///
static void			
reportError		    (pANTLR3_BASE_RECOGNIZER recognizer)
{
    	// Invoke the debugger event if there is a debugger listening to us
	//
	if	(recognizer->debugger != NULL)
	{
		recognizer->debugger->recognitionException(recognizer->debugger, recognizer->state->exception);
	}

    if	(recognizer->state->errorRecovery == ANTLR3_TRUE)
    {
		// Already in error recovery so don't display another error while doing so
		//
		return;
    }

    // Signal we are in error recovery now
    //
    recognizer->state->errorRecovery = ANTLR3_TRUE;
	
	// Indicate this recognizer had an error while processing.
	//
	recognizer->state->errorCount++;

	// Call the error display routine
	//
    recognizer->displayRecognitionError(recognizer, recognizer->state->tokenNames);
}

static void
beginBacktrack		(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level)
{
	if	(recognizer->debugger != NULL)
	{
		recognizer->debugger->beginBacktrack(recognizer->debugger, level);
	}
}

static void
endBacktrack		(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level, ANTLR3_BOOLEAN successful)
{
	if	(recognizer->debugger != NULL)
	{
		recognizer->debugger->endBacktrack(recognizer->debugger, level, successful);
	}
}
static void			
beginResync		    (pANTLR3_BASE_RECOGNIZER recognizer)
{
	if	(recognizer->debugger != NULL)
	{
		recognizer->debugger->beginResync(recognizer->debugger);
	}
}

static void			
endResync		    (pANTLR3_BASE_RECOGNIZER recognizer)
{
	if	(recognizer->debugger != NULL)
	{
		recognizer->debugger->endResync(recognizer->debugger);
	}
}

/// Compute the error recovery set for the current rule.
/// Documentation below is from the Java implementation.
///
/// During rule invocation, the parser pushes the set of tokens that can
/// follow that rule reference on the stack; this amounts to
/// computing FIRST of what follows the rule reference in the
/// enclosing rule. This local follow set only includes tokens
/// from within the rule; i.e., the FIRST computation done by
/// ANTLR stops at the end of a rule.
//
/// EXAMPLE
//
/// When you find a "no viable alt exception", the input is not
/// consistent with any of the alternatives for rule r.  The best
/// thing to do is to consume tokens until you see something that
/// can legally follow a call to r *or* any rule that called r.
/// You don't want the exact set of viable next tokens because the
/// input might just be missing a token--you might consume the
/// rest of the input looking for one of the missing tokens.
///
/// Consider grammar:
///
/// a : '[' b ']'
///   | '(' b ')'
///   ;
/// b : c '^' INT ;
/// c : ID
///   | INT
///   ;
///
/// At each rule invocation, the set of tokens that could follow
/// that rule is pushed on a stack.  Here are the various "local"
/// follow sets:
///
/// FOLLOW(b1_in_a) = FIRST(']') = ']'
/// FOLLOW(b2_in_a) = FIRST(')') = ')'
/// FOLLOW(c_in_b) = FIRST('^') = '^'
///
/// Upon erroneous input "[]", the call chain is
///
/// a -> b -> c
///
/// and, hence, the follow context stack is:
///
/// depth  local follow set     after call to rule
///   0         <EOF>                    a (from main())
///   1          ']'                     b
///   3          '^'                     c
///
/// Notice that ')' is not included, because b would have to have
/// been called from a different context in rule a for ')' to be
/// included.
///
/// For error recovery, we cannot consider FOLLOW(c)
/// (context-sensitive or otherwise).  We need the combined set of
/// all context-sensitive FOLLOW sets--the set of all tokens that
/// could follow any reference in the call chain.  We need to
/// resync to one of those tokens.  Note that FOLLOW(c)='^' and if
/// we resync'd to that token, we'd consume until EOF.  We need to
/// sync to context-sensitive FOLLOWs for a, b, and c: {']','^'}.
/// In this case, for input "[]", LA(1) is in this set so we would
/// not consume anything and after printing an error rule c would
/// return normally.  It would not find the required '^' though.
/// At this point, it gets a mismatched token error and throws an
/// exception (since LA(1) is not in the viable following token
/// set).  The rule exception handler tries to recover, but finds
/// the same recovery set and doesn't consume anything.  Rule b
/// exits normally returning to rule a.  Now it finds the ']' (and
/// with the successful match exits errorRecovery mode).
///
/// So, you can see that the parser walks up call chain looking
/// for the token that was a member of the recovery set.
///
/// Errors are not generated in errorRecovery mode.
///
/// ANTLR's error recovery mechanism is based upon original ideas:
///
/// "Algorithms + Data Structures = Programs" by Niklaus Wirth
///
/// and
///
/// "A note on error recovery in recursive descent parsers":
/// http://portal.acm.org/citation.cfm?id=947902.947905
///
/// Later, Josef Grosch had some good ideas:
///
/// "Efficient and Comfortable Error Recovery in Recursive Descent
/// Parsers":
/// ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip
///
/// Like Grosch I implemented local FOLLOW sets that are combined
/// at run-time upon error to avoid overhead during parsing.
///
static pANTLR3_BITSET		
computeErrorRecoverySet	    (pANTLR3_BASE_RECOGNIZER recognizer)
{
    return   recognizer->combineFollows(recognizer, ANTLR3_FALSE);
}

/// Compute the context-sensitive FOLLOW set for current rule.
/// Documentation below is from the Java runtime.
///
/// This is the set of token types that can follow a specific rule
/// reference given a specific call chain.  You get the set of
/// viable tokens that can possibly come next (look ahead depth 1)
/// given the current call chain.  Contrast this with the
/// definition of plain FOLLOW for rule r:
///
///  FOLLOW(r)={x | S=>*alpha r beta in G and x in FIRST(beta)}
///
/// where x in T* and alpha, beta in V*; T is set of terminals and
/// V is the set of terminals and non terminals.  In other words,
/// FOLLOW(r) is the set of all tokens that can possibly follow
/// references to r in///any* sentential form (context).  At
/// runtime, however, we know precisely which context applies as
/// we have the call chain.  We may compute the exact (rather
/// than covering superset) set of following tokens.
///
/// For example, consider grammar:
///
/// stat : ID '=' expr ';'      // FOLLOW(stat)=={EOF}
///      | "return" expr '.'
///      ;
/// expr : atom ('+' atom)* ;   // FOLLOW(expr)=={';','.',')'}
/// atom : INT                  // FOLLOW(atom)=={'+',')',';','.'}
///      | '(' expr ')'
///      ;
///
/// The FOLLOW sets are all inclusive whereas context-sensitive
/// FOLLOW sets are precisely what could follow a rule reference.
/// For input input "i=(3);", here is the derivation:
///
/// stat => ID '=' expr ';'
///      => ID '=' atom ('+' atom)* ';'
///      => ID '=' '(' expr ')' ('+' atom)* ';'
///      => ID '=' '(' atom ')' ('+' atom)* ';'
///      => ID '=' '(' INT ')' ('+' atom)* ';'
///      => ID '=' '(' INT ')' ';'
///
/// At the "3" token, you'd have a call chain of
///
///   stat -> expr -> atom -> expr -> atom
///
/// What can follow that specific nested ref to atom?  Exactly ')'
/// as you can see by looking at the derivation of this specific
/// input.  Contrast this with the FOLLOW(atom)={'+',')',';','.'}.
///
/// You want the exact viable token set when recovering from a
/// token mismatch.  Upon token mismatch, if LA(1) is member of
/// the viable next token set, then you know there is most likely
/// a missing token in the input stream.  "Insert" one by just not
/// throwing an exception.
///
static pANTLR3_BITSET		
computeCSRuleFollow	    (pANTLR3_BASE_RECOGNIZER recognizer)
{
    return   recognizer->combineFollows(recognizer, ANTLR3_FALSE);
}

/// Compute the current followset for the input stream.
///
static pANTLR3_BITSET		
combineFollows		    (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_BOOLEAN exact)
{
    pANTLR3_BITSET	followSet;
    pANTLR3_BITSET	localFollowSet;
    ANTLR3_UINT32	top;
    ANTLR3_UINT32	i;

    top	= recognizer->state->following->size(recognizer->state->following);

    followSet	    = antlr3BitsetNew(0);
	localFollowSet	= NULL;

    for (i = top; i>0; i--)
    {
		localFollowSet = antlr3BitsetLoad((pANTLR3_BITSET_LIST) recognizer->state->following->get(recognizer->state->following, i-1));

		if  (localFollowSet != NULL)
		{
			followSet->borInPlace(followSet, localFollowSet);

			if	(exact == ANTLR3_TRUE)
			{
				if	(localFollowSet->isMember(localFollowSet, ANTLR3_EOR_TOKEN_TYPE) == ANTLR3_FALSE)
				{
					// Only leave EOR in the set if at top (start rule); this lets us know
					// if we have to include the follow(start rule); I.E., EOF
					//
					if	(i>1)
					{
						followSet->remove(followSet, ANTLR3_EOR_TOKEN_TYPE);
					}
				}
				else
				{
					break;	// Cannot see End Of Rule from here, just drop out
				}
			}
			localFollowSet->free(localFollowSet);
			localFollowSet = NULL;
		}
    }

	if	(localFollowSet != NULL)
	{
		localFollowSet->free(localFollowSet);
	}
    return  followSet;
}

/// Standard/Example error display method.
/// No generic error message display funciton coudl possibly do everything correctly
/// for all possible parsers. Hence you are provided with this example routine, which
/// you should override in your parser/tree parser to do as you will.
///
/// Here we depart somewhat from the Java runtime as that has now split up a lot
/// of the error display routines into spearate units. However, ther is little advantage
/// to this in the C version as you will probably implement all such routines as a 
/// separate translation unit, rather than install them all as pointers to functions
/// in the base recognizer.
///
static void			
displayRecognitionError	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames)
{
	pANTLR3_PARSER			parser;
	pANTLR3_TREE_PARSER	    tparser;
	pANTLR3_INT_STREAM	    is;
	pANTLR3_STRING			ttext;
	pANTLR3_STRING			ftext;
	pANTLR3_EXCEPTION	    ex;
	pANTLR3_COMMON_TOKEN    theToken;
	pANTLR3_BASE_TREE	    theBaseTree;
	pANTLR3_COMMON_TREE	    theCommonTree;

	// Retrieve some info for easy reading.
	//
	ex	    =		recognizer->state->exception;
	ttext   =		NULL;

	// See if there is a 'filename' we can use
	//
	if	(ex->streamName == NULL)
	{
		if	(((pANTLR3_COMMON_TOKEN)(ex->token))->type == ANTLR3_TOKEN_EOF)
		{
			ANTLR3_FPRINTF(stderr, "-end of input-(");
		}
		else
		{
			ANTLR3_FPRINTF(stderr, "-unknown source-(");
		}
	}
	else
	{
		ftext = ex->streamName->to8(ex->streamName);
		ANTLR3_FPRINTF(stderr, "%s(", ftext->chars);
	}

	// Next comes the line number
	//

	ANTLR3_FPRINTF(stderr, "%d) ", recognizer->state->exception->line);
	ANTLR3_FPRINTF(stderr, " : error %d : %s", 
										recognizer->state->exception->type,
					(pANTLR3_UINT8)	   (recognizer->state->exception->message));


	// How we determine the next piece is dependent on which thing raised the
	// error.
	//
	switch	(recognizer->type)
	{
	case	ANTLR3_TYPE_PARSER:

		// Prepare the knowledge we know we have
		//
		parser	    = (pANTLR3_PARSER) (recognizer->super);
		tparser	    = NULL;
		is			= parser->tstream->istream;
		theToken    = (pANTLR3_COMMON_TOKEN)(recognizer->state->exception->token);
		ttext	    = theToken->toString(theToken);

		ANTLR3_FPRINTF(stderr, ", at offset %d", recognizer->state->exception->charPositionInLine);
		if  (theToken != NULL)
		{
			if (theToken->type == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(stderr, ", at <EOF>");
			}
			else
			{
				// Guard against null text in a token
				//
				ANTLR3_FPRINTF(stderr, "\n    near %s\n    ", ttext == NULL ? (pANTLR3_UINT8)"<no text for the token>" : ttext->chars);
			}
		}
		break;

	case	ANTLR3_TYPE_TREE_PARSER:

		tparser		= (pANTLR3_TREE_PARSER) (recognizer->super);
		parser		= NULL;
		is			= tparser->ctnstream->tnstream->istream;
		theBaseTree	= (pANTLR3_BASE_TREE)(recognizer->state->exception->token);
		ttext		= theBaseTree->toStringTree(theBaseTree);

		if  (theBaseTree != NULL)
		{
			theCommonTree	= (pANTLR3_COMMON_TREE)	    theBaseTree->super;

			if	(theCommonTree != NULL)
			{
				theToken	= (pANTLR3_COMMON_TOKEN)    theBaseTree->getToken(theBaseTree);
			}
			ANTLR3_FPRINTF(stderr, ", at offset %d", theBaseTree->getCharPositionInLine(theBaseTree));
			ANTLR3_FPRINTF(stderr, ", near %s", ttext->chars);
		}
		break;

	default:

		ANTLR3_FPRINTF(stderr, "Base recognizer function displayRecognitionError called by unknown parser type - provide override for this function\n");
		return;
		break;
	}

	// Although this function should generally be provided by the implementation, this one
	// should be as helpful as possible for grammar developers and serve as an example
	// of what you can do with each exception type. In general, when you make up your
	// 'real' handler, you should debug the routine with all possible errors you expect
	// which will then let you be as specific as possible about all circumstances.
	//
	// Note that in the general case, errors thrown by tree parsers indicate a problem
	// with the output of the parser or with the tree grammar itself. The job of the parser
	// is to produce a perfect (in traversal terms) syntactically correct tree, so errors
	// at that stage should really be semantic errors that your own code determines and handles
	// in whatever way is appropriate.
	//
	switch  (ex->type)
	{
	case	ANTLR3_UNWANTED_TOKEN_EXCEPTION:

		// Indicates that the recognizer was fed a token which seesm to be
		// spurious input. We can detect this when the token that follows
		// this unwanted token would normally be part of the syntactically
		// correct stream. Then we can see that the token we are looking at
		// is just something that should not be there and throw this exception.
		//
		if	(tokenNames == NULL)
		{
			ANTLR3_FPRINTF(stderr, " : Extraneous input...");
		}
		else
		{
			if	(ex->expecting == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(stderr, " : Extraneous input - expected <EOF>\n");
			}
			else
			{
				ANTLR3_FPRINTF(stderr, " : Extraneous input - expected %s ...\n", tokenNames[ex->expecting]);
			}
		}
		break;

	case	ANTLR3_MISSING_TOKEN_EXCEPTION:

		// Indicates that the recognizer detected that the token we just
		// hit would be valid syntactically if preceeded by a particular 
		// token. Perhaps a missing ';' at line end or a missing ',' in an
		// expression list, and such like.
		//
		if	(tokenNames == NULL)
		{
			ANTLR3_FPRINTF(stderr, " : Missing token (%d)...\n", ex->expecting);
		}
		else
		{
			if	(ex->expecting == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(stderr, " : Missing <EOF>\n");
			}
			else
			{
				ANTLR3_FPRINTF(stderr, " : Missing %s \n", tokenNames[ex->expecting]);
			}
		}
		break;

	case	ANTLR3_RECOGNITION_EXCEPTION:

		// Indicates that the recognizer received a token
		// in the input that was not predicted. This is the basic exception type 
		// from which all others are derived. So we assume it was a syntax error.
		// You may get this if there are not more tokens and more are needed
		// to complete a parse for instance.
		//
		ANTLR3_FPRINTF(stderr, " : syntax error...\n");    
		break;

	case    ANTLR3_MISMATCHED_TOKEN_EXCEPTION:

		// We were expecting to see one thing and got another. This is the
		// most common error if we coudl not detect a missing or unwanted token.
		// Here you can spend your efforts to
		// derive more useful error messages based on the expected
		// token set and the last token and so on. The error following
		// bitmaps do a good job of reducing the set that we were looking
		// for down to something small. Knowing what you are parsing may be
		// able to allow you to be even more specific about an error.
		//
		if	(tokenNames == NULL)
		{
			ANTLR3_FPRINTF(stderr, " : syntax error...\n");
		}
		else
		{
			if	(ex->expecting == ANTLR3_TOKEN_EOF)
			{
				ANTLR3_FPRINTF(stderr, " : expected <EOF>\n");
			}
			else
			{
				ANTLR3_FPRINTF(stderr, " : expected %s ...\n", tokenNames[ex->expecting]);
			}
		}
		break;

	case	ANTLR3_NO_VIABLE_ALT_EXCEPTION:

		// We could not pick any alt decision from the input given
		// so god knows what happened - however when you examine your grammar,
		// you should. It means that at the point where the current token occurred
		// that the DFA indicates nowhere to go from here.
		//
		ANTLR3_FPRINTF(stderr, " : cannot match to any predicted input...\n");

		break;

	case	ANTLR3_MISMATCHED_SET_EXCEPTION:

		{
			ANTLR3_UINT32	  count;
			ANTLR3_UINT32	  bit;
			ANTLR3_UINT32	  size;
			ANTLR3_UINT32	  numbits;
			pANTLR3_BITSET	  errBits;

			// This means we were able to deal with one of a set of
			// possible tokens at this point, but we did not see any
			// member of that set.
			//
			ANTLR3_FPRINTF(stderr, " : unexpected input...\n  expected one of : ");

			// What tokens could we have accepted at this point in the
			// parse?
			//
			count   = 0;
			errBits = antlr3BitsetLoad		(ex->expectingSet);
			numbits = errBits->numBits		(errBits);
			size    = errBits->size			(errBits);

			if  (size > 0)
			{
				// However many tokens we could have dealt with here, it is usually
				// not useful to print ALL of the set here. I arbitrarily chose 8
				// here, but you should do whatever makes sense for you of course.
				// No token number 0, so look for bit 1 and on.
				//
				for	(bit = 1; bit < numbits && count < 8 && count < size; bit++)
				{
					// TODO: This doesn;t look right - should be asking if the bit is set!!
					//
					if  (tokenNames[bit])
					{
						ANTLR3_FPRINTF(stderr, "%s%s", count > 0 ? ", " : "", tokenNames[bit]); 
						count++;
					}
				}
				ANTLR3_FPRINTF(stderr, "\n");
			}
			else
			{
				ANTLR3_FPRINTF(stderr, "Actually dude, we didn't seem to be expecting anything here, or at least\n");
				ANTLR3_FPRINTF(stderr, "I could not work out what I was expecting, like so many of us these days!\n");
			}
		}
		break;

	case	ANTLR3_EARLY_EXIT_EXCEPTION:

		// We entered a loop requiring a number of token sequences
		// but found a token that ended that sequence earlier than
		// we should have done.
		//
		ANTLR3_FPRINTF(stderr, " : missing elements...\n");
		break;

	default:

		// We don't handle any other exceptions here, but you can
		// if you wish. If we get an exception that hits this point
		// then we are just going to report what we know about the
		// token.
		//
		ANTLR3_FPRINTF(stderr, " : syntax not recognized...\n");
		break;
	}

	// Here you have the token that was in error which if this is
	// the standard implementation will tell you the line and offset
	// and also record the address of the start of the line in the
	// input stream. You could therefore print the source line and so on.
	// Generally though, I would expect that your lexer/parser will keep
	// its own map of lines and source pointers or whatever as there
	// are a lot of specific things you need to know about the input
	// to do something like that.
	// Here is where you do it though :-).
	//
}

/// Return how many syntax errors were detected by this recognizer
///
static ANTLR3_UINT32
getNumberOfSyntaxErrors(pANTLR3_BASE_RECOGNIZER recognizer)
{
	return	recognizer->state->errorCount;
}

/// Recover from an error found on the input stream.  Mostly this is
/// NoViableAlt exceptions, but could be a mismatched token that
/// the match() routine could not recover from.
///
static void			
recover			    (pANTLR3_BASE_RECOGNIZER recognizer)
{
    // Used to compute the follow set of tokens
    //
    pANTLR3_BITSET			followSet;
    pANTLR3_PARSER			parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

		parser  = (pANTLR3_PARSER) (recognizer->super);
		tparser	= NULL;
		is		= parser->tstream->istream;

	break;

    case	ANTLR3_TYPE_TREE_PARSER:

		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
		parser	= NULL;
		is		= tparser->ctnstream->tnstream->istream;

	break;

    default:
	    
		ANTLR3_FPRINTF(stderr, "Base recognizer function recover called by unknown parser type - provide override for this function\n");
		return;

	break;
    }

	// Are we about to repeat the same error?
	//
    if	(recognizer->state->lastErrorIndex == is->index(is))
    {
		// The last error was at the same token index point. This must be a case
		// where LT(1) is in the recovery token set so nothing is
		// consumed. Consume a single token so at least to prevent
		// an infinite loop; this is a failsafe.
		//
		is->consume(is);
    }

    // Record error index position
    //
    recognizer->state->lastErrorIndex	 = is->index(is);
    
    // Work out the follows set for error recovery
    //
    followSet	= recognizer->computeErrorRecoverySet(recognizer);

    // Call resync hook (for debuggers and so on)
    //
    recognizer->beginResync(recognizer);

    // Consume tokens until we have resynced to something in the follows set
    //
    recognizer->consumeUntilSet(recognizer, followSet);

    // End resync hook 
    //
    recognizer->endResync(recognizer);

    // Destroy the temporary bitset we produced.
    //
    followSet->free(followSet);

    // Reset the inError flag so we don't re-report the exception
    //
    recognizer->state->error	= ANTLR3_FALSE;
    recognizer->state->failed	= ANTLR3_FALSE;
}


/// Attempt to recover from a single missing or extra token.
///
/// EXTRA TOKEN
///
/// LA(1) is not what we are looking for.  If LA(2) has the right token,
/// however, then assume LA(1) is some extra spurious token.  Delete it
/// and LA(2) as if we were doing a normal match(), which advances the
/// input.
///
/// MISSING TOKEN
///
/// If current token is consistent with what could come after
/// ttype then it is ok to "insert" the missing token, else throw
/// exception For example, Input "i=(3;" is clearly missing the
/// ')'.  When the parser returns from the nested call to expr, it
/// will have call chain:
///
///    stat -> expr -> atom
///
/// and it will be trying to match the ')' at this point in the
/// derivation:
///
///       => ID '=' '(' INT ')' ('+' atom)* ';'
///                          ^
/// match() will see that ';' doesn't match ')' and report a
/// mismatched token error.  To recover, it sees that LA(1)==';'
/// is in the set of tokens that can follow the ')' token
/// reference in rule atom.  It can assume that you forgot the ')'.
///
/// The exception that was passed in, in the java implementation is
/// sorted in the recognizer exception stack in the C version. To 'throw' it we set the
/// error flag and rules cascade back when this is set.
///
static void *	
recoverFromMismatchedToken  (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow)
{
	pANTLR3_PARSER			  parser;
	pANTLR3_TREE_PARSER	      tparser;
	pANTLR3_INT_STREAM	      is;
	void					* matchedSymbol;



	switch	(recognizer->type)
	{
	case	ANTLR3_TYPE_PARSER:

		parser  = (pANTLR3_PARSER) (recognizer->super);
		tparser	= NULL;
		is	= parser->tstream->istream;

		break;

	case	ANTLR3_TYPE_TREE_PARSER:

		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
		parser	= NULL;
		is	= tparser->ctnstream->tnstream->istream;

		break;

	default:

		ANTLR3_FPRINTF(stderr, "Base recognizer function recoverFromMismatchedToken called by unknown parser type - provide override for this function\n");
		return NULL;

		break;
	}

	// Create an exception if we need one
	//
	if	(recognizer->state->exception == NULL)
	{
		antlr3RecognitionExceptionNew(recognizer);
	}

	// If the next token after the one we are looking at in the input stream
	// is what we are looking for then we remove the one we have discovered
	// from the stream by consuming it, then consume this next one along too as
	// if nothing had happened.
	//
	if	( recognizer->mismatchIsUnwantedToken(recognizer, is, ttype) == ANTLR3_TRUE)
	{
		recognizer->state->exception->type		= ANTLR3_UNWANTED_TOKEN_EXCEPTION;
		recognizer->state->exception->message	= ANTLR3_UNWANTED_TOKEN_EXCEPTION_NAME;

		// Call resync hook (for debuggers and so on)
		//
		if	(recognizer->debugger != NULL)
		{
			recognizer->debugger->beginResync(recognizer->debugger);
		}

		// "delete" the extra token
		//
		recognizer->beginResync(recognizer);
		is->consume(is);
		recognizer->endResync(recognizer);
		// End resync hook 
		//
		if	(recognizer->debugger != NULL)
		{
			recognizer->debugger->endResync(recognizer->debugger);
		}

		// Print out the error after we consume so that ANTLRWorks sees the
		// token in the exception.
		//
		recognizer->reportError(recognizer);

		// Return the token we are actually matching
		//
		matchedSymbol = recognizer->getCurrentInputSymbol(recognizer, is);

		// Consume the token that the rule actually expected to get as if everything
		// was hunky dory.
		//
		is->consume(is);

		recognizer->state->error  = ANTLR3_FALSE;	// Exception is not outstanding any more

		return	matchedSymbol;
	}

	// Single token deletion (Unwanted above) did not work
	// so we see if we can insert a token instead by calculating which
	// token would be missing
	//
	if	(mismatchIsMissingToken(recognizer, is, follow))
	{
		// We can fake the missing token and proceed
		//
		matchedSymbol = recognizer->getMissingSymbol(recognizer, is, recognizer->state->exception, ttype, follow);
		recognizer->state->exception->type		= ANTLR3_MISSING_TOKEN_EXCEPTION;
		recognizer->state->exception->message	= ANTLR3_MISSING_TOKEN_EXCEPTION_NAME;
		recognizer->state->exception->token		= matchedSymbol;
		recognizer->state->exception->expecting	= ttype;

		// Print out the error after we insert so that ANTLRWorks sees the
		// token in the exception.
		//
		recognizer->reportError(recognizer);

		recognizer->state->error  = ANTLR3_FALSE;	// Exception is not outstanding any more

		return	matchedSymbol;
	}


	// Neither deleting nor inserting tokens allows recovery
	// must just report the exception.
	//
	recognizer->state->error	    = ANTLR3_TRUE;
	return NULL;
}

static void *
recoverFromMismatchedSet	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST follow)
{
    pANTLR3_PARSER			parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;
	pANTLR3_COMMON_TOKEN	matchedSymbol;

    switch	(recognizer->type)
    {
    case	ANTLR3_TYPE_PARSER:

		parser  = (pANTLR3_PARSER) (recognizer->super);
		tparser	= NULL;
		is	= parser->tstream->istream;

	break;

    case	ANTLR3_TYPE_TREE_PARSER:

		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
		parser	= NULL;
		is	= tparser->ctnstream->tnstream->istream;

	break;

    default:
	    
		ANTLR3_FPRINTF(stderr, "Base recognizer function recoverFromMismatchedSet called by unknown parser type - provide override for this function\n");
		return NULL;

	break;
    }

	if	(recognizer->mismatchIsMissingToken(recognizer, is, follow) == ANTLR3_TRUE)
	{
		// We can fake the missing token and proceed
		//
		matchedSymbol = recognizer->getMissingSymbol(recognizer, is, recognizer->state->exception, ANTLR3_TOKEN_INVALID, follow);
		recognizer->state->exception->type	= ANTLR3_MISSING_TOKEN_EXCEPTION;
		recognizer->state->exception->token	= matchedSymbol;

		// Print out the error after we insert so that ANTLRWorks sees the
		// token in the exception.
		//
		recognizer->reportError(recognizer);

		recognizer->state->error  = ANTLR3_FALSE;	// Exception is not outstanding any more

		return	matchedSymbol;
	}

    // TODO - Single token deletion like in recoverFromMismatchedToken()
    //
    recognizer->state->error	= ANTLR3_TRUE;
	recognizer->state->failed	= ANTLR3_TRUE;
	return NULL;
}

/// This code is factored out from mismatched token and mismatched set
///  recovery.  It handles "single token insertion" error recovery for
/// both.  No tokens are consumed to recover from insertions.  Return
/// true if recovery was possible else return false.
///
static ANTLR3_BOOLEAN	
recoverFromMismatchedElement	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST followBits)
{
    pANTLR3_BITSET	    viableToksFollowingRule;
    pANTLR3_BITSET	    follow;
    pANTLR3_PARSER	    parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
    case	ANTLR3_TYPE_PARSER:

		parser  = (pANTLR3_PARSER) (recognizer->super);
		tparser	= NULL;
		is	= parser->tstream->istream;

	break;

    case	ANTLR3_TYPE_TREE_PARSER:

		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
		parser	= NULL;
		is	= tparser->ctnstream->tnstream->istream;

	break;

    default:
	    
		ANTLR3_FPRINTF(stderr, "Base recognizer function recover called by unknown parser type - provide override for this function\n");
		return ANTLR3_FALSE;

	break;
    }

    follow	= antlr3BitsetLoad(followBits);

    if	(follow == NULL)
    {
		/* The follow set is NULL, which means we don't know what can come 
		 * next, so we "hit and hope" by just signifying that we cannot
		 * recover, which will just cause the next token to be consumed,
		 * which might dig us out.
		 */
		return	ANTLR3_FALSE;
    }

    /* We have a bitmap for the follow set, hence we can compute 
     * what can follow this grammar element reference.
     */
    if	(follow->isMember(follow, ANTLR3_EOR_TOKEN_TYPE) == ANTLR3_TRUE)
    {
		/* First we need to know which of the available tokens are viable
		 * to follow this reference.
		 */
		viableToksFollowingRule	= recognizer->computeCSRuleFollow(recognizer);

		/* Remove the EOR token, which we do not wish to compute with
		 */
		follow->remove(follow, ANTLR3_EOR_TOKEN_TYPE);
		viableToksFollowingRule->free(viableToksFollowingRule);
		/* We now have the computed set of what can follow the current token
		 */
    }

    /* We can now see if the current token works with the set of tokens
     * that could follow the current grammar reference. If it looks like it
     * is consistent, then we can "insert" that token by not throwing
     * an exception and assuming that we saw it. 
     */
    if	( follow->isMember(follow, is->_LA(is, 1)) == ANTLR3_TRUE)
    {
		/* report the error, but don't cause any rules to abort and stuff
		 */
		recognizer->reportError(recognizer);
		if	(follow != NULL)
		{
			follow->free(follow);
		}
		recognizer->state->error			= ANTLR3_FALSE;
		recognizer->state->failed			= ANTLR3_FALSE;
		return ANTLR3_TRUE;	/* Success in recovery	*/
    }

    if	(follow != NULL)
    {
		follow->free(follow);
    }

    /* We could not find anything viable to do, so this is going to 
     * cause an exception.
     */
    return  ANTLR3_FALSE;
}

/// Eat tokens from the input stream until we get one of JUST the right type
///
static void		
consumeUntil	(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 tokenType)
{
    ANTLR3_UINT32			ttype;
    pANTLR3_PARSER			parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			is	= parser->tstream->istream;

			break;

		case	ANTLR3_TYPE_TREE_PARSER:

			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
			parser	= NULL;
			is	= tparser->ctnstream->tnstream->istream;

			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function 'consumeUntil' called by unknown parser type - provide override for this function\n");
			return;

			break;
    }

    // What do have at the moment?
    //
    ttype	= is->_LA(is, 1);

    // Start eating tokens until we get to the one we want.
    //
    while   (ttype != ANTLR3_TOKEN_EOF && ttype != tokenType)
    {
		is->consume(is);
		ttype	= is->_LA(is, 1);
    }
}

/// Eat tokens from the input stream until we find one that
/// belongs to the supplied set.
///
static void		
consumeUntilSet			    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET set)
{
    ANTLR3_UINT32	    ttype;
    pANTLR3_PARSER	    parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			is	= parser->tstream->istream;

			break;

		case	ANTLR3_TYPE_TREE_PARSER:

			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
			parser	= NULL;
			is	= tparser->ctnstream->tnstream->istream;

			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function 'consumeUntilSet' called by unknown parser type - provide override for this function\n");
			return;

			break;
    }

    // What do have at the moment?
    //
    ttype	= is->_LA(is, 1);

    // Start eating tokens until we get to one we want.
    //
    while   (ttype != ANTLR3_TOKEN_EOF && set->isMember(set, ttype) == ANTLR3_FALSE)
    {
		is->consume(is);
		ttype	= is->_LA(is, 1);
    }
}

/** Return the rule invocation stack (how we got here in the parse.
 *  In the java version Ter just asks the JVM for all the information
 *  but in C we don't get this information, so I am going to do nothing 
 *  right now.
 */
static pANTLR3_STACK	
getRuleInvocationStack		    (pANTLR3_BASE_RECOGNIZER recognizer)
{
    return NULL;
}

static pANTLR3_STACK	
getRuleInvocationStackNamed	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 name)
{
    return NULL;
}

/** Convenience method for template rewrites - NYI.
 */
static pANTLR3_HASH_TABLE	
toStrings			    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_HASH_TABLE tokens)
{
    return NULL;
}

static	void ANTLR3_CDECL
freeIntTrie    (void * trie)
{
    ((pANTLR3_INT_TRIE)trie)->free((pANTLR3_INT_TRIE)trie);
}


/** Pointer to a function to return whether the rule has parsed input starting at the supplied 
 *  start index before. If the rule has not parsed input starting from the supplied start index,
 *  then it will return ANTLR3_MEMO_RULE_UNKNOWN. If it has parsed from the suppled start point
 *  then it will return the point where it last stopped parsing after that start point.
 *
 * \remark
 * The rule memos are an ANTLR3_LIST of ANTLR3_LISTS, however if this becomes any kind of performance
 * issue (it probably won't, the hash tables are pretty quick) then we could make a special int only
 * version of the table.
 */
static ANTLR3_MARKER	
getRuleMemoization		    (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_INTKEY ruleIndex, ANTLR3_MARKER ruleParseStart)
{
    /* The rule memos are an ANTLR3_LIST of ANTLR3_LIST.
     */
    pANTLR3_INT_TRIE	ruleList;
    ANTLR3_MARKER	stopIndex;
    pANTLR3_TRIE_ENTRY	entry;

    /* See if we have a list in the ruleMemos for this rule, and if not, then create one
     * as we will need it eventually if we are being asked for the memo here.
     */
    entry	= recognizer->state->ruleMemo->get(recognizer->state->ruleMemo, (ANTLR3_INTKEY)ruleIndex);

    if	(entry == NULL)
    {
		/* Did not find it, so create a new one for it, with a bit depth based on the 
		 * size of the input stream. We need the bit depth to incorporate the number if
		 * bits required to represent the largest possible stop index in the input, which is the
		 * last character. An int stream is free to return the largest 64 bit offset if it has
		 * no idea of the size, but you should remember that this will cause the leftmost
		 * bit match algorithm to run to 63 bits, which will be the whole time spent in the trie ;-)
		 */
		ruleList    = antlr3IntTrieNew(63);	/* Depth is theoretically 64 bits, but probably not ;-)	*/

		if (ruleList != NULL)
		{
			recognizer->state->ruleMemo->add(recognizer->state->ruleMemo, (ANTLR3_INTKEY)ruleIndex, ANTLR3_HASH_TYPE_STR, 0, ANTLR3_FUNC_PTR(ruleList), freeIntTrie);
		}

		/* We cannot have a stopIndex in a trie we have just created of course
		 */
		return	MEMO_RULE_UNKNOWN;
    }

    ruleList	= (pANTLR3_INT_TRIE) (entry->data.ptr);

    /* See if there is a stop index associated with the supplied start index.
     */
    stopIndex	= 0;

    entry = ruleList->get(ruleList, ruleParseStart);
    if (entry != NULL)
    {
		stopIndex = (ANTLR3_MARKER)(entry->data.intVal);
    }

    if	(stopIndex == 0)
    {
		return MEMO_RULE_UNKNOWN;
    }

    return  stopIndex;
}

/** Has this rule already parsed input at the current index in the
 *  input stream?  Return ANTLR3_TRUE if we have and ANTLR3_FALSE
 *  if we have not.
 *
 *  This method has a side-effect: if we have seen this input for
 *  this rule and successfully parsed before, then seek ahead to
 *  1 past the stop token matched for this rule last time.
 */
static ANTLR3_BOOLEAN	
alreadyParsedRule		    (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex)
{
    ANTLR3_MARKER			stopIndex;
    pANTLR3_LEXER			lexer;
    pANTLR3_PARSER			parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			lexer	= NULL;
			is	= parser->tstream->istream;

			break;

		case	ANTLR3_TYPE_TREE_PARSER:

			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
			parser	= NULL;
			lexer	= NULL;
			is	= tparser->ctnstream->tnstream->istream;

			break;

		case	ANTLR3_TYPE_LEXER:

			lexer	= (pANTLR3_LEXER)   (recognizer->super);
			parser	= NULL;
			tparser	= NULL;
			is	= lexer->input->istream;
			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function 'alreadyParsedRule' called by unknown parser type - provide override for this function\n");
			return ANTLR3_FALSE;

			break;
    }

    /* See if we have a memo marker for this.
     */
    stopIndex	    = recognizer->getRuleMemoization(recognizer, ruleIndex, is->index(is));

    if	(stopIndex  == MEMO_RULE_UNKNOWN)
    {
		return ANTLR3_FALSE;
    }

    if	(stopIndex == MEMO_RULE_FAILED)
    {
		recognizer->state->failed = ANTLR3_TRUE;
    }
    else
    {
		is->seek(is, stopIndex+1);
    }

    /* If here then the rule was executed for this input already
     */
    return  ANTLR3_TRUE;
}

/** Record whether or not this rule parsed the input at this position
 *  successfully.
 */
static void		
memoize	(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex, ANTLR3_MARKER ruleParseStart)
{
    /* The rule memos are an ANTLR3_LIST of ANTLR3_LIST.
     */
    pANTLR3_INT_TRIE	    ruleList;
    pANTLR3_TRIE_ENTRY	    entry;
    ANTLR3_MARKER	    stopIndex;
    pANTLR3_LEXER	    lexer;
    pANTLR3_PARSER	    parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			is	= parser->tstream->istream;

			break;

		case	ANTLR3_TYPE_TREE_PARSER:

			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
			parser	= NULL;
			is	= tparser->ctnstream->tnstream->istream;

			break;

		case	ANTLR3_TYPE_LEXER:

			lexer	= (pANTLR3_LEXER)   (recognizer->super);
			parser	= NULL;
			tparser	= NULL;
			is		= lexer->input->istream;
			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function consumeUntilSet called by unknown parser type - provide override for this function\n");
			return;

			break;
    }
    
    stopIndex	= recognizer->state->failed == ANTLR3_TRUE ? MEMO_RULE_FAILED : is->index(is) - 1;

    entry	= recognizer->state->ruleMemo->get(recognizer->state->ruleMemo, (ANTLR3_INTKEY)ruleIndex);

    if	(entry != NULL)
    {
		ruleList = (pANTLR3_INT_TRIE)(entry->data.ptr);

		/* If we don't already have this entry, append it. The memoize trie does not
		 * accept duplicates so it won't add it if already there and we just ignore the
		 * return code as we don't care if it is there already.
		 */
		ruleList->add(ruleList, ruleParseStart, ANTLR3_HASH_TYPE_INT, stopIndex, NULL, NULL);
    }
}
/** A syntactic predicate.  Returns true/false depending on whether
 *  the specified grammar fragment matches the current input stream.
 *  This resets the failed instance var afterwards.
 */
static ANTLR3_BOOLEAN	
synpred	(pANTLR3_BASE_RECOGNIZER recognizer, void * ctx, void (*predicate)(void * ctx))
{
    ANTLR3_MARKER   start;
    pANTLR3_PARSER	    parser;
    pANTLR3_TREE_PARSER	    tparser;
    pANTLR3_INT_STREAM	    is;

    switch	(recognizer->type)
    {
		case	ANTLR3_TYPE_PARSER:

			parser  = (pANTLR3_PARSER) (recognizer->super);
			tparser	= NULL;
			is	= parser->tstream->istream;

			break;

		case	ANTLR3_TYPE_TREE_PARSER:

			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
			parser	= NULL;
			is	= tparser->ctnstream->tnstream->istream;

			break;

		default:
		    
			ANTLR3_FPRINTF(stderr, "Base recognizer function 'synPred' called by unknown parser type - provide override for this function\n");
			return ANTLR3_FALSE;

			break;
    }

    /* Begin backtracking so we can get back to where we started after trying out
     * the syntactic predicate.
     */
    start   = is->mark(is);
    recognizer->state->backtracking++;

    /* Try the syntactical predicate
     */
    predicate(ctx);

    /* Reset
     */
    is->rewind(is, start);
    recognizer->state->backtracking--;

    if	(recognizer->state->failed == ANTLR3_TRUE)
    {
		/* Predicate failed
		 */
		recognizer->state->failed = ANTLR3_FALSE;
		return	ANTLR3_FALSE;
    }
    else
    {
		/* Predicate was successful
		 */
		recognizer->state->failed	= ANTLR3_FALSE;
		return	ANTLR3_TRUE;
    }
}

static void
reset(pANTLR3_BASE_RECOGNIZER recognizer)
{
    if	(recognizer->state->following != NULL)
    {
		recognizer->state->following->free(recognizer->state->following);
    }

	// Reset the state flags
	//
	recognizer->state->errorRecovery	= ANTLR3_FALSE;
	recognizer->state->lastErrorIndex	= -1;
	recognizer->state->failed			= ANTLR3_FALSE;
	recognizer->state->errorCount		= 0;
	recognizer->state->backtracking		= 0;
	recognizer->state->following		= NULL;

	if	(recognizer->state != NULL)
	{
		if	(recognizer->state->ruleMemo != NULL)
		{
			recognizer->state->ruleMemo->free(recognizer->state->ruleMemo);
			recognizer->state->ruleMemo = antlr3IntTrieNew(15);	/* 16 bit depth is enough for 32768 rules! */
		}
	}
	

    // Install a new following set
    //
    recognizer->state->following   = antlr3StackNew(8);

}

// Default implementation is for parser and assumes a token stream as supplied by the runtime.
// You MAY need override this function if the standard TOKEN_STREAM is not what you are using.
//
static void *				
getCurrentInputSymbol		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream)
{
	return ((pANTLR3_TOKEN_STREAM)istream->super)->_LT((pANTLR3_TOKEN_STREAM)istream->super, 1);
}

// Default implementation is for parser and assumes a token stream as supplied by the runtime.
// You MAY need override this function if the standard COMMON_TOKEN_STREAM is not what you are using.
//
static void *				
getMissingSymbol			(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM	istream, pANTLR3_EXCEPTION	e,
									ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow)
{
	pANTLR3_TOKEN_STREAM			ts;
	pANTLR3_COMMON_TOKEN_STREAM		cts;
	pANTLR3_COMMON_TOKEN			token;
	pANTLR3_COMMON_TOKEN			current;
	pANTLR3_STRING					text;

	// Dereference the standard pointers
	//
	ts		= (pANTLR3_TOKEN_STREAM)istream->super;
	cts		= (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
	
	// Work out what to use as the current symbol to make a line and offset etc
	// If we are at EOF, we use the token before EOF
	//
	current	= ts->_LT(ts, 1);
	if	(current->getType(current) == ANTLR3_TOKEN_EOF)
	{
		current = ts->_LT(ts, -1);
	}

	// Create a new empty token
	//
	if	(recognizer->state->tokFactory == NULL)
	{
		// We don't yet have a token factory for making tokens
		// we just need a fake one using the input stream of the current
		// token.
		//
		recognizer->state->tokFactory = antlr3TokenFactoryNew(current->input);
	}
	token	= recognizer->state->tokFactory->newToken(recognizer->state->tokFactory);

	// Set some of the token properties based on the current token
	//
	token->setLine					(token, current->getLine(current));
	token->setCharPositionInLine	(token, current->getCharPositionInLine(current));
	token->setChannel				(token, ANTLR3_TOKEN_DEFAULT_CHANNEL);
	token->setType					(token, expectedTokenType);
    token->user1                    = current->user1;
    token->user2                    = current->user2;
    token->user3                    = current->user3;
    token->custom                   = current->custom;
    token->lineStart                = current->lineStart;
    
	// Create the token text that shows it has been inserted
	//
	token->setText8(token, (pANTLR3_UINT8)"<missing ");
	text = token->getText(token);

	if	(text != NULL)
	{
		text->append8(text, (const char *)recognizer->state->tokenNames[expectedTokenType]);
		text->append8(text, (const char *)">");
	}
	
	// Finally return the pointer to our new token
	//
	return	token;
}


#ifdef	ANTLR3_WINDOWS
#pragma warning( default : 4100 )
#endif

/// @}
///