File: TokenReader.cs

package info (click to toggle)
monodevelop 4.0.12%2Bdfsg-6
  • links: PTS, VCS
  • area: main
  • in suites: jessie, jessie-kfreebsd
  • size: 219,596 kB
  • ctags: 253,200
  • sloc: cs: 1,486,058; xml: 952,347; java: 60,981; makefile: 4,213; sh: 1,727; ansic: 867; objc: 302; sql: 111
file content (350 lines) | stat: -rw-r--r-- 9,823 bytes parent folder | download | duplicates (5)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
// Copyright (c) 2009-2013 AlphaSierraPapa for the SharpDevelop Team
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy of this
// software and associated documentation files (the "Software"), to deal in the Software
// without restriction, including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
// to whom the Software is furnished to do so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in all copies or
// substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
// PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
// FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using ICSharpCode.NRefactory.Editor;

namespace ICSharpCode.NRefactory.Xml
{
	class TokenReader
	{
		protected readonly ITextSource input;
		protected readonly int inputLength;
		int currentLocation;
		
		// CurrentLocation is assumed to be touched and the fact does not
		//   have to be recorded in this variable.
		// This stores any value bigger than that if applicable.
		// Actual value is max(currentLocation, maxTouchedLocation).
		int maxTouchedLocation;
		
		public int InputLength {
			get { return inputLength; }
		}
		
		public int CurrentLocation {
			get { return currentLocation; }
		}
		
		public int MaxTouchedLocation {
			// add 1 to currentLocation because single-char-peek does not increment maxTouchedLocation
			get { return Math.Max(currentLocation + 1, maxTouchedLocation); }
		}
		
		public TokenReader(ITextSource input)
		{
			this.input = input;
			this.inputLength = input.TextLength;
		}
		
		protected bool IsEndOfFile()
		{
			return currentLocation == inputLength;
		}
		
		protected bool HasMoreData()
		{
			return currentLocation < inputLength;
		}
		
		protected void AssertHasMoreData()
		{
			Log.Assert(HasMoreData(), "Unexpected end of file");
		}
		
		protected bool TryMoveNext()
		{
			if (currentLocation == inputLength) return false;
			
			currentLocation++;
			return true;
		}
		
		protected void Skip(int count)
		{
			Log.Assert(currentLocation + count <= inputLength, "Skipping after the end of file");
			currentLocation += count;
		}
		
		protected void GoBack(int oldLocation)
		{
			Log.Assert(oldLocation <= currentLocation, "Trying to move forward");
			// add 1 because single-char-peek does not increment maxTouchedLocation
			maxTouchedLocation = Math.Max(maxTouchedLocation, currentLocation + 1);
			currentLocation = oldLocation;
		}
		
		protected bool TryRead(char c)
		{
			if (currentLocation == inputLength) return false;
			
			if (input.GetCharAt(currentLocation) == c) {
				currentLocation++;
				return true;
			} else {
				return false;
			}
		}
		
		protected bool TryReadAnyOf(params char[] c)
		{
			if (currentLocation == inputLength) return false;
			
			if (c.Contains(input.GetCharAt(currentLocation))) {
				currentLocation++;
				return true;
			} else {
				return false;
			}
		}
		
		protected bool TryRead(string text)
		{
			if (TryPeek(text)) {
				currentLocation += text.Length;
				return true;
			} else {
				return false;
			}
		}
		
		protected bool TryPeekPrevious(char c, int back)
		{
			if (currentLocation - back == inputLength) return false;
			if (currentLocation - back < 0 ) return false;
			
			return input.GetCharAt(currentLocation - back) == c;
		}
		
		protected bool TryPeek(char c)
		{
			if (currentLocation == inputLength) return false;
			
			return input.GetCharAt(currentLocation) == c;
		}
		
		protected bool TryPeekAnyOf(params char[] chars)
		{
			if (currentLocation == inputLength) return false;
			
			return chars.Contains(input.GetCharAt(currentLocation));
		}
		
		protected bool TryPeek(string text)
		{
			if (!TryPeek(text[0])) return false; // Early exit
			
			maxTouchedLocation = Math.Max(maxTouchedLocation, currentLocation + text.Length);
			// The following comparison 'touches' the end of file - it does depend on the end being there
			if (currentLocation + text.Length > inputLength) return false;
			
			return input.GetText(currentLocation, text.Length) == text;
		}
		
		protected bool TryPeekWhiteSpace()
		{
			if (currentLocation == inputLength) return false;
			
			char c = input.GetCharAt(currentLocation);
			return ((int)c <= 0x20) && (c == ' ' || c == '\t' || c == '\n' || c == '\r');
		}
		
		// The move functions do not have to move if already at target
		// The move functions allow 'overriding' of the document length
		
		protected bool TryMoveTo(char c)
		{
			return TryMoveTo(c, inputLength);
		}
		
		protected bool TryMoveTo(char c, int inputLength)
		{
			if (currentLocation == inputLength) return false;
			int index = input.IndexOf(c, currentLocation, inputLength - currentLocation);
			if (index != -1) {
				currentLocation = index;
				return true;
			} else {
				currentLocation = inputLength;
				return false;
			}
		}
		
		protected bool TryMoveToAnyOf(params char[] c)
		{
			return TryMoveToAnyOf(c, inputLength);
		}
		
		protected bool TryMoveToAnyOf(char[] c, int inputLength)
		{
			if (currentLocation == inputLength) return false;
			int index = input.IndexOfAny(c, currentLocation, inputLength - currentLocation);
			if (index != -1) {
				currentLocation = index;
				return true;
			} else {
				currentLocation = inputLength;
				return false;
			}
		}
		
		protected bool TryMoveTo(string text)
		{
			return TryMoveTo(text, inputLength);
		}
		
		protected bool TryMoveTo(string text, int inputLength)
		{
			if (currentLocation == inputLength) return false;
			int index = input.IndexOf(text, currentLocation, inputLength - currentLocation, StringComparison.Ordinal);
			if (index != -1) {
				maxTouchedLocation = index + text.Length;
				currentLocation = index;
				return true;
			} else {
				currentLocation = inputLength;
				return false;
			}
		}
		
		protected bool TryMoveToNonWhiteSpace()
		{
			return TryMoveToNonWhiteSpace(inputLength);
		}
		
		protected bool TryMoveToNonWhiteSpace(int inputLength)
		{
			while(true) {
				if (currentLocation == inputLength) return false; // Reject end of file
				char c = input.GetCharAt(currentLocation);
				if (((int)c <= 0x20) && (c == ' ' || c == '\t' || c == '\n' || c == '\r')) {
					currentLocation++;                            // Accept white-space
					continue;
				} else {
					return true;  // Found non-white-space
				}
			}
		}
		
		/// <summary>
		/// Read a name token.
		/// The following characters are not allowed:
		///   ""         End of file
		///   " \n\r\t"  Whitesapce
		///   "=\'\""    Attribute value
		///   "&lt;>/?"  Tags
		/// </summary>
		/// <returns>Returns the length of the name</returns>
		protected bool TryReadName()
		{
			int start = currentLocation;
			// Keep reading up to invalid character
			while (HasMoreData()) {
				char c = input.GetCharAt(currentLocation);
				if (0x41 <= (int)c) {                                   // Accpet from 'A' onwards
					currentLocation++;
					continue;
				}
				if (c == ' ' || c == '\n' || c == '\r' || c == '\t' ||  // Reject whitesapce
				    c == '=' || c == '\'' || c == '"'  ||               // Reject attributes
				    c == '<' || c == '>'  || c == '/'  || c == '?') {   // Reject tags
					break;
				} else {
					currentLocation++;
					continue;                                            // Accept other character
				}
			}
			return currentLocation > start;
		}
		
		protected bool TryReadName(out string name)
		{
			int start = currentLocation;
			if (TryReadName()) {
				name = GetCachedString(GetText(start, currentLocation));
				return true;
			} else {
				name = string.Empty;
				return false;
			}
		}
		
		protected string GetText(int start, int end)
		{
			Log.Assert(end <= currentLocation, "Reading ahead of current location");
			return input.GetText(start, end - start);
		}
		
		Dictionary<string, string> stringCache = new Dictionary<string, string>();
		
		#if DEBUG
		int stringCacheRequestedCount;
		int stringCacheRequestedSize;
		int stringCacheStoredCount;
		int stringCacheStoredSize;
		#endif
		
		internal void PrintStringCacheStats()
		{
			#if DEBUG
			Log.WriteLine("String cache: Requested {0} ({1} bytes);  Actaully stored {2} ({3} bytes); {4}% stored", stringCacheRequestedCount, stringCacheRequestedSize, stringCacheStoredCount, stringCacheStoredSize, stringCacheRequestedSize == 0 ? 0 : stringCacheStoredSize * 100 / stringCacheRequestedSize);
			#endif
		}
		
		[Conditional("DEBUG")]
		void AddToRequestedSize(string text)
		{
			#if DEBUG
			stringCacheRequestedCount += 1;
			stringCacheRequestedSize += 8 + 2 * text.Length;
			#endif
		}
		
		[Conditional("DEBUG")]
		void AddToStoredSize(string text)
		{
			#if DEBUG
			stringCacheStoredCount += 1;
			stringCacheStoredSize += 8 + 2 * text.Length;
			#endif
		}
		
		protected string GetCachedString(string cached)
		{
			AddToRequestedSize(cached);
			// Do not bother with long strings
			if (cached.Length > 32) {
				AddToStoredSize(cached);
				return cached;
			}
			string result;
			if (stringCache.TryGetValue(cached, out result)) {
				// Get the instance from the cache instead
				return result;
			} else {
				// Add to cache
				AddToStoredSize(cached);
				stringCache.Add(cached, cached);
				return cached;
			}
		}
	}
}