1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
|
#!/usr/local/bin/python
""" pytag - tag strings given a table of things to look for (Version 0.6)
* includes the Python version pytag() of the tagging engine
* this module includes a debugger for tag tables; to enable it,
call pytag.use_debugger(); when the debugger prompts, enter 'h'
to see a help screen
* it also allows for verbose output while trying to tag a string;
call pytag.set_verbosity(1) to see the whole process of tagging
XXX: doesn't know anything about the new commands introduced in
the C version !!!
XXX: NOT SUPPORTED ANYMORE (well, at least for now)
Copyright (c) 2000, Marc-Andre Lemburg; mailto:mal@lemburg.com
Copyright (c) 2000-2014, eGenix.com Software GmbH; mailto:info@egenix.com
See the documentation for further information on copyrights,
or contact the author. All Rights Reserved.
"""
import sys
# globals
verbose = 0
debugging = 0
breakpoints = []
# some useful constants
from mx.TextTools import *
import types
strtype = types.StringType
tabletype = types.TupleType
fcttype = types.FunctionType
MatchError = 'MatchError'
not_set = 0
#
# Python version of tag():
#
def pytag(text,table,startindex=0,len_text=not_set,taglist=not_set):
"""Tag text[startindex:len_text] using the Tag Table given in table.
- returns a tuple (success, taglist, nextindex)
- if taglist == None, then no taglist is created
"""
global debugging
if len_text is not_set:
# use all chars in text
len_text = len(text)
if taglist is not_set:
# create a fresh new list
taglist = []
if verbose: print '\ntag()-call'
i = 0
x = startindex
loopcounter = -1
while i < len(table):
entry = table[i]
lentry = len(entry)
if verbose:
print 'text[%i]:table[%i]\tentry\t%s'%(x,i,format_entry(table,i))
print '\t\t\tin\t%s'%(repr(text[x:len_text])[:40])
if lentry == 4:
tagobj, cmd, match, jne = entry
je = +1
elif lentry == 5:
tagobj, cmd, match, jne, je = entry
else:
tagobj, cmd, match = entry
jne = 0
je = +1
if debugging:
if debugging == 3 and (x not in breakpoints):
pass
else:
debugging,x,i = debugger(text,x,len_text,table,i,loopcounter,taglist)
subtags = None
flags = cmd & ~0xFF
cmd = cmd & 0xFF
if cmd == AllIn:
if type(match) != strtype: raise TypeError,'match must be a string'
start = x
while x < len_text and text[x] in match:
x = x + 1
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nAllIn matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == AllNotIn:
if type(match) != strtype: raise TypeError,'match must be a string'
start = x
while x < len_text and text[x] not in match:
x = x + 1
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nAllNotIn matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == Is:
if type(match) != strtype: raise TypeError,'match must be a string'
start = x
if x < len_text and text[x] == match:
x = x + 1
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nIs matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == IsIn:
if type(match) != strtype: raise TypeError,'match must be a string'
start = x
if x < len_text and text[x] in match:
x = x + 1
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nIsIn matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == IsNotIn:
if type(match) != strtype: raise TypeError,'match must be a string'
start = x
if x < len_text and text[x] not in match:
x = x + 1
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nIsNotIn matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == Word:
if type(match) != strtype: raise TypeError,'match must be a string'
start = x
if text[x:x+len(match)] == match and x < len_text:
x = x + len(match)
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nWord matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == NoWord:
if type(match) != strtype: raise TypeError,'match must be a string'
start = x
lw = len(match)
while x < len_text and text[x:x+lw] != match:
x = x + 1
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nNoWord matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == Table:
if type(match) != tabletype: raise TypeError,'match must be a table'
start = x
if debugging == 2:
d = debugging
debugging = 0
result, subtags, nextindex = tag(text,match,x,len_text,[])
debugging = d
else:
result, subtags, nextindex = tag(text,match,x,len_text,[])
if not result:
if verbose: print '\nTable did not match\n'
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
x = nextindex
if verbose: print '\nTable matched',repr(tagobj),start,':',x,repr(text[x:nextindex])[:40]
elif cmd == TableInList:
start = x
tablelist, entry = match
if debugging == 2:
d = debugging
debugging = 0
result, subtags, nextindex = tag(text,tablelist[entry],x,len_text,[])
debugging = d
else:
result, subtags, nextindex = tag(text,tablelist[entry],x,len_text,[])
if not result:
if verbose: print '\nTableInList did not match\n'
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
x = nextindex
if verbose: print '\nTableInList matched',repr(tagobj),start,':',x,repr(text[x:nextindex])[:40]
elif cmd == Call:
start = x
x = match(text,start,len_text)
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nCall matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == CallArg:
start = x
try:
fct, arg = match
except:
raise TypeError,'match must be a tuple (fct,arg)'
x = fct(text,start,len_text,arg)
if start == x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
if verbose: print '\nCallArg matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
elif cmd == Loop:
start = x
# loop-construct
if loopcounter > 0:
# next
loopcounter = loopcounter - 1
elif loopcounter < 0:
# init
loopcounter = match
loopstartpos = x
if verbose: print '\nLoop count =',loopcounter,' startpos =',loopstartpos
if loopcounter == 0:
# end
loopcounter = -1
# matched
start = loopstartpos
if verbose: print '\nLoop matched',repr(tagobj),start,':',x,repr(text[start:x])[:40]
je = jne
else:
# continue loops
i = i + je
continue
elif cmd == LoopControl:
loopcounter = match
i = i + je
continue
elif cmd == EOF:
# match end-of-string
if len_text > x:
# not matched
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
else:
# matched
x = len_text
if verbose: print '\nmatched EOF'
je = MatchOk
elif cmd == Fail:
if not jne:
# match failed
return (0,taglist,x)
# try an other entry
i = i + jne
continue
elif cmd == Skip:
# skip match bytes (back or forward) in text
x = x + match
if verbose:
if match > 0: y,z = x - match,x
else: y,z = x,x - match
print '\nSkip ',repr(tagobj),'to',x,'skipped:',repr(text[y:z])[:40]
else:
raise TypeError,'tag-command unknown: '+repr(cmd)
# append to taglist
if tagobj is not None:
if flags > 0:
if flags & CallTag:
tagobj(taglist,text,start,x,subtags)
elif flags & AppendToTag:
tagobj.append((start,x,subtags))
else:
raise TypeError,'tag-command-flag unknown: '+repr(flag)
else:
if taglist is not None:
taglist.append((tagobj,start,x,subtags))
if verbose: print 72*'-'
# goto next table entry
i = i + je
# table matched ok
return (1,taglist,x)
#
# Override the C version
#
TextTools.tag = pytag
#
# get all the goodies from TextTools
#
from mx.TextTools import *
#
# setup functions for the Python version pytag()
#
def set_verbosity(level = 1):
""" set verbosity for tagging: 0=off 1=on """
global verbose
verbose = level
def use_debugger(bp=[]):
""" use the tag table debugger
* bp can be a list of preset breakpoints (bytes into text)
"""
global debugging, breakpoints
debugging = 1
breakpoints = bp
def debugger(text,x,len_text,table,i,loopvar,taglist):
""" the tag table debugger front end
* returns (rc,x,i)
with rc = 0 ... stop debugging #
= 1 ... do one step #
= 2 ... step over table #
"""
global breakpoints, verbose
rc = -1
print 75*'_'
print '| table[%i]=(%s)'%(i,format_entry(table,i))
print '| text[%i:]=%s'%(x,repr(text[x:len_text])[:55])
print '| h = help; vars: loop counter =',loopvar
print 75*'-'
while rc < 0:
s = raw_input('tag-debugger >>> ')
try:
if len(s) > 0:
c = s[0]
else:
c = 's' # hitting return is like entering 's'
if c == 's':
rc = 1
elif c == 'n':
rc = 2
elif c == 'w':
print 'table[',i,']=(',format_entry(table,i),')'
print 'text [',x,']:',repr(text[x:len_text])[:55]
print 'vars: loop counter =',loopvar
elif c == 'q':
rc = 0
elif c == 'r':
rc = 3
elif c == 'b':
breakpoints.append(int(s[1:]))
elif c == 'c':
breakpoints.remove(int(s[1:]))
elif c == 'l':
print 'breakpoints:'
for b in breakpoints:
print ' ',b,'text =',repr(text[b:])[:60]
elif c == 'g':
x = int(s[1:])
elif c == 'v':
verbose = 1 - verbose
elif c == 't':
print 'taglist:'
print_tags(text,taglist)
elif c == 'm':
print 'current matching table:'
print format_table(table,i)
elif c == 'p':
y = int(s[1:])
print 'text[',y,']:',repr(text[y:len_text])[:60]
else:
raise 'help'
except 'help':
print 'Commands:'
print 's = step next entry | return = step'
print 'w = where are we | q = run without debugging'
print 'b*= add breakpoint * | c*= delete breakpoint *'
print 'l = list breakpoints | v = switch verbosity on/off'
print 't = show taglist | r = run to next breakpoint/end'
print 'n = step over table | p*= print text from byte *'
print 'g*= goto position * | m = show current matching table'
print
print '*...these take an argument, e.g. b1200 or p 1200'
except:
print 'Internal Debugger Error -- last request not processed !'
if verbose: print 72*'_'
return rc,x,i
|
