File: genexamp.py

package info (click to toggle)
allegro4.2 2:4.2.0-5
  • links: PTS
  • area: main
  • in suites: etch, etch-m68k
  • size: 24,436 kB
  • ctags: 14,714
  • sloc: ansic: 126,425; asm: 17,011; cpp: 3,846; sh: 2,120; objc: 925; makefile: 715; python: 216; pascal: 179; perl: 73
file content (421 lines) | stat: -rw-r--r-- 14,852 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
#!/usr/bin/env python
# -*- mode:Python; tab-width: 3 -*-
"""
This is a helper script which reads the .c files of the examples
directory, extracts the comments from the source, and updates
allegro._tx's appropriate section with the comments after
hyperlinking functions and examples. It is not meant to be run by
the end user, only by the maintainer packaging Allegro.

To run the script go to the Allegro root dir and type "python
misc/genexamp.py". If everything goes ok, you will end up with a
patch in stdout you can apply manually after having verified it is
ok. Usage example:

   python misc/genexamp.py | less
   ...review...
   python misc/genexamp.py | patch -p0

In order to work, this script requires Python (tested with 2.3.3) and
the diff binary in your path. Written by Grzegorz Adam Hankiewicz,
gradha@users.sourceforge.net. Notify me of any broken behaviour,
like uncaught exceptions. This script falls under Allegro's giftware
license.
"""
import glob
import os
import os.path
import popen2
import re
import string
import sys

path_to_documentation = apply(os.path.join, ["docs", "src", "allegro._tx"])
path_to_example_dir = apply(os.path.join, ["examples"])
# number of max erefs, below 1 desactivates limit
limit_example_erefs = 10
limit_example_reference = "Available Allegro examples"

# text tags used to detect where the examples should be put
START_MARK = "start genexamp.py chunk"
END_MARK = "end genexamp.py chunk"
XREF_WIDTH = 70
regular_expression_for_tx_identifiers = r"@[@\\][^@]+@(?P<name>\w+)"
comment_line = re.compile(r"Example pr|Modified by")

# the order of the examples is aimed at the newbie, going from easy
# to difficult
examples_order = ["exhello", "exmem", "expal", "expat", "exflame",
   "exdbuf", "exflip", "exfixed", "exfont", "exmouse", "extimer", "exkeys",
   "exjoy", "exsample", "exmidi", "exgui", "excustom", "exunicod",
   "exbitmap", "exscale", "exconfig", "exdata", "exsprite", "exexedat",
   "extrans", "extruec", "excolmap", "exrgbhsv", "exshade", "exblend",
   "exxfade", "exalpha", "exlights", "ex3d", "excamera", "exquat",
   "exstars", "exscn3d", "exzbuf", "exscroll", "ex3buf", "ex12bit",
   "exaccel", "exspline", "exsyscur", "exupdate", "exswitch",
   "exstream", "expackf"]

# Holds examples' short descriptions. Loaded from disk at runtime.
short_descriptions = {}


def detect_all_available_examples(path):
   """func(path_to_directory)

   Checks if the maintainer is doing his work and didn't miss
   some example in the global list, printing any missing examples
   to stderr.
   """
   examples = filter(lambda x: x not in examples_order,
      map(lambda x: os.path.splitext(os.path.basename(x))[0],
      glob.glob(os.path.join(path, "*.c"))))
   if examples:
      sys.stderr.write("Warning! There are examples not listed in the default\n"
         "examples_order variable. You should correct this:\n")
      for example in examples:
         sys.stderr.write("\t%s\n" % example)
      examples_order.extend(examples)
      


def load_example_short_descriptions(path):
   """func(path_to_directory)

   Loads the file examples.txt from the specified directory and
   extracts the short example descriptions which are in "exblah -
   text" format. The short descriptions are stored in the global
   dictionary short_descriptions.
   """
   file_input = file(os.path.join(path, "examples.txt"), "rt")
   try:
      regex = re.compile(r"(ex\w+)[.]c\s+-\s(.*)")
      line = file_input.readline()
      while line:
         m = regex.match(line)
         if m:
            short_descriptions[m.group(1)] = m.group(2)
            
         line = file_input.readline()
   finally:
      file_input.close()

   

def retrieve_documentation_identifiers(file_name):
   """func(path_to_documentation._tx) -> {documentation_identifiers}

   Used on a text file with the format of Allegro's documentation,
   extracts all the valid identifiers and returns them as the keys
   of a dictionary, whose values are all 1 and can be ignored.
   """
   dic = {}
   exp = re.compile(regular_expression_for_tx_identifiers)
   file = open(file_name, "rt")
   for line in file.readlines():
      match = exp.match(line)
      # avoid the identifiers of existant examples
      if match and match.group("name") not in examples_order:
         dic[match.group("name")] = 1
   file.close()
   return dic

   

def retrieve_source_data(file_name):
   """func(path_to_example.c) -> ([comment_lines], {detected_identifiers})

   Used on the source code of an example, extracts the first comment
   after deleting the initial lines which credit authors. The comment
   will be used as the text for the documentation. Also returns a
   dictionary with all the keys being hypotetical identifiers used
   in the code, that is, every single word susceptible of being
   used as an allegro function, like return, printf, allegro_init,
   SCREEN_W, etc.
   """
   dic = {}
   exp = re.compile(r"\w+")
   file = open(file_name, "rt")
   getting_comment = 1
   comment = []
   for line in file.readlines():
      if getting_comment:
         # stop reading after first comment end marker
         if string.find(line, "*/") >= 0:
            getting_comment = 0
         else:
            # ok, retrieve all lines removing first 3 characters
            comment.append(string.strip(line[2:]))
      else:
         # retrieve all possible keywords
         for match in re.findall(exp, line):
            dic[match] = 1
   file.close()

   # post-process comment, removing initial credits
   while len(comment):
      if string.strip(comment[0]) == "":
         comment.pop(0)
      elif comment_line.search(comment[0]):
         while len(comment) and string.strip(comment[0]) != "":
            comment.pop(0)
      else:
         break
   return comment, dic

   

def get_valid_example_ids(example_ids, global_ids, incremental = None):
   """func({example_ids}, {global_ids}, bool) -> [valid_identifiers]

   Compares the dictionary of a source example with all its possible
   identifiers against the dictionary with all the available
   documentation identifiers and returns a filtered list of the
   example identifiers which are also in the global dictionary.

   If the incremental parameter is true, the identifiers of
   global_ids that are also contained in example_ids will be
   erased. This is useful if you are building example after example
   and you don't want new lists to include the identifiers returned
   in previous calls to this function.
   """
   example_ids = example_ids.keys()
   example_ids.sort()
   valid_ids = []
   if incremental:
      for id in example_ids:
         try:
            del global_ids[id]
            valid_ids.append(id)
         except KeyError:
            pass
   else:
      for id in example_ids:
         if global_ids.has_key(id):
            valid_ids.append(id)
   return valid_ids



def build_formatted_lines(word_list):
   """func([word1, word2, ...]) -> [line1, line2, ...]

   Consumes the given list of words and returns a list of text
   lines. Each line will contain several words joined with the string
   ", ". All lines will have a width <= XREF_WIDTH.
   """
   new_list = []
   new_line = []
   length = 0
   while len(word_list) > 0:
      new_word = word_list.pop(0)
      if length + len(new_word) + 2 < XREF_WIDTH:
         length = length + len(new_word) + 2
         new_line.append(new_word)
      else:
         new_list.append(string.join(new_line, ", "))
         new_line = [new_word]
         length = len(new_word) + 2

   new_list.append(string.join(new_line, ", "))
   return new_list

      

def build_xref_block(xref_list, xref_tag = None):
   """func([word1, word2, ...], string) -> [line1, line2, ...]

   Behaves like build_formatted_lines, adding to the beginning of
   each returned line the string given as second parameter. If the
   string is None, "@xref" will be prepended.
   """
   lines = []
   if not xref_tag: xref_tag = "@xref"
   for line in build_formatted_lines(xref_list):
      lines.append("%s %s\n" % (xref_tag, line))
   return lines


   
def build_example_output(example_name, comment_lines, xref_list):
   """func(example_name, [comment], [xrefs]) -> [formatted_lines]

   Given an example name (the part of the file_name without
   extension), its source commentary, and the references it makes,
   returns a list of text lines formatted like this:

   @@Example @example_name
   @@xrefs
   @shortdesc
      comment
   """
   lines = ["@@Example @%s\n" % example_name]
   lines.extend(build_xref_block(xref_list))
   if short_descriptions.has_key(example_name):
      lines.append("@shortdesc %s\n" % short_descriptions[example_name])
   lines.extend(map(lambda x: "   %s\n" % x, comment_lines))
   lines.append("\n")
   return lines


def generate_example_chapter(path_to_documentation, example_files, incremental = None):
   """func(path._tx, [paths_to_examples], bool) -> ([doc], {ids_to_examples})

   First retrieves fom path_to_documentation the valid identifiers.
   Then goes through the list of paths to examples and extracts
   from each of them the used identifiers. Returns a list of lines
   which contains the new generated documentation. Also returns a
   dictionary whose keys are identifiers found in the documentation
   file, with the values being a list of the examples which refer
   to the key.

   If incremental is false, each documented example will reference
   all the possible identifiers found in the documentation. But if
   it's true, each example will create references only to identifiers
   which still have not been referenced before.
   """
   reverse_docs = {}
   doc_ids = retrieve_documentation_identifiers(path_to_documentation)
   lines = []
   for example in example_files:
      comment, ex_ids = retrieve_source_data(example)
      name = os.path.splitext(os.path.basename(example))[0]
      valid_ids = get_valid_example_ids(ex_ids, doc_ids, incremental)
      # build reverse lookup
      for id in valid_ids:
         try:
            reverse_docs[id].append(name)
         except KeyError:
            reverse_docs[id] = [name]
      # finally add the text chunk
      lines.extend(build_example_output(name, comment, valid_ids))
   return lines, reverse_docs

   

def replace_example_chapter(path_to_documentation, chapter_lines):
   """func(path_to_doc._tx, [new_chapter]) -> [regenerated_documentation]

   Opens the documentation and searches for the text section
   separated through the global marks START_MARK/END_MARK. Returns
   the opened file with that section replaced by chapter_lines.
   """
   lines = []
   file = open(path_to_documentation, "rt")
   state = 0
   for line in file.readlines():
      if state == 0:
         lines.append(line)
         if string.find(line, START_MARK) > 0:
            state = state + 1
      elif state == 1:
         if string.find(line, END_MARK) > 0:
            lines.extend(chapter_lines)
            lines.append(line)
            state = state + 1
      else:
         lines.append(line)

   return lines

   

def find_differences(file_name, lines):
   """func(path_to_doc._tx, [text_lines]) -> [diff_lines]

   This function runs diff over the given document file and text
   lines.  To make the comparison a temporary file with extension
   .tmp is build and later removed. The function returns the output
   of the diff commant as a list of lines.
   """
   temp_name = string.replace(path_to_documentation, "._tx", ".tmp")
   file = open(temp_name, "wt")
   file.writelines(lines)
   file.close()

   # call external tool and read it's stdout
   stdout, stdin = popen2.popen2 (["diff", "-u", file_name, temp_name])
   stdin.close()
   diff_lines = stdout.readlines()
   stdout.close()
   os.unlink(temp_name)
   return diff_lines


   
def replace_example_references(documentation, ids_to_examples):
   """func([lines], {id: [words]}) -> [new_lines]

   Goes through the documentation in memory searching for the
   identifiers.  When one is found, it looks it up in the provided
   dictionary. If it's not found, nothing happens. Otherwise whatches
   closely the following text lines for @erefs and updates them
   accordingly. If no @erefs are found, they are created. Finally
   a new list is returned with the updated documentation.

   There is a special hardcoded case: if limit_example_erefs
   is greater than 0, this value will be used as the limit of
   example references generated per identifier. When the number
   of example references is greater than this limit, a single
   reference is generated to the value stored in the string
   limit_example_reference, which should point to the correct
   Allegro documentation chapter.
   """
   new_lines = []
   short_desc = []
   found_id = ""
   exp = re.compile(regular_expression_for_tx_identifiers)
   for line in documentation:
      if found_id:
         if line[0] == '@':
            # Don't append erefs, which will be regenerated.
            if line[1:5] == "xref" or line[1] == "@" or line[1:3] == "\\ ":
               new_lines.append(line)
            # Append shortdesc, but after @erefs as a special case.
            if line[1:10] == "shortdesc":
               short_desc = [line]
         else:
            # create the eref block and append before normal text
            eref_list = ids_to_examples[found_id]
            eref_list.sort()
            if len(eref_list) > limit_example_erefs:
               new_lines.append("@eref %s\n" % limit_example_reference)
            else:
               new_lines.extend(build_xref_block(eref_list, "@eref"))
               
            if short_desc:
               new_lines.extend(short_desc)
               short_desc = []
               
            new_lines.append(line)
            found_id = ""
      else:
         if short_desc:
            new_lines.extend(short_desc)
            short_desc = []
            
         new_lines.append(line)
         match = exp.match(line)
         if match and ids_to_examples.has_key(match.group("name")):
            found_id = match.group("name")
            
   return new_lines

   

def main(path_to_documentation, path_to_example_dir, incremental):        
   detect_all_available_examples(path_to_example_dir)
   load_example_short_descriptions(path_to_example_dir)
   chapter, ids_to_examples = generate_example_chapter(path_to_documentation,
      map(lambda x: os.path.join(path_to_example_dir, "%s.c" % x),
      examples_order), incremental)

   documentation = replace_example_chapter(path_to_documentation, chapter)
   documentation = replace_example_references(documentation, ids_to_examples)

   differences = find_differences(path_to_documentation, documentation)
   sys.stdout.writelines(differences)



if __name__ == "__main__":
   main(path_to_documentation, path_to_example_dir, None)