File: epydoc2stan.py

package info (click to toggle)
pydoctor 19.11.0%2Bgit20200303.47424e7-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye
  • size: 1,400 kB
  • sloc: python: 6,290; makefile: 11; sh: 8
file content (631 lines) | stat: -rw-r--r-- 20,649 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
"""
Convert epydoc markup into renderable content.
"""

from __future__ import print_function

import astor

from importlib import import_module
import inspect
import itertools
import os
import sys

from pydoctor import model
from six.moves import builtins
from six.moves.urllib.parse import quote
from twisted.web.template import Tag, tags
from pydoctor.epydoc.markup import DocstringLinker, ParsedDocstring
import pydoctor.epydoc.markup.plaintext

try:
    import exceptions
except ImportError:
    exceptions = builtins


STDLIB_DIR = os.path.dirname(os.__file__)
STDLIB_URL = 'http://docs.python.org/library/'


def link(o):
    return quote(o.fullName()+'.html')


def get_parser(obj):
    formatname = obj.system.options.docformat
    try:
        mod = import_module('pydoctor.epydoc.markup.' + formatname)
    except ImportError as e:
        msg = 'Error trying to import %r parser:\n\n    %s: %s\n\nUsing plain text formatting only.'%(
            formatname, e.__class__.__name__, e)
        obj.system.msg('epydoc2stan', msg, thresh=-1, once=True)
        mod = pydoctor.epydoc.markup.plaintext
    return mod.parse_docstring


def get_docstring(obj):
    for source in obj.docsources():
        doc = source.docstring
        if doc is not None:
            if doc.strip():
                doc = inspect.cleandoc(doc)
            else:
                # Treat empty docstring as undocumented.
                doc = None
            return doc, source
    return None, None


def stdlib_doc_link_for_name(name):
    parts = name.split('.')
    for i in range(len(parts), 0, -1):
        sub_parts = parts[:i]
        filename = '/'.join(sub_parts)
        sub_name = '.'.join(sub_parts)
        if sub_name == 'os.path' \
               or os.path.exists(os.path.join(STDLIB_DIR, filename) + '.py') \
               or os.path.exists(os.path.join(STDLIB_DIR, filename, '__init__.py')) \
               or os.path.exists(os.path.join(STDLIB_DIR, 'lib-dynload', filename) + '.so') \
               or sub_name in sys.builtin_module_names:
            return STDLIB_URL + sub_name + '.html#' + name
    part0 = parts[0]
    if part0 in builtins.__dict__ and not part0.startswith('__'):
        bltin = builtins.__dict__[part0]
        if part0 in exceptions.__dict__:
            return STDLIB_URL + 'exceptions.html#exceptions.' + name
        elif isinstance(bltin, type):
            return STDLIB_URL + 'stdtypes.html#' + name
        elif callable(bltin):
            return STDLIB_URL + 'functions.html#' + name
        else:
            return STDLIB_URL + 'constants.html#' + name
    return None


class _EpydocLinker(DocstringLinker):

    def __init__(self, obj):
        self.obj = obj

    def _objLink(self, obj, prettyID):
        if obj.documentation_location is model.DocLocation.PARENT_PAGE:
            p = obj.parent
            if isinstance(p, model.Module) and p.name == '__init__':
                p = p.parent
            linktext = link(p) + '#' + quote(obj.name)
        elif obj.documentation_location is model.DocLocation.OWN_PAGE:
            linktext = link(obj)
        else:
            raise AssertionError(
                "Unknown documentation_location: %s" % obj.documentation_location)
        return tags.a(tags.code(prettyID), href=linktext)

    def look_for_name(self, name, candidates):
        part0 = name.split('.')[0]
        potential_targets = []
        for src in candidates:
            if part0 not in src.contents:
                continue
            target = src.resolveName(name)
            if target is not None and target not in potential_targets:
                potential_targets.append(target)
        if len(potential_targets) == 1:
            return potential_targets[0]
        elif len(potential_targets) > 1:
            self.obj.system.msg(
                "translate_identifier_xref", "%s:%s ambiguous ref to %s, could be %s" % (
                    self.obj.fullName(), self.obj.linenumber, name,
            ', '.join([ob.fullName() for ob in potential_targets])),
                thresh=-1)
        return None

    def look_for_intersphinx(self, name):
        """
        Return link for `name` based on intersphinx inventory.

        Return None if link is not found.
        """
        return self.obj.system.intersphinx.getLink(name)

    def translate_identifier_xref(self, fullID, prettyID):
        """Figure out what ``L{fullID}`` should link to.

        There is a lot of DWIM here.  The order goes:

          1. Check if fullID refers to an object by Python name resolution in
             our context.

          2. Walk up the object tree and see if fullID refers to an object by
             Python name resolution in each context.

          3. Check if fullID is the fullName of an object.

          4. Check to see if fullID names a builtin or standard library
             module.

          4. Walk up the object tree again and see if fullID refers to an
             object in an "uncle" object.  (So if p.m1 has a class C, the
             docstring for p.m2 can say L{C} to refer to the class in m1).  If
             at any level fullID refers to more than one object, complain.

          5. Examine every module and package in the system and see if fullID
             names an object in each one.  Again, if more than one object is
             found, complain.

        """
        src = self.obj
        while src is not None:
            target = src.resolveName(fullID)
            if target is not None:
                return self._objLink(target, prettyID)
            src = src.parent
        target = self.obj.system.objForFullName(fullID)
        if target is not None:
            return self._objLink(target, prettyID)
        fullerID = self.obj.expandName(fullID)
        linktext = stdlib_doc_link_for_name(fullerID)
        if linktext is not None:
            return tags.a(tags.code(prettyID), href=linktext)
        src = self.obj
        while src is not None:
            target = self.look_for_name(fullID, src.contents.values())
            if target is not None:
                return self._objLink(target, prettyID)
            src = src.parent
        target = self.look_for_name(fullID, itertools.chain(
            self.obj.system.objectsOfType(model.Module),
            self.obj.system.objectsOfType(model.Package)))
        if target is not None:
            return self._objLink(target, prettyID)

        target = self.look_for_intersphinx(fullerID)
        if not target:
            # FIXME: https://github.com/twisted/pydoctor/issues/125
            # expandName is unreliable so in the case fullerID fails, we
            # try our luck with fullID.
            target = self.look_for_intersphinx(fullID)
        if target:
            return tags.a(tags.code(prettyID), href=target)
        if fullID != fullerID:
            self.obj.system.msg(
                "translate_identifier_xref", "%s:%s invalid ref to '%s' "
                "resolved as '%s'" % (
                    self.obj.fullName(), self.obj.linenumber, fullID, fullerID),
                thresh=-1)
        return tags.code(prettyID)


class FieldDesc(object):
    def __init__(self):
        self.kind = None
        self.name = None
        self.type = None
        self.body = None
    def format(self):
        if self.body is None:
            body = ''
        else:
            body = self.body
        if self.type is not None:
            body = body, ' (type: ', self.type, ')'
        return body
    def __repr__(self):
        contents = []
        for k, v in self.__dict__.items():
            contents.append("%s=%r"%(k, v))
        return "<%s(%s)>"%(self.__class__.__name__, ', '.join(contents))


def format_desc_list(singular, descs, plural=None):
    if plural is None:
        plural = singular + 's'
    if not descs:
        return ''
    if len(descs) > 1:
        label = plural
    else:
        label = singular
    r = []
    first = True
    for d in descs:
        if first:
            row = tags.tr(class_="fieldStart")
            row(tags.td(class_="fieldName")(label))
            first = False
        else:
            row = tags.tr()
            row(tags.td())
        if d.name is None:
            row(tags.td(colspan="2")(d.format()))
        else:
            row(tags.td(class_="fieldArg")(d.name), tags.td(d.format()))
        r.append(row)
    return r

def format_field_list(obj, singular, fields, plural=None):
    if plural is None:
        plural = singular + 's'
    if not fields:
        return ''
    if len(fields) > 1:
        label = plural
    else:
        label = singular
    rows = []
    first = True
    for field in fields:
        if first:
            row = tags.tr(class_="fieldStart")
            row(tags.td(class_="fieldName")(label))
            first=False
        else:
            row = tags.tr()
            row(tags.td())
        row(tags.td(colspan="2")(field.body))
        rows.append(row)
    return rows


class Field(object):
    """Like pydoctor.epydoc.markup.Field, but without the gross accessor
    methods and with a formatted body."""
    def __init__(self, field, obj):
        self.tag = field.tag()
        self.arg = field.arg()
        self.body = field.body().to_stan(_EpydocLinker(obj))

    def __repr__(self):
        r = repr(self.body)
        if len(r) > 25:
            r = r[:20] + '...' + r[-2:]
        return "<%s %r %r %s>"%(self.__class__.__name__,
                             self.tag, self.arg, r)

class FieldHandler(object):
    def __init__(self, obj):
        self.obj = obj

        self.types = {}

        self.parameter_descs = []
        self.return_desc = None
        self.raise_descs = []
        self.seealsos = []
        self.notes = []
        self.authors = []
        self.sinces = []
        self.unknowns = []
        self.unattached_types = {}

    def redef(self, field):
        self.obj.system.msg(
            "epytext",
            "on %r: redefinition of @type %s"%(self.obj.fullName(), field.arg),
            thresh=-1)

    def handle_return(self, field):
        if not self.return_desc:
            self.return_desc = FieldDesc()
        if self.return_desc.body:
            self.obj.system.msg('epydoc2stan', 'XXX')
        self.return_desc.body = field.body
    handle_returns = handle_return

    def handle_returntype(self, field):
        if not self.return_desc:
            self.return_desc = FieldDesc()
        if self.return_desc.type:
            self.obj.system.msg('epydoc2stan', 'XXX')
        self.return_desc.type = field.body
    handle_rtype = handle_returntype

    def add_type_info(self, desc_list, field):
        if desc_list and desc_list[-1].name == field.arg:
            if desc_list[-1].type is not None:
                self.redef(field)
            desc_list[-1].type = field.body
        else:
            d = FieldDesc()
            d.kind = field.tag
            d.name = field.arg
            d.type = field.body
            desc_list.append(d)

    def add_info(self, desc_list, field):
        d = FieldDesc()
        d.kind = field.tag
        d.name = field.arg
        d.body = field.body
        desc_list.append(d)

    def handle_type(self, field):
        self.types[field.arg] = field.body

    def handle_param(self, field):
        self.add_info(self.parameter_descs, field)
    handle_arg = handle_param
    handle_keyword = handle_param


    def handled_elsewhere(self, field):
        # Some fields are handled by extract_fields below.
        pass

    handle_ivar = handled_elsewhere
    handle_cvar = handled_elsewhere
    handle_var = handled_elsewhere

    def handle_raises(self, field):
        self.add_info(self.raise_descs, field)
    handle_raise = handle_raises

    def handle_seealso(self, field):
        self.seealsos.append(field)
    handle_see = handle_seealso

    def handle_note(self, field):
        self.notes.append(field)

    def handle_author(self, field):
        self.authors.append(field)

    def handle_since(self, field):
        self.sinces.append(field)

    def handleUnknownField(self, field):
        self.obj.system.msg(
            'epydoc2stan',
            'found unknown field on %r: %r'%(self.obj.fullName(), field),
            thresh=-1)
        self.add_info(self.unknowns, field)

    def handle(self, field):
        m = getattr(self, 'handle_' + field.tag, self.handleUnknownField)
        m(field)

    def resolve_types(self):
        for pd in self.parameter_descs:
            if pd.name in self.types:
                pd.type = self.types[pd.name]

    def format(self):
        r = []

        r.append(format_desc_list('Parameters', self.parameter_descs, 'Parameters'))
        if self.return_desc:
            r.append(tags.tr(class_="fieldStart")(tags.td(class_="fieldName")('Returns'),
                               tags.td(colspan="2")(self.return_desc.format())))
        r.append(format_desc_list("Raises", self.raise_descs, "Raises"))
        for s, p, l in (('Author', 'Authors', self.authors),
                        ('See Also', 'See Also', self.seealsos),
                        ('Present Since', 'Present Since', self.sinces),
                        ('Note', 'Notes', self.notes)):
            r.append(format_field_list(self.obj, s, l, p))
        unknowns = {}
        unknownsinorder = []
        for fieldinfo in self.unknowns:
            tag = fieldinfo.kind
            if tag in unknowns:
                unknowns[tag].append(fieldinfo)
            else:
                unknowns[tag] = [fieldinfo]
                unknownsinorder.append(unknowns[tag])
        for fieldlist in unknownsinorder:
            label = "Unknown Field: " + fieldlist[0].kind
            r.append(format_desc_list(label, fieldlist, label))

        return tags.table(class_='fieldTable')(r)


def reportErrors(obj, errs):
    for err in errs:
        if isinstance(err, str):
            linenumber = '??'
            descr = err
        else:
            obj_linenum = getattr(obj, 'linenumber', None)
            if obj_linenum is None:
                linenumber = '??'
            else:
                linenumber = str(obj_linenum + err.linenum())
            descr = err._descr
        obj.system.msg(
            'epydoc2stan2',
            '%s:%s epytext error %r' % (obj.fullName(), linenumber, descr))
    if errs and obj.fullName() not in obj.system.epytextproblems:
        obj.system.epytextproblems.append(obj.fullName())
        obj.system.msg('epydoc2stan',
                       'epytext error in %s'%(obj,), thresh=1)
        p = lambda m:obj.system.msg('epydoc2stan', m, thresh=2)
        for i, l in enumerate(obj.docstring.splitlines()):
            p("%4s"%(i+1)+' '+l)
        for err in errs:
            p(err)


def parse_docstring(obj, doc, source):
    """Parse a docstring.
    @rtype: L{ParsedDocstring}
    """

    parser = get_parser(obj)
    errs = []
    try:
        pdoc = parser(doc, errs)
    except Exception as e:
        errs.append('%s: %s' % (e.__class__.__name__, e))
        pdoc = None
    if pdoc is None:
        pdoc = pydoctor.epydoc.markup.plaintext.parse_docstring(doc, errs)
    if errs:
        reportErrors(source, errs)
    return pdoc


def format_docstring(obj):
    """Generate an HTML representation of a docstring"""

    doc, source = get_docstring(obj)

    # Use cached or split version if possible.
    pdoc = getattr(obj, 'parsed_docstring', None)

    if pdoc is None:
        if doc is None:
            return tags.div(class_='undocumented')("Undocumented")
        pdoc = parse_docstring(obj, doc, source)
        obj.parsed_docstring = pdoc
    elif source is None:
        # A split field is documented by its parent.
        source = obj.parent

    try:
        stan = pdoc.to_stan(_EpydocLinker(source))
    except Exception as e:
        errs = ['%s: %s' % (e.__class__.__name__, e)]
        if doc is None:
            stan = tags.p(class_="undocumented")('Broken description')
        else:
            pdoc_plain = pydoctor.epydoc.markup.plaintext.parse_docstring(doc, errs)
            stan = pdoc_plain.to_stan(_EpydocLinker(source))
        reportErrors(source, errs)

    content = [stan] if stan.tagName else stan.children
    fields = pdoc.fields
    s = tags.div(*content)
    if fields:
        fh = FieldHandler(obj)
        for field in fields:
            fh.handle(Field(field, obj))
        fh.resolve_types()
        s(fh.format())
    return s


def format_summary(obj):
    """Generate an shortened HTML representation of a docstring."""

    doc, source = get_docstring(obj)
    if doc is None:
        # Attributes can be documented as fields in their parent's docstring.
        if isinstance(obj, model.Attribute):
            pdoc = getattr(obj, 'parsed_docstring', None)
        else:
            pdoc = None
        if pdoc is None:
            return format_undocumented(obj)
        source = obj.parent
    else:
        # Use up to three first non-empty lines of doc string as summary.
        lines = itertools.dropwhile(lambda line: not line.strip(),
                                    doc.split('\n'))
        lines = itertools.takewhile(lambda line: line.strip(), lines)
        lines = [ line.strip() for line in lines ]
        if len(lines) > 3:
            return tags.span(class_='undocumented')("No summary")
        pdoc = parse_docstring(obj, ' '.join(lines), source)

    try:
        stan = pdoc.to_stan(_EpydocLinker(source))
    except Exception:
        # This problem will likely be reported by the full docstring as well,
        # so don't spam the log.
        return tags.span(class_='undocumented')("Broken description")

    content = [stan] if stan.tagName else stan.children
    if content and isinstance(content[0], Tag) and content[0].tagName == 'p':
        content = content[0].children
    return tags.span(*content)


def format_undocumented(obj):
    """Generate an HTML representation for an object lacking a docstring."""
    subdocstrings = {}
    subcounts = {}
    for subob in obj.contents.values():
        k = subob.kind.lower()
        subcounts[k] = subcounts.get(k, 0) + 1
        if subob.docstring is not None:
            subdocstrings[k] = subdocstrings.get(k, 0) + 1
    if isinstance(obj, model.Package):
        subcounts['module'] -= 1
    if subdocstrings:
        plurals = {'class': 'classes'}
        text = "No %s docstring; %s documented" % (
            obj.kind.lower(),
            ', '.join(
                "%s/%s %s" % (subdocstrings.get(k, 0), subcounts[k],
                              plurals.get(k, k + 's'))
                for k in sorted(subcounts)
                )
            )
    else:
        text = "Undocumented"
    return tags.span(class_='undocumented')(text)


def type2stan(obj):
    parsed_type = get_parsed_type(obj)
    if parsed_type is None:
        return None
    else:
        return parsed_type.to_stan(_EpydocLinker(obj))

def get_parsed_type(obj):
    parsed_type = getattr(obj, 'parsed_type', None)
    if parsed_type is not None:
        return parsed_type

    annotation = getattr(obj, 'annotation', None)
    if annotation is not None:
        return AnnotationDocstring(annotation)

    return None


class AnnotationDocstring(ParsedDocstring):

    def __init__(self, annotation):
        ParsedDocstring.__init__(self, ())
        self.annotation = annotation

    def to_stan(self, docstring_linker):
        src = astor.to_source(self.annotation).strip()
        return tags.code(src)


field_name_to_human_name = {
    'ivar': 'Instance Variable',
    'cvar': 'Class Variable',
    'var': 'Variable',
    }


def extract_fields(obj):
    doc, source = get_docstring(obj)
    if doc is None:
        return

    pdoc = parse_docstring(obj, doc, source)
    obj.parsed_docstring = pdoc

    for field in pdoc.fields:
        tag = field.tag()
        if tag in ['ivar', 'cvar', 'var', 'type']:
            arg = field.arg()
            if arg is None:
                obj.system.msg('epydoc2stan', '%s: Missing field name in @%s'
                               % (obj.fullName(), tag))
                continue
            attrobj = obj.contents.get(arg)
            if attrobj is None:
                attrobj = obj.system.Attribute(obj.system, arg, None, obj)
                attrobj.kind = None
                obj.system.addObject(attrobj)
            if tag == 'type':
                attrobj.parsed_type = field.body()
            else:
                attrobj.parsed_docstring = field.body()
                attrobj.kind = field_name_to_human_name[tag]