File: doctools.py

package info (click to toggle)
smart-open 7.5.0-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 980 kB
  • sloc: python: 8,054; sh: 90; makefile: 14
file content (261 lines) | stat: -rw-r--r-- 7,081 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
# -*- coding: utf-8 -*-
#
# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com>
#
# This code is distributed under the terms and conditions
# from the MIT License (MIT).
#

"""Common functions for working with docstrings.

For internal use only.
"""

import contextlib
import inspect
import io
import os.path
import re

from . import compression

PLACEHOLDER = '    smart_open/doctools.py magic goes here'


def extract_kwargs(docstring):
    """Extract keyword argument documentation from a function's docstring.

    Parameters
    ----------
    docstring: str
        The docstring to extract keyword arguments from.

    Returns
    -------
    list of (str, str, list str)

    str
        The name of the keyword argument.
    str
        Its type.
    str
        Its documentation as a list of lines.

    Notes
    -----
    The implementation is rather fragile.  It expects the following:

    1. The parameters are under an underlined Parameters section
    2. Keyword parameters have the literal ", optional" after the type
    3. Names and types are not indented
    4. Descriptions are indented with 4 spaces
    5. The Parameters section ends with an empty line.

    Examples
    --------

    >>> docstring = '''The foo function.
    ... Parameters
    ... ----------
    ... bar: str, optional
    ...     This parameter is the bar.
    ... baz: int, optional
    ...     This parameter is the baz.
    ...
    ... '''
    >>> kwargs = extract_kwargs(docstring)
    >>> kwargs[0]
    ('bar', 'str, optional', ['This parameter is the bar.'])

    """
    if not docstring:
        return []

    lines = inspect.cleandoc(docstring).split('\n')
    kwargs = []

    #
    # 1. Find the underlined 'Parameters' section
    # 2. Once there, continue parsing parameters until we hit an empty line
    #
    while lines and lines[0] != 'Parameters':
        lines.pop(0)

    if not lines:
        return []

    lines.pop(0)
    lines.pop(0)

    for line in lines:
        if not line.strip():  # stop at the first empty line encountered
            break
        is_arg_line = not line.startswith(' ')
        if is_arg_line:
            name, type_ = line.split(':', 1)
            name, type_, description = name.strip(), type_.strip(), []
            kwargs.append([name, type_, description])
            continue
        is_description_line = line.startswith('    ')
        if is_description_line:
            kwargs[-1][-1].append(line.strip())

    return kwargs


def to_docstring(kwargs, lpad=''):
    """Reconstruct a docstring from keyword argument info.

    Basically reverses :func:`extract_kwargs`.

    Parameters
    ----------
    kwargs: list
        Output from the extract_kwargs function
    lpad: str, optional
        Padding string (from the left).

    Returns
    -------
    str
        The docstring snippet documenting the keyword arguments.

    Examples
    --------

    >>> kwargs = [
    ...     ('bar', 'str, optional', ['This parameter is the bar.']),
    ...     ('baz', 'int, optional', ['This parameter is the baz.']),
    ... ]
    >>> print(to_docstring(kwargs), end='')
    bar: str, optional
        This parameter is the bar.
    baz: int, optional
        This parameter is the baz.

    """
    buf = io.StringIO()
    for name, type_, description in kwargs:
        buf.write('%s%s: %s\n' % (lpad, name, type_))
        for line in description:
            buf.write('%s    %s\n' % (lpad, line))
    return buf.getvalue()


def extract_examples_from_readme_rst(indent='    '):
    """Extract examples from this project's README.rst file.

    Parameters
    ----------
    indent: str
        Prepend each line with this string.  Should contain some number of spaces.

    Returns
    -------
    str
        The examples.

    Notes
    -----
    Quite fragile, depends on named labels inside the README.rst file.
    """
    curr_dir = os.path.dirname(os.path.abspath(__file__))
    readme_path = os.path.join(curr_dir, '..', 'README.rst')
    try:
        with open(readme_path) as fin:
            lines = list(fin)
        start = lines.index('.. _doctools_before_examples:\n')
        end = lines.index(".. _doctools_after_examples:\n")
        lines = lines[start+4:end-2]
        return ''.join([indent + re.sub('^  ', '', line) for line in lines])
    except Exception:
        return indent + 'See README.rst'


def tweak_open_docstring(f):
    buf = io.StringIO()
    seen = set()

    root_path = os.path.dirname(os.path.dirname(__file__))

    with contextlib.redirect_stdout(buf):
        print('    smart_open supports the following transport mechanisms:')
        print()
        for scheme, submodule in sorted(transport._REGISTRY.items()):
            if scheme == transport.NO_SCHEME or submodule in seen:
                continue
            seen.add(submodule)

            try:
                schemes = submodule.SCHEMES
            except AttributeError:
                schemes = [scheme]

            relpath = os.path.relpath(submodule.__file__, start=root_path)
            heading = '%s (%s)' % ("/".join(schemes), relpath)
            print('    %s' % heading)
            print('    %s' % ('~' * len(heading)))
            print('    %s' % submodule.__doc__.split('\n')[0])
            print()

            kwargs = extract_kwargs(submodule.open.__doc__)
            if kwargs:
                print(to_docstring(kwargs, lpad=u'    '))

        print('    Examples')
        print('    --------')
        print()
        print(extract_examples_from_readme_rst())

        print('    This function also supports transparent compression and decompression ')
        print('    using the following codecs:')
        print()
        for extension in compression.get_supported_extensions():
            print('    * %s' % extension)
        print()
        print('    The function depends on the file extension to determine the appropriate codec.')

    #
    # The docstring can be None if -OO was passed to the interpreter.
    #
    if f.__doc__:
        f.__doc__ = f.__doc__.replace(PLACEHOLDER, buf.getvalue())


def tweak_parse_uri_docstring(f):
    from . import transport

    buf = io.StringIO()
    seen = set()
    schemes = []
    examples = []

    for scheme, submodule in sorted(transport._REGISTRY.items()):
        if scheme == transport.NO_SCHEME or submodule in seen:
            continue

        seen.add(submodule)

        try:
            examples.extend(submodule.URI_EXAMPLES)
        except AttributeError:
            pass

        try:
            schemes.extend(submodule.SCHEMES)
        except AttributeError:
            schemes.append(scheme)

    with contextlib.redirect_stdout(buf):
        print('    Supported URI schemes are:')
        print()
        for scheme in schemes:
            print('    * %s' % scheme)
        print()
        print('    Valid URI examples::')
        print()
        for example in examples:
            print('    * %s' % example)

    if f.__doc__:
        f.__doc__ = f.__doc__.replace(PLACEHOLDER, buf.getvalue())