File: manual.py

package info (click to toggle)
python-stem 1.7.1-1.1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 5,768 kB
  • sloc: python: 29,441; java: 312; makefile: 125; sh: 17
file content (810 lines) | stat: -rw-r--r-- 28,133 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
# Copyright 2015-2018, Damian Johnson and The Tor Project
# See LICENSE for licensing information

"""
Information available about Tor from `its manual
<https://www.torproject.org/docs/tor-manual.html.en>`_. This provides three
methods of getting this information...

* :func:`~stem.manual.Manual.from_cache` provides manual content bundled with
  Stem. This is the fastest and most reliable method but only as up-to-date as
  Stem's release.

* :func:`~stem.manual.Manual.from_man` reads Tor's local man page for
  information about it.

* :func:`~stem.manual.Manual.from_remote` fetches the latest manual information
  remotely. This is the slowest and least reliable method but provides the most
  recent information about Tor.

Manual information includes arguments, signals, and probably most usefully the
torrc configuration options. For example, say we want a little script that told
us what our torrc options do...

.. literalinclude::  /_static/example/manual_config_options.py
   :language: python

|

.. image:: /_static/manual_output.png

|

**Module Overview:**

::

  query - performs a query on our cached sqlite manual information
  is_important - Indicates if a configuration option is of particularly common importance.
  download_man_page - Downloads tor's latest man page.

  Manual - Information about Tor available from its manual.
   | |- from_cache - Provides manual information cached with Stem.
   | |- from_man - Retrieves manual information from its man page.
   | +- from_remote - Retrieves manual information remotely from tor's latest manual.
   |
   +- save - writes the manual contents to a given location

.. versionadded:: 1.5.0
"""

import os
import shutil
import sys
import tempfile

import stem.prereq
import stem.util
import stem.util.conf
import stem.util.enum
import stem.util.log
import stem.util.system

try:
  # added in python 2.7
  from collections import OrderedDict
except ImportError:
  from stem.util.ordereddict import OrderedDict

if stem.prereq._is_lru_cache_available():
  from functools import lru_cache
else:
  from stem.util.lru_cache import lru_cache

try:
  # account for urllib's change between python 2.x and 3.x
  import urllib.request as urllib
except ImportError:
  import urllib2 as urllib

Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN')
GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt'
CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_manual.sqlite')
DATABASE = None  # cache database connections
HAS_ENCODING_ARG = not stem.util.system.is_mac() and not stem.util.system.is_bsd() and not stem.util.system.is_slackware()

SCHEMA_VERSION = 1  # version of our scheme, bump this if you change the following
SCHEMA = (
  'CREATE TABLE schema(version INTEGER)',
  'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION,

  'CREATE TABLE metadata(name TEXT, synopsis TEXT, description TEXT, man_commit TEXT, stem_commit TEXT)',
  'CREATE TABLE commandline(name TEXT PRIMARY KEY, description TEXT)',
  'CREATE TABLE signals(name TEXT PRIMARY KEY, description TEXT)',
  'CREATE TABLE files(name TEXT PRIMARY KEY, description TEXT)',
  'CREATE TABLE torrc(key TEXT PRIMARY KEY, name TEXT, category TEXT, usage TEXT, summary TEXT, description TEXT, position INTEGER)',
)

CATEGORY_SECTIONS = OrderedDict((
  ('GENERAL OPTIONS', Category.GENERAL),
  ('CLIENT OPTIONS', Category.CLIENT),
  ('SERVER OPTIONS', Category.RELAY),
  ('DIRECTORY SERVER OPTIONS', Category.DIRECTORY),
  ('DIRECTORY AUTHORITY SERVER OPTIONS', Category.AUTHORITY),
  ('HIDDEN SERVICE OPTIONS', Category.HIDDEN_SERVICE),
  ('DENIAL OF SERVICE MITIGATION OPTIONS', Category.DENIAL_OF_SERVICE),
  ('TESTING NETWORK OPTIONS', Category.TESTING),
))


class SchemaMismatch(IOError):
  """
  Database schema doesn't match what Stem supports.

  .. versionadded:: 1.6.0

  :var int database_schema: schema of the database
  :var tuple supported_schemas: schemas library supports
  """

  def __init__(self, message, database_schema, library_schema):
    super(SchemaMismatch, self).__init__(message)
    self.database_schema = database_schema
    self.library_schema = library_schema


def query(query, *param):
  """
  Performs the given query on our sqlite manual cache. This database should
  be treated as being read-only. File permissions generally enforce this, and
  in the future will be enforced by this function as well.

  ::

    >>> import stem.manual
    >>> print(stem.manual.query('SELECT description FROM torrc WHERE key=?', 'CONTROLSOCKET').fetchone()[0])
    Like ControlPort, but listens on a Unix domain socket, rather than a TCP socket.  0 disables ControlSocket. (Unix and Unix-like systems only.) (Default: 0)

  .. versionadded:: 1.6.0

  :param str query: query to run on the cache
  :param list param: query parameters

  :returns: :class:`sqlite3.Cursor` with the query results

  :raises:
    * **ImportError** if the sqlite3 module is unavailable
    * **sqlite3.OperationalError** if query fails
  """

  if not stem.prereq.is_sqlite_available():
    raise ImportError('Querying requires the sqlite3 module')

  import sqlite3

  # The only reason to explicitly close the sqlite connection is to ensure
  # transactions are committed. Since we're only using read-only access this
  # doesn't matter, and can allow interpreter shutdown to do the needful.
  #
  # TODO: When we only support python 3.4+ we can use sqlite's uri argument
  # to enforce a read-only connection...
  #
  #   https://docs.python.org/3/library/sqlite3.html#sqlite3.connect

  global DATABASE

  if DATABASE is None:
    DATABASE = sqlite3.connect(CACHE_PATH)

  return DATABASE.execute(query, param)


class ConfigOption(object):
  """
  Tor configuration attribute found in its torrc.

  :var str name: name of the configuration option
  :var stem.manual.Category category: category the config option was listed
    under, this is Category.UNKNOWN if we didn't recognize the category
  :var str usage: arguments accepted by the option
  :var str summary: brief description of what the option does
  :var str description: longer manual description with details
  """

  def __init__(self, name, category = Category.UNKNOWN, usage = '', summary = '', description = ''):
    self.name = name
    self.category = category
    self.usage = usage
    self.summary = summary
    self.description = description

  def __hash__(self):
    return stem.util._hash_attr(self, 'name', 'category', 'usage', 'summary', 'description', cache = True)

  def __eq__(self, other):
    return hash(self) == hash(other) if isinstance(other, ConfigOption) else False

  def __ne__(self, other):
    return not self == other


@lru_cache()
def _config(lowercase = True):
  """
  Provides a dictionary for our settings.cfg. This has a couple categories...

    * manual.important (list) - configuration options considered to be important
    * manual.summary.* (str) - summary descriptions of config options

  :param bool lowercase: uses lowercase keys if **True** to allow for case
    insensitive lookups
  """

  config = stem.util.conf.Config()
  config_path = os.path.join(os.path.dirname(__file__), 'settings.cfg')

  try:
    config.load(config_path)
    config_dict = dict([(key.lower() if lowercase else key, config.get_value(key)) for key in config.keys() if key.startswith('manual.summary.')])
    config_dict['manual.important'] = [name.lower() if lowercase else name for name in config.get_value('manual.important', [], multiple = True)]
    return config_dict
  except Exception as exc:
    stem.util.log.warn("BUG: stem failed to load its internal manual information from '%s': %s" % (config_path, exc))
    return {}


def _manual_differences(previous_manual, new_manual):
  """
  Provides a description of how two manuals differ.
  """

  lines = []

  for attr in ('name', 'synopsis', 'description', 'commandline_options', 'signals', 'files', 'config_options'):
    previous_attr = getattr(previous_manual, attr)
    new_attr = getattr(new_manual, attr)

    if previous_attr != new_attr:
      lines.append("* Manual's %s attribute changed\n" % attr)

      if attr in ('name', 'synopsis', 'description'):
        lines.append('  Previously...\n\n%s\n' % previous_attr)
        lines.append('  Updating to...\n\n%s' % new_attr)
      elif attr == 'config_options':
        for config_name, config_attr in new_attr.items():
          previous = previous_attr.get(config_name)

          if previous is None:
            lines.append('  adding new config option => %s' % config_name)
          elif config_attr != previous:
            for attr in ('name', 'category', 'usage', 'summary', 'description'):
              if getattr(config_attr, attr) != getattr(previous, attr):
                lines.append('  modified %s (%s) => %s' % (config_name, attr, getattr(config_attr, attr)))

        for config_name in set(previous_attr.keys()).difference(new_attr.keys()):
          lines.append('  removing config option => %s' % config_name)
      else:
        added_items = set(new_attr.items()).difference(previous_attr.items())
        removed_items = set(previous_attr.items()).difference(new_attr.items())

        for added_item in added_items:
          lines.append('  adding %s => %s' % added_item)

        for removed_item in removed_items:
          lines.append('  removing %s => %s' % removed_item)

      lines.append('\n')

  return '\n'.join(lines)


def is_important(option):
  """
  Indicates if a configuration option of particularly common importance or not.

  :param str option: tor configuration option to check

  :returns: **bool** that's **True** if this is an important option and
    **False** otherwise
  """

  return option.lower() in _config()['manual.important']


def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, timeout = 20):
  """
  Downloads tor's latest man page from `gitweb.torproject.org
  <https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. This method is
  both slow and unreliable - please see the warnings on
  :func:`~stem.manual.Manual.from_remote`.

  :param str path: path to save tor's man page to
  :param file file_handle: file handler to save tor's man page to
  :param str url: url to download tor's asciidoc manual from
  :param int timeout: seconds to wait before timing out the request

  :raises: **IOError** if unable to retrieve the manual
  """

  if not path and not file_handle:
    raise ValueError("Either the path or file_handle we're saving to must be provided")
  elif not stem.util.system.is_available('a2x'):
    raise IOError('We require a2x from asciidoc to provide a man page')

  dirpath = tempfile.mkdtemp()
  asciidoc_path = os.path.join(dirpath, 'tor.1.txt')
  manual_path = os.path.join(dirpath, 'tor.1')

  try:
    try:
      with open(asciidoc_path, 'wb') as asciidoc_file:
        request = urllib.urlopen(url, timeout = timeout)
        shutil.copyfileobj(request, asciidoc_file)
    except:
      exc = sys.exc_info()[1]
      raise IOError("Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc))

    try:
      stem.util.system.call('a2x -f manpage %s' % asciidoc_path)

      if not os.path.exists(manual_path):
        raise OSError('no man page was generated')
    except stem.util.system.CallError as exc:
      raise IOError("Unable to run '%s': %s" % (exc.command, exc.stderr))

    if path:
      try:
        path_dir = os.path.dirname(path)

        if not os.path.exists(path_dir):
          os.makedirs(path_dir)

        shutil.copyfile(manual_path, path)
      except OSError as exc:
        raise IOError(exc)

    if file_handle:
      with open(manual_path, 'rb') as manual_file:
        shutil.copyfileobj(manual_file, file_handle)
        file_handle.flush()
  finally:
    shutil.rmtree(dirpath)


class Manual(object):
  """
  Parsed tor man page. Tor makes no guarantees about its man page format so
  this may not always be compatible. If not you can use the cached manual
  information stored with Stem.

  This does not include every bit of information from the tor manual. For
  instance, I've excluded the 'THE CONFIGURATION FILE FORMAT' section. If
  there's a part you'd find useful then `file an issue
  <https://trac.torproject.org/projects/tor/wiki/doc/stem/bugs>`_ and we can
  add it.

  :var str name: brief description of the tor command
  :var str synopsis: brief tor command usage
  :var str description: general description of what tor does

  :var collections.OrderedDict commandline_options: mapping of commandline arguments to their descripton
  :var collections.OrderedDict signals: mapping of signals tor accepts to their description
  :var collections.OrderedDict files: mapping of file paths to their description

  :var collections.OrderedDict config_options: :class:`~stem.manual.ConfigOption` tuples for tor configuration options

  :var str man_commit: latest tor commit editing the man page when this
    information was cached
  :var str stem_commit: stem commit to cache this manual information
  """

  def __init__(self, name, synopsis, description, commandline_options, signals, files, config_options):
    self.name = name
    self.synopsis = synopsis
    self.description = description
    self.commandline_options = OrderedDict(commandline_options)
    self.signals = OrderedDict(signals)
    self.files = OrderedDict(files)
    self.config_options = OrderedDict(config_options)
    self.man_commit = None
    self.stem_commit = None
    self.schema = None

  @staticmethod
  def from_cache(path = None):
    """
    Provides manual information cached with Stem. Unlike
    :func:`~stem.manual.Manual.from_man` and
    :func:`~stem.manual.Manual.from_remote` this doesn't have any system
    requirements, and is faster too. Only drawback is that this manual
    content is only as up to date as the Stem release we're using.

    .. versionchanged:: 1.6.0
       Added support for sqlite cache. Support for
       :class:`~stem.util.conf.Config` caches will be dropped in Stem 2.x.

    :param str path: cached manual content to read, if not provided this uses
      the bundled manual information

    :returns: :class:`~stem.manual.Manual` with our bundled manual information

    :raises:
      * **ImportError** if cache is sqlite and the sqlite3 module is
        unavailable
      * **IOError** if a **path** was provided and we were unable to read
        it or the schema is out of date
    """

    # TODO: drop _from_config_cache() with stem 2.x

    if path is None:
      path = CACHE_PATH

    if path is not None and path.endswith('.sqlite'):
      return Manual._from_sqlite_cache(path)
    else:
      return Manual._from_config_cache(path)

  @staticmethod
  def _from_sqlite_cache(path):
    if not stem.prereq.is_sqlite_available():
      raise ImportError('Reading a sqlite cache requires the sqlite3 module')

    import sqlite3

    if not os.path.exists(path):
      raise IOError("%s doesn't exist" % path)

    with sqlite3.connect(path) as conn:
      try:
        schema = conn.execute('SELECT version FROM schema').fetchone()[0]

        if schema != SCHEMA_VERSION:
          raise SchemaMismatch("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema), schema, (SCHEMA_VERSION,))

        name, synopsis, description, man_commit, stem_commit = conn.execute('SELECT name, synopsis, description, man_commit, stem_commit FROM metadata').fetchone()
      except sqlite3.OperationalError as exc:
        raise IOError('Failed to read database metadata from %s: %s' % (path, exc))

      commandline = dict(conn.execute('SELECT name, description FROM commandline').fetchall())
      signals = dict(conn.execute('SELECT name, description FROM signals').fetchall())
      files = dict(conn.execute('SELECT name, description FROM files').fetchall())

      config_options = OrderedDict()

      for entry in conn.execute('SELECT name, category, usage, summary, description FROM torrc ORDER BY position').fetchall():
        option, category, usage, summary, option_description = entry
        config_options[option] = ConfigOption(option, category, usage, summary, option_description)

      manual = Manual(name, synopsis, description, commandline, signals, files, config_options)
      manual.man_commit = man_commit
      manual.stem_commit = stem_commit
      manual.schema = schema

      return manual

  @staticmethod
  def _from_config_cache(path):
    conf = stem.util.conf.Config()
    conf.load(path, commenting = False)

    config_options = OrderedDict()

    for key in conf.keys():
      if key.startswith('config_options.'):
        key = key.split('.')[1]

        if key not in config_options:
          config_options[key] = ConfigOption(
            conf.get('config_options.%s.name' % key, ''),
            conf.get('config_options.%s.category' % key, ''),
            conf.get('config_options.%s.usage' % key, ''),
            conf.get('config_options.%s.summary' % key, ''),
            conf.get('config_options.%s.description' % key, '')
          )

    manual = Manual(
      conf.get('name', ''),
      conf.get('synopsis', ''),
      conf.get('description', ''),
      conf.get('commandline_options', OrderedDict()),
      conf.get('signals', OrderedDict()),
      conf.get('files', OrderedDict()),
      config_options,
    )

    manual.man_commit = conf.get('man_commit', None)
    manual.stem_commit = conf.get('stem_commit', None)

    return manual

  @staticmethod
  def from_man(man_path = 'tor'):
    """
    Reads and parses a given man page.

    On OSX the man command doesn't have an '--encoding' argument so its results
    may not quite match other platforms. For instance, it normalizes long
    dashes into '--'.

    :param str man_path: path argument for 'man', for example you might want
      '/path/to/tor/doc/tor.1' to read from tor's git repository

    :returns: :class:`~stem.manual.Manual` for the system's man page

    :raises: **IOError** if unable to retrieve the manual
    """

    man_cmd = 'man %s -P cat %s' % ('--encoding=ascii' if HAS_ENCODING_ARG else '', man_path)

    try:
      man_output = stem.util.system.call(man_cmd, env = {'MANWIDTH': '10000000'})
    except OSError as exc:
      raise IOError("Unable to run '%s': %s" % (man_cmd, exc))

    categories, config_options = _get_categories(man_output), OrderedDict()

    for category_header, category_enum in CATEGORY_SECTIONS.items():
      _add_config_options(config_options, category_enum, categories.get(category_header, []))

    for category in categories:
      if category.endswith(' OPTIONS') and category not in CATEGORY_SECTIONS and category not in ('COMMAND-LINE OPTIONS', 'NON-PERSISTENT OPTIONS'):
        _add_config_options(config_options, Category.UNKNOWN, categories.get(category, []))

    return Manual(
      _join_lines(categories.get('NAME', [])),
      _join_lines(categories.get('SYNOPSIS', [])),
      _join_lines(categories.get('DESCRIPTION', [])),
      _get_indented_descriptions(categories.get('COMMAND-LINE OPTIONS', [])),
      _get_indented_descriptions(categories.get('SIGNALS', [])),
      _get_indented_descriptions(categories.get('FILES', [])),
      config_options,
    )

  @staticmethod
  def from_remote(timeout = 60):
    """
    Reads and parses the latest tor man page `from gitweb.torproject.org
    <https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. Note that
    while convenient, this reliance on GitWeb means you should alway call with
    a fallback, such as...

    ::

      try:
        manual = stem.manual.from_remote()
      except IOError:
        manual = stem.manual.from_cache()

    In addition to our GitWeb dependency this requires 'a2x' which is part of
    `asciidoc <http://asciidoc.org/INSTALL.html>`_ and... isn't quick.
    Personally this takes ~7.41s, breaking down for me as follows...

      * 1.67s to download tor.1.txt
      * 5.57s to convert the asciidoc to a man page
      * 0.17s for stem to read and parse the manual

    :param int timeout: seconds to wait before timing out the request

    :returns: latest :class:`~stem.manual.Manual` available for tor

    :raises: **IOError** if unable to retrieve the manual
    """

    with tempfile.NamedTemporaryFile() as tmp:
      download_man_page(file_handle = tmp, timeout = timeout)
      return Manual.from_man(tmp.name)

  def save(self, path):
    """
    Persists the manual content to a given location.

    .. versionchanged:: 1.6.0
       Added support for sqlite cache. Support for
       :class:`~stem.util.conf.Config` caches will be dropped in Stem 2.x.

    :param str path: path to save our manual content to

    :raises:
      * **ImportError** if saving as sqlite and the sqlite3 module is
        unavailable
      * **IOError** if unsuccessful
    """

    # TODO: drop _save_as_config() with stem 2.x

    if path.endswith('.sqlite'):
      return self._save_as_sqlite(path)
    else:
      return self._save_as_config(path)

  def _save_as_sqlite(self, path):
    if not stem.prereq.is_sqlite_available():
      raise ImportError('Saving a sqlite cache requires the sqlite3 module')

    import sqlite3
    tmp_path = path + '.new'

    if os.path.exists(tmp_path):
      os.remove(tmp_path)

    with sqlite3.connect(tmp_path) as conn:
      for cmd in SCHEMA:
        conn.execute(cmd)

      conn.execute('INSERT INTO metadata(name, synopsis, description, man_commit, stem_commit) VALUES (?,?,?,?,?)', (self.name, self.synopsis, self.description, self.man_commit, self.stem_commit))

      for k, v in self.commandline_options.items():
        conn.execute('INSERT INTO commandline(name, description) VALUES (?,?)', (k, v))

      for k, v in self.signals.items():
        conn.execute('INSERT INTO signals(name, description) VALUES (?,?)', (k, v))

      for k, v in self.files.items():
        conn.execute('INSERT INTO files(name, description) VALUES (?,?)', (k, v))

      for i, v in enumerate(self.config_options.values()):
        conn.execute('INSERT INTO torrc(key, name, category, usage, summary, description, position) VALUES (?,?,?,?,?,?,?)', (v.name.upper(), v.name, v.category, v.usage, v.summary, v.description, i))

    if os.path.exists(path):
      os.remove(path)

    os.rename(tmp_path, path)

  def _save_as_config(self, path):
    conf = stem.util.conf.Config()
    conf.set('name', self.name)
    conf.set('synopsis', self.synopsis)
    conf.set('description', self.description)

    if self.man_commit:
      conf.set('man_commit', self.man_commit)

    if self.stem_commit:
      conf.set('stem_commit', self.stem_commit)

    for k, v in self.commandline_options.items():
      conf.set('commandline_options', '%s => %s' % (k, v), overwrite = False)

    for k, v in self.signals.items():
      conf.set('signals', '%s => %s' % (k, v), overwrite = False)

    for k, v in self.files.items():
      conf.set('files', '%s => %s' % (k, v), overwrite = False)

    for k, v in self.config_options.items():
      conf.set('config_options.%s.category' % k, v.category)
      conf.set('config_options.%s.name' % k, v.name)
      conf.set('config_options.%s.usage' % k, v.usage)
      conf.set('config_options.%s.summary' % k, v.summary)
      conf.set('config_options.%s.description' % k, v.description)

    conf.save(path)

  def __hash__(self):
    return stem.util._hash_attr(self, 'name', 'synopsis', 'description', 'commandline_options', 'signals', 'files', 'config_options', cache = True)

  def __eq__(self, other):
    return hash(self) == hash(other) if isinstance(other, Manual) else False

  def __ne__(self, other):
    return not self == other


def _get_categories(content):
  """
  The man page is headers followed by an indented section. First pass gets
  the mapping of category titles to their lines.
  """

  # skip header and footer lines

  if content and 'TOR(1)' in content[0]:
    content = content[1:]

  if content and content[-1].startswith('Tor'):
    content = content[:-1]

  categories = OrderedDict()
  category, lines = None, []

  for line in content:
    # replace non-ascii characters
    #
    #   \u2019 - smart single quote
    #   \u2014 - extra long dash
    #   \xb7 - centered dot

    char_for = chr if stem.prereq.is_python_3() else unichr
    line = line.replace(char_for(0x2019), "'").replace(char_for(0x2014), '-').replace(char_for(0xb7), '*')

    if line and not line.startswith(' '):
      if category:
        if lines and lines[-1] == '':
          lines = lines[:-1]  # sections end with an extra empty line

        categories[category] = lines

      category, lines = line.strip(), []
    else:
      if line.startswith('       '):
        line = line[7:]  # contents of a section have a seven space indentation

      lines.append(line)

  if category:
    categories[category] = lines

  return categories


def _get_indented_descriptions(lines):
  """
  Parses the commandline argument and signal sections. These are options
  followed by an indented description. For example...

  ::

    -f FILE
        Specify a new configuration file to contain further Tor configuration
        options OR pass - to make Tor read its configuration from standard
        input. (Default: /usr/local/etc/tor/torrc, or $HOME/.torrc if that file
        is not found)

  There can be additional paragraphs not related to any particular argument but
  ignoring those.
  """

  options, last_arg = OrderedDict(), None

  for line in lines:
    if line and not line.startswith(' '):
      options[line], last_arg = [], line
    elif last_arg and line.startswith('    '):
      options[last_arg].append(line[4:])

  return dict([(arg, ' '.join(desc_lines)) for arg, desc_lines in options.items() if desc_lines])


def _add_config_options(config_options, category, lines):
  """
  Parses a section of tor configuration options. These have usage information,
  followed by an indented description. For instance...

  ::

    ConnLimit NUM
        The minimum number of file descriptors that must be available to the
        Tor process before it will start. Tor will ask the OS for as many file
        descriptors as the OS will allow (you can find this by "ulimit -H -n").
        If this number is less than ConnLimit, then Tor will refuse to start.


        You probably don't need to adjust this. It has no effect on Windows
        since that platform lacks getrlimit(). (Default: 1000)
  """

  last_option, usage, description = None, None, []

  # Drop the section description. Each ends with a paragraph saying 'The
  # following options...'.

  desc_paragraph_index = None

  for i, line in enumerate(lines):
    if 'The following options' in line:
      desc_paragraph_index = i
      break

  if desc_paragraph_index is not None:
    lines = lines[desc_paragraph_index:]  # trim to the description paragrah
    lines = lines[lines.index(''):]  # drop the paragraph

  for line in lines:
    if line and not line.startswith(' '):
      if last_option:
        summary = _config().get('manual.summary.%s' % last_option.lower(), '')
        config_options[last_option] = ConfigOption(last_option, category, usage, summary, _join_lines(description).strip())

      if ' ' in line:
        last_option, usage = line.split(' ', 1)
      else:
        last_option, usage = line, ''

      description = []
    else:
      if line.startswith('    '):
        line = line[4:]

      description.append(line)

  if last_option:
    summary = _config().get('manual.summary.%s' % last_option.lower(), '')
    config_options[last_option] = ConfigOption(last_option, category, usage, summary, _join_lines(description).strip())


def _join_lines(lines):
  """
  Simple join, except we want empty lines to still provide a newline.
  """

  result = []

  for line in lines:
    if not line:
      if result and result[-1] != '\n':
        result.append('\n')
    else:
      result.append(line + '\n')

  return ''.join(result).strip()