File: picklable_file_io.py

package info (click to toggle)
mdanalysis 2.10.0-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 116,696 kB
  • sloc: python: 92,135; ansic: 8,156; makefile: 215; sh: 138
file content (617 lines) | stat: -rw-r--r-- 17,629 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*-
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
#
# MDAnalysis --- https://www.mdanalysis.org
# Copyright (c) 2006-2017 The MDAnalysis Development Team and contributors
# (see the file AUTHORS for the full list of names)
#
# Released under the Lesser GNU Public Licence, v2.1 or any higher version
#
# Please cite your use of MDAnalysis in published work:
#
# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler,
# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein.
# MDAnalysis: A Python package for the rapid analysis of molecular dynamics
# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th
# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy.
# doi: 10.25080/majora-629e541a-00e
#
# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein.
# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations.
# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787
#
"""
Picklable read-only I/O classes --- :mod:`MDAnalysis.lib.picklable_file_io`
===========================================================================

Provide with an interface for pickling read-only IO file object.
These classes are used for further pickling :class:`MDAnalysis.core.universe`
in a object composition approach.

.. autoclass:: FileIOPicklable
   :members:

.. autoclass:: BufferIOPicklable
   :members:

.. autoclass:: TextIOPicklable
   :members:

.. autoclass:: BZ2Picklable
   :members:

.. autoclass:: GzipPicklable
   :members:

.. autofunction:: pickle_open

.. autofunction:: bz2_pickle_open

.. autofunction:: gzip_pickle_open


.. versionadded:: 2.0.0
"""
import io
import os

import bz2
import gzip


class FileIOPicklable(io.FileIO):
    """File object (read-only) that can be pickled.

    This class provides a file-like object (as returned by :func:`open`,
    namely :class:`io.FileIO`) that, unlike standard Python file objects,
    can be pickled. Only read mode is supported.

    When the file is pickled, filename and position of the open file handle in
    the file are saved. On unpickling, the file is opened by filename,
    and the file is seeked to the saved position.
    This means that for a successful unpickle, the original file still has to
    be accessible with its filename.

    Note
    ----
    This class only supports reading files in binary mode. If you need to open
    a file in text mode, use the :func:`pickle_open`.

    Parameters
    ----------
    name : str
        either a text or byte string giving the name (and the path
        if the file isn't in the current working directory) of the file to
        be opened.
    mode : str
        only reading ('r') mode works. It exists to be consistent
        with a wider API.

    Example
    -------
    ::

        >>> import pickle
        >>> from MDAnalysis.tests.datafiles import PDB
        >>> file = FileIOPicklable(PDB)
        >>> _ = file.readline()
        >>> file_pickled = pickle.loads(pickle.dumps(file))
        >>> print(file.tell(), file_pickled.tell())
        55 55

    See Also
    ---------
    TextIOPicklable
    BufferIOPicklable


    .. versionadded:: 2.0.0
    """

    def __init__(self, name, mode="r"):
        self._mode = mode
        super().__init__(name, mode)

    def __setstate__(self, state):
        name = state["name_val"]
        self.__init__(name, mode="r")
        try:
            self.seek(state["tell_val"])
        except KeyError:
            pass

    def __reduce_ex__(self, prot):
        if self._mode != "r":
            raise RuntimeError(
                "Can only pickle files that were opened "
                "in read mode, not {}".format(self._mode)
            )
        return (
            self.__class__,
            (self.name, self._mode),
            {"name_val": self.name, "tell_val": self.tell()},
        )


class BufferIOPicklable(io.BufferedReader):
    """A picklable buffer object for read-only FileIO object.

    This class provides a buffered :class:`io.BufferedReader`
    that can be pickled.
    Note that this only works in read mode.

    Parameters
    ----------
    raw : FileIO object

    Example
    -------
    ::

        file = FileIOPicklable('filename')
        buffer_wrapped = BufferIOPicklable(file)

    See Also
    ---------
    FileIOPicklable
    TextIOPicklable


    .. versionadded:: 2.0.0
    """

    def __init__(self, raw):
        super().__init__(raw)
        self.raw_class = raw.__class__

    def __setstate__(self, state):
        raw_class = state["raw_class"]
        name = state["name_val"]
        raw = raw_class(name)
        self.__init__(raw)
        self.seek(state["tell_val"])

    def __reduce_ex__(self, prot):
        # don't ask, for Python 3.12+ see:
        # https://github.com/python/cpython/pull/104370
        return (
            self.raw_class,
            (self.name,),
            {
                "raw_class": self.raw_class,
                "name_val": self.name,
                "tell_val": self.tell(),
            },
        )


class TextIOPicklable(io.TextIOWrapper):
    """Character and line based picklable file-like object.

    This class provides a file-like :class:`io.TextIOWrapper` object that can
    be pickled. Note that this only works in read mode.

    Parameters
    ----------
    raw : FileIO object

    Example
    -------
    ::

        file = FileIOPicklable('filename')
        text_wrapped = TextIOPicklable(file)

    See Also
    ---------
    FileIOPicklable
    BufferIOPicklable


    .. versionadded:: 2.0.0
    .. versionchanged:: 2.8.0
       The raw class instance instead of the class name
       that is wrapped inside will be serialized.
       After deserialization, the current position is no longer reset
       so `universe.trajectory[i]` is not needed to seek to the
       original position.
    """

    def __init__(self, raw):
        super().__init__(raw)
        self.raw_class = raw.__class__

    def __setstate__(self, args):
        raw_class = args["raw_class"]
        name = args["name_val"]
        tell = args["tell_val"]
        # raw_class is used for further expansion this functionality to
        # Gzip files, which also requires a text wrapper.
        raw = raw_class(name)
        self.__init__(raw)
        if tell is not None:
            self.seek(tell)

    def __reduce_ex__(self, prot):
        try:
            curr_loc = self.tell()
        # some readers (e.g. GMS) disable tell() due to using next()
        except OSError:
            curr_loc = None
        try:
            name = self.name
        except AttributeError:
            # This is kind of ugly--BZ2File does not save its name.
            name = self.buffer._fp.name
        return (
            self.__class__.__new__,
            (self.__class__,),
            {
                "raw_class": self.raw_class,
                "name_val": name,
                "tell_val": curr_loc,
            },
        )


class BZ2Picklable(bz2.BZ2File):
    """File object (read-only) for bzip2 (de)compression that can be pickled.

    This class provides a file-like object (as returned by :func:`bz2.open`,
    namely :class:`bz2.BZ2File`) that, unlike standard Python file objects,
    can be pickled. Only read mode is supported.

    When the file is pickled, filename and position of the open file handle in
    the file are saved. On unpickling, the file is opened by filename,
    and the file is seeked to the saved position.
    This means that for a successful unpickle, the original file still has to
    be accessible with its filename.

    Note
    ----
    This class only supports reading files in binary mode. If you need to open
    to open a compressed file in text mode, use :func:`bz2_pickle_open`.

    Parameters
    ----------
    name : str
        either a text or byte string giving the name (and the path
        if the file isn't in the current working directory) of the file to
        be opened.
    mode : str
        can only be 'r', 'rb' to make pickle work.

    Example
    -------
    ::

        >>> import pickle
        >>> from MDAnalysis.tests.datafiles import XYZ_bz2
        >>> file = BZ2Picklable(XYZ_bz2)
        >>> _ = file.readline()
        >>> file_pickled = pickle.loads(pickle.dumps(file))
        >>> print(file.tell(), file_pickled.tell())
        5 5

    See Also
    ---------
    FileIOPicklable
    BufferIOPicklable
    TextIOPicklable
    GzipPicklable


    .. versionadded:: 2.0.0
    """

    def __init__(self, name, mode="rb"):
        self._bz_mode = mode
        super().__init__(name, mode)

    def __getstate__(self):
        if not self._bz_mode.startswith("r"):
            raise RuntimeError(
                "Can only pickle files that were opened "
                "in read mode, not {}".format(self._bz_mode)
            )
        return {"name_val": self._fp.name, "tell_val": self.tell()}

    def __setstate__(self, args):
        name = args["name_val"]
        tell = args["tell_val"]
        self.__init__(name)
        try:
            self.seek(tell)
        except KeyError:
            pass


class GzipPicklable(gzip.GzipFile):
    """Gzip file object (read-only) that can be pickled.

    This class provides a file-like object (as returned by :func:`gzip.open`,
    namely :class:`gzip.GzipFile`) that, unlike standard Python file objects,
    can be pickled. Only read mode is supported.

    When the file is pickled, filename and position of the open file handle in
    the file are saved. On unpickling, the file is opened by filename,
    and the file is seeked to the saved position.
    This means that for a successful unpickle, the original file still has to
    be accessible with its filename.

    Note
    ----
    This class only supports reading files in binary mode. If you need to open
    to open a compressed file in text mode, use the :func:`gzip_pickle_open`.

    Parameters
    ----------
    name : str
        either a text or byte string giving the name (and the path
        if the file isn't in the current working directory) of the file to
        be opened.
    mode : str
        can only be 'r', 'rb' to make pickle work.

    Example
    -------
    ::

        >>> import pickle
        >>> from MDAnalysis.tests.datafiles import MMTF_gz
        >>> file = GzipPicklable(MMTF_gz)
        >>> _ = file.readline()
        >>> file_pickled = pickle.loads(pickle.dumps(file))
        >>> print(file.tell(), file_pickled.tell())
        1218 1218

    See Also
    ---------
    FileIOPicklable
    BufferIOPicklable
    TextIOPicklable
    BZ2Picklable


    .. versionadded:: 2.0.0
    """

    def __init__(self, name, mode="rb"):
        self._gz_mode = mode
        super().__init__(name, mode)

    def __getstate__(self):
        if not self._gz_mode.startswith("r"):
            raise RuntimeError(
                "Can only pickle files that were opened "
                "in read mode, not {}".format(self._gz_mode)
            )
        return {"name_val": self.name, "tell_val": self.tell()}

    def __setstate__(self, args):
        name = args["name_val"]
        tell = args["tell_val"]
        self.__init__(name)
        try:
            self.seek(tell)
        except KeyError:
            pass


def pickle_open(name, mode="rt"):
    """Open file and return a stream with pickle function implemented.

    This function returns a FileIOPicklable object wrapped in a
    BufferIOPicklable class when given the "rb" reading mode,
    or a FileIOPicklable object wrapped in a TextIOPicklable class with the "r"
    or "rt" reading mode. It can be used as a context manager, and replace the
    built-in :func:`open` function in read mode that only returns an
    unpicklable file object.
    In order to serialize a :class:`MDAnalysis.core.Universe`, this function
    can used to open trajectory/topology files. This object composition is more
    flexible and easier than class inheritance to implement pickling
    for new readers.

    Note
    ----
    Can be only used with read mode.

    Parameters
    ----------
    name : str
        either a text or byte string giving the name (and the path
        if the file isn't in the current working directory) of the file to
        be opened.
    mode: {'r', 'rt', 'rb'} (optional)
        'r':  open for reading in text mode;
        'rt': read in text mode (default);
        'rb': read in binary mode;

    Returns
    -------
    stream-like object: BufferIOPicklable or TextIOPicklable
        when mode is 'r' or 'rt', returns TextIOPicklable;
        when mode is 'rb', returns BufferIOPicklable

    Raises
    ------
    ValueError
        if `mode` is not one of the allowed read modes

    Examples
    -------
    open as context manager::

        with pickle_open('filename') as f:
            line = f.readline()

    open as function::

        f = pickle_open('filename')
        line = f.readline()
        f.close()

    See Also
    --------
    :func:`MDAnalysis.lib.util.anyopen`
    :func:`io.open`


    .. versionadded:: 2.0.0
    """
    if mode not in {"r", "rt", "rb"}:
        raise ValueError(
            "Only read mode ('r', 'rt', 'rb') " "files can be pickled."
        )
    name = os.fspath(name)
    raw = FileIOPicklable(name)
    if mode == "rb":
        return BufferIOPicklable(raw)
    elif mode in {"r", "rt"}:
        return TextIOPicklable(raw)


def bz2_pickle_open(name, mode="rb"):
    """Open a bzip2-compressed file in binary or text mode
    with pickle function implemented.

    This function returns a BZ2Picklable object when given the "rb" or "r"
    reading mode, or a BZ2Picklable object wrapped in a TextIOPicklable class
    with the "rt" reading mode.
    It can be used as a context manager, and replace the built-in
    :func:`bz2.open` function in read mode that only returns an
    unpicklable file object.

    Note
    ----
    Can be only used with read mode.

    Parameters
    ----------
    name : str
        either a text or byte string giving the name (and the path
        if the file isn't in the current working directory) of the file to
        be opened.
    mode: {'r', 'rt', 'rb'} (optional)
        'r':  open for reading in binary mode;
        'rt': read in text mode;
        'rb': read in binary mode; (default)

    Returns
    -------
    stream-like object: BZ2Picklable or TextIOPicklable
        when mode is 'rt', returns TextIOPicklable;
        when mode is 'r' or 'rb', returns BZ2Picklable

    Raises
    ------
    ValueError
        if `mode` is not one of the allowed read modes

    Examples
    -------
    open as context manager::

        with bz2_pickle_open('filename') as f:
            line = f.readline()

    open as function::

        f = bz2_pickle_open('filename')
        line = f.readline()
        f.close()

    See Also
    --------
    :func:`io.open`
    :func:`bz2.open`
    :func:`MDAnalysis.lib.util.anyopen`
    :func:`MDAnalysis.lib.picklable_file_io.pickle_open`
    :func:`MDAnalysis.lib.picklable_file_io.gzip_pickle_open`


    .. versionadded:: 2.0.0
    """
    if mode not in {"r", "rt", "rb"}:
        raise ValueError(
            "Only read mode ('r', 'rt', 'rb') " "files can be pickled."
        )
    bz_mode = mode.replace("t", "")
    binary_file = BZ2Picklable(name, bz_mode)
    if "t" in mode:
        return TextIOPicklable(binary_file)
    else:
        return binary_file


def gzip_pickle_open(name, mode="rb"):
    """Open a gzip-compressed file in binary or text mode
    with pickle function implemented.

    This function returns a GzipPicklable object when given the "rb" or "r"
    reading mode, or a GzipPicklable object wrapped in a TextIOPicklable class
    with the "rt" reading mode.
    It can be used as a context manager, and replace the built-in
    :func:`gzip.open` function in read mode that only returns an
    unpicklable file object.

    Note
    ----
    Can be only used with read mode.

    Parameters
    ----------
    name : str
        either a text or byte string giving the name (and the path
        if the file isn't in the current working directory) of the file to
        be opened.
    mode: {'r', 'rt', 'rb'} (optional)
        'r':  open for reading in binary mode;
        'rt': read in text mode;
        'rb': read in binary mode; (default)

    Returns
    -------
    stream-like object: GzipPicklable or TextIOPicklable
        when mode is 'rt', returns TextIOPicklable;
        when mode is 'r' or 'rb', returns GzipPicklable

    Raises
    ------
    ValueError
        if `mode` is not one of the allowed read modes

    Examples
    -------
    open as context manager::

        with gzip_pickle_open('filename') as f:
            line = f.readline()

    open as function::

        f = gzip_pickle_open('filename')
        line = f.readline()
        f.close()

    See Also
    --------
    :func:`io.open`
    :func:`gzip.open`
    :func:`MDAnalysis.lib.util.anyopen`
    :func:`MDAnalysis.lib.picklable_file_io.pickle_open`
    :func:`MDAnalysis.lib.picklable_file_io.bz2_pickle_open`


    .. versionadded:: 2.0.0
    """
    if mode not in {"r", "rt", "rb"}:
        raise ValueError(
            "Only read mode ('r', 'rt', 'rb') " "files can be pickled."
        )
    gz_mode = mode.replace("t", "")
    binary_file = GzipPicklable(name, gz_mode)
    if "t" in mode:
        return TextIOPicklable(binary_file)
    else:
        return binary_file