File: file_transfer.py

package info (click to toggle)
python-ftputil 3.4-3
  • links: PTS, VCS
  • area: main
  • in suites: bullseye
  • size: 848 kB
  • sloc: python: 3,308; makefile: 3
file content (192 lines) | stat: -rw-r--r-- 6,452 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
# Copyright (C) 2013-2014, Stefan Schwarzer <sschwarzer@sschwarzer.net>
# and ftputil contributors (see `doc/contributors.txt`)
# See the file LICENSE for licensing terms.

"""
file_transfer.py - upload, download and generic file copy
"""

from __future__ import unicode_literals

import io
import os

import ftputil.stat


#TODO Think a bit more about the API before making it public.
# # Only `chunks` should be used by clients of the ftputil library. Any
# #  other functionality is supposed to be used via `FTPHost` objects.
# __all__ = ["chunks"]
__all__ = []

# Maximum size of chunk in `FTPHost.copyfileobj` in bytes.
MAX_COPY_CHUNK_SIZE = 64 * 1024


class LocalFile(object):
    """
    Represent a file on the local side which is to be transferred or
    is already transferred.
    """

    def __init__(self, name, mode):
        self.name = os.path.abspath(name)
        self.mode = mode

    def exists(self):
        """
        Return `True` if the path representing this file exists.
        Otherwise return `False`.
        """
        return os.path.exists(self.name)

    def mtime(self):
        """Return the timestamp for the last modification in seconds."""
        return os.path.getmtime(self.name)

    def mtime_precision(self):
        """Return the precision of the last modification time in seconds."""
        # Derived classes might want to use `self`.
        # pylint: disable=no-self-use
        #
        # Assume modification timestamps for local file systems are
        # at least precise up to a second.
        return 1.0

    def fobj(self):
        """Return a file object for the name/path in the constructor."""
        return io.open(self.name, self.mode)


class RemoteFile(object):
    """
    Represent a file on the remote side which is to be transferred or
    is already transferred.
    """

    def __init__(self, ftp_host, name, mode):
        self._host = ftp_host
        self._path = ftp_host.path
        self.name = self._path.abspath(name)
        self.mode = mode

    def exists(self):
        """
        Return `True` if the path representing this file exists.
        Otherwise return `False`.
        """
        return self._path.exists(self.name)

    def mtime(self):
        """Return the timestamp for the last modification in seconds."""
        # Convert to client time zone (see definition of time
        # shift in docstring of `FTPHost.set_time_shift`).
        return self._path.getmtime(self.name) - self._host.time_shift()

    def mtime_precision(self):
        """Return the precision of the last modification time in seconds."""
        # I think using `stat` instead of `lstat` makes more sense here.
        return self._host.stat(self.name)._st_mtime_precision

    def fobj(self):
        """Return a file object for the name/path in the constructor."""
        return self._host.open(self.name, self.mode)


def source_is_newer_than_target(source_file, target_file):
    """
    Return `True` if the source is newer than the target, else `False`.

    Both arguments are `LocalFile` or `RemoteFile` objects.

    It's assumed that the actual modification time is

      reported_mtime <= actual_mtime <= reported_mtime + mtime_precision

    i. e. that the reported mtime is the actual mtime or rounded down
    (truncated).

    For the purpose of this test the source is newer than the target
    if any of the possible actual source modification times is greater
    than the reported target modification time. In other words: If in
    doubt, the file should be transferred.

    This is the only situation where the source is _not_ considered
    newer than the target:

    |/////////////////////|              possible source mtime
                            |////////|   possible target mtime

    That is, the latest possible actual source modification time is
    before the first possible actual target modification time.
    """
    if source_file.mtime_precision() is ftputil.stat.UNKNOWN_PRECISION:
        return True
    else:
        return (source_file.mtime() + source_file.mtime_precision() >=
                target_file.mtime())


def chunks(fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE):
    """
    Return an iterator which yields the contents of the file object.

    For each iteration, at most `max_chunk_size` bytes are read from
    `fobj` and yielded as a byte string. If the file object is
    exhausted, then don't yield any more data but stop the iteration,
    so the client does _not_ get an empty byte string.

    Any exceptions resulting from reading the file object are passed
    through to the client.
    """
    while True:
        chunk = fobj.read(max_chunk_size)
        if not chunk:
            break
        yield chunk


def copyfileobj(source_fobj, target_fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE,
                callback=None):
    """Copy data from file-like object source to file-like object target."""
    # Inspired by `shutil.copyfileobj` (I don't use the `shutil`
    # code directly because it might change)
    for chunk in chunks(source_fobj, max_chunk_size):
        target_fobj.write(chunk)
        if callback is not None:
            callback(chunk)


def copy_file(source_file, target_file, conditional, callback):
    """
    Copy a file from `source_file` to `target_file`.

    These are `LocalFile` or `RemoteFile` objects. Which of them
    is a local or a remote file, respectively, is determined by
    the arguments. If `conditional` is true, the file is only
    copied if the target doesn't exist or is older than the
    source. If `conditional` is false, the file is copied
    unconditionally. Return `True` if the file was copied, else
    `False`.
    """
    if conditional:
        # Evaluate condition: The target file either doesn't exist or is
        # older than the source file. If in doubt (due to imprecise
        # timestamps), perform the transfer.
        transfer_condition = not target_file.exists() or \
          source_is_newer_than_target(source_file, target_file)
        if not transfer_condition:
            # We didn't transfer.
            return False
    source_fobj = source_file.fobj()
    try:
        target_fobj = target_file.fobj()
        try:
            copyfileobj(source_fobj, target_fobj, callback=callback)
        finally:
            target_fobj.close()
    finally:
        source_fobj.close()
    # Transfer accomplished
    return True