1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
|
# Copyright (C) 2013-2014, Stefan Schwarzer <sschwarzer@sschwarzer.net>
# and ftputil contributors (see `doc/contributors.txt`)
# See the file LICENSE for licensing terms.
"""
file_transfer.py - upload, download and generic file copy
"""
from __future__ import unicode_literals
import io
import os
import ftputil.stat
#TODO Think a bit more about the API before making it public.
# # Only `chunks` should be used by clients of the ftputil library. Any
# # other functionality is supposed to be used via `FTPHost` objects.
# __all__ = ["chunks"]
__all__ = []
# Maximum size of chunk in `FTPHost.copyfileobj` in bytes.
MAX_COPY_CHUNK_SIZE = 64 * 1024
class LocalFile(object):
"""
Represent a file on the local side which is to be transferred or
is already transferred.
"""
def __init__(self, name, mode):
self.name = os.path.abspath(name)
self.mode = mode
def exists(self):
"""
Return `True` if the path representing this file exists.
Otherwise return `False`.
"""
return os.path.exists(self.name)
def mtime(self):
"""Return the timestamp for the last modification in seconds."""
return os.path.getmtime(self.name)
def mtime_precision(self):
"""Return the precision of the last modification time in seconds."""
# Derived classes might want to use `self`.
# pylint: disable=no-self-use
#
# Assume modification timestamps for local file systems are
# at least precise up to a second.
return 1.0
def fobj(self):
"""Return a file object for the name/path in the constructor."""
return io.open(self.name, self.mode)
class RemoteFile(object):
"""
Represent a file on the remote side which is to be transferred or
is already transferred.
"""
def __init__(self, ftp_host, name, mode):
self._host = ftp_host
self._path = ftp_host.path
self.name = self._path.abspath(name)
self.mode = mode
def exists(self):
"""
Return `True` if the path representing this file exists.
Otherwise return `False`.
"""
return self._path.exists(self.name)
def mtime(self):
"""Return the timestamp for the last modification in seconds."""
# Convert to client time zone (see definition of time
# shift in docstring of `FTPHost.set_time_shift`).
return self._path.getmtime(self.name) - self._host.time_shift()
def mtime_precision(self):
"""Return the precision of the last modification time in seconds."""
# I think using `stat` instead of `lstat` makes more sense here.
return self._host.stat(self.name)._st_mtime_precision
def fobj(self):
"""Return a file object for the name/path in the constructor."""
return self._host.open(self.name, self.mode)
def source_is_newer_than_target(source_file, target_file):
"""
Return `True` if the source is newer than the target, else `False`.
Both arguments are `LocalFile` or `RemoteFile` objects.
It's assumed that the actual modification time is
reported_mtime <= actual_mtime <= reported_mtime + mtime_precision
i. e. that the reported mtime is the actual mtime or rounded down
(truncated).
For the purpose of this test the source is newer than the target
if any of the possible actual source modification times is greater
than the reported target modification time. In other words: If in
doubt, the file should be transferred.
This is the only situation where the source is _not_ considered
newer than the target:
|/////////////////////| possible source mtime
|////////| possible target mtime
That is, the latest possible actual source modification time is
before the first possible actual target modification time.
"""
if source_file.mtime_precision() is ftputil.stat.UNKNOWN_PRECISION:
return True
else:
return (source_file.mtime() + source_file.mtime_precision() >=
target_file.mtime())
def chunks(fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE):
"""
Return an iterator which yields the contents of the file object.
For each iteration, at most `max_chunk_size` bytes are read from
`fobj` and yielded as a byte string. If the file object is
exhausted, then don't yield any more data but stop the iteration,
so the client does _not_ get an empty byte string.
Any exceptions resulting from reading the file object are passed
through to the client.
"""
while True:
chunk = fobj.read(max_chunk_size)
if not chunk:
break
yield chunk
def copyfileobj(source_fobj, target_fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE,
callback=None):
"""Copy data from file-like object source to file-like object target."""
# Inspired by `shutil.copyfileobj` (I don't use the `shutil`
# code directly because it might change)
for chunk in chunks(source_fobj, max_chunk_size):
target_fobj.write(chunk)
if callback is not None:
callback(chunk)
def copy_file(source_file, target_file, conditional, callback):
"""
Copy a file from `source_file` to `target_file`.
These are `LocalFile` or `RemoteFile` objects. Which of them
is a local or a remote file, respectively, is determined by
the arguments. If `conditional` is true, the file is only
copied if the target doesn't exist or is older than the
source. If `conditional` is false, the file is copied
unconditionally. Return `True` if the file was copied, else
`False`.
"""
if conditional:
# Evaluate condition: The target file either doesn't exist or is
# older than the source file. If in doubt (due to imprecise
# timestamps), perform the transfer.
transfer_condition = not target_file.exists() or \
source_is_newer_than_target(source_file, target_file)
if not transfer_condition:
# We didn't transfer.
return False
source_fobj = source_file.fobj()
try:
target_fobj = target_file.fobj()
try:
copyfileobj(source_fobj, target_fobj, callback=callback)
finally:
target_fobj.close()
finally:
source_fobj.close()
# Transfer accomplished
return True
|