# Part of the A-A-P recipe executive: copy and move files (remotely)

# Copyright (C) 2002-2003 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here: http://www.a-a-p.org/COPYING

#
# These are functions to copy and move files.  Not only on the local system,
# also to remote systems, using http://, ftp://, etc.
#

import os
import os.path
import string
import re

from Dictlist import str2dictlist
from Error import *
from Process import recipe_error, option_error
from Util import *
from Work import getrpstack
from Message import *


def copy_move(line_nr, recdict, arg, copy):
    """Implementation of ":copy -x from to" and ":move -x from to".
       When "copy" is non-zero copying is done, otherwise moving.
       "arg" is the whole argument.
       "line_nr" is used for error messages."""

    # Skip when not actually building.
    cmdname = (copy and ':copy') or ':move'
    if skip_commands():
        msg_skip(line_nr, recdict, cmdname + ' ' + arg)
        return
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    # TODO: check if all options are handled properly
    from Commands import get_args
    opt = {"f": "force", "force": "force",
           "i": "interactive", "interactive": "interactive",
           "e": "exist", "exist": "exist", "exists": "exist",
           "m": "mkdir", "mkdir": "mkdir",
           "c": "continue", "continue": "continue",
           "q": "quiet", "quiet": "quiet"}
    if copy:
        opt.update({
                "u": "unlink", "unlink": "unlink",
                "p": "preserve", "preserve": "preserve",
                "r": "recursive", "recursive": "recursive",
                "k": "keepdir", "keepdir": "keepdir"})

    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg, opt)
    if attrdict:
        option_error(rpstack, attrdict, cmdname)
    if len(argdictlist) < 2:
        recipe_error(rpstack, _("%s command requires at least two arguments")
                                                                     % cmdname)

    remote_copy_move(rpstack, recdict, copy, argdictlist[:-1],
                                    argdictlist[-1], optiondict, 1, errmsg = 1)

# Schemes handled by scheme_copy().  When changing this also need to change
# code and comments below!
scheme_copy_names = ["rcp", "scp", "rsync"]

def scheme_copy(rpstack, recdict, scheme, fromfile, tmach, destpath, fcount):
    """
    Use rcp, scp or rsync, specified with "scheme" to copy "fromfile" (with
    "fcount" file names in quotes) to tmach:destpath.
    Return non-zero for success.
    """
    # Install rcp/scp/rsync when needed.
    from DoInstall import assert_pkg
    assert_pkg(rpstack, recdict, scheme)

    from Remote import get_progname_rsync, get_progname_rcp, get_progname_scp
    if scheme == "rcp":
        cmd = "%s %s '%s:%s'" % (get_progname_rcp(recdict),
                                                     fromfile, tmach, destpath)
    elif scheme == "scp":
        cmd = "%s %s '%s:%s'" % (get_progname_scp(recdict),
                                                     fromfile, tmach, destpath)
    else:
        # A bug in rsync: it can't handle "file(with)parenthesis" in the
        # destination, but escaping them in the source is not allowed!
        d = re.sub("([()])", r"\\\1", destpath)
        cmd = "%s %s '%s:%s'" % (get_progname_rsync(recdict),
                                                            fromfile, tmach, d)

    if os.name == "posix":
        # On Unix we can use "tee".
        ok, text = redir_system_int(recdict, cmd)
    else:
        # Scp may ask for a password.  When we can't use "tee" we need to
        # see the message.  Can't catch the text output then...
        ok = logged_system(recdict, "{interactive} " + cmd) == 0
        text = ''

    if text:
        msg_log(recdict, text)
        # Check for an error connecting, scp doesn't exit with an error value
        # then.  Usual messages are:
        #    "Secure connection to vim.sf.net refused."
        #    "lost connection"
        #    "Read-only file system"
        i = string.find(text, "connection to")
        if i > 0:
            e = string.find(text, "\n", i)
            if e > 0:
                i = string.find(text, "refused", i, e)
            else:
                i = -1
        if i > 0 or string.find(text, "lost connection") >= 0:
            ok = 0
        elif string.find(text, "Permission denied") >= 0:
            # Happens when a file exists and permissions don't allow
            # overwriting: scp: vim/new/con_cvs.php: Permission denied
            # TODO: check the file name and only mark this one as failed.
            ok = 0
        elif string.find(text, "Read-only file system") >= 0:
            # Happens when the file system was mounted read-only.
            # TODO: check the file name and only mark this one as failed.
            ok = 0
        elif string.find(text, "Not a directory") >= 0:
            # Happens when copying multiple files into a file.
            ok = 0
        elif string.find(text, "No such file or directory") >= 0:
            ok = 0
            msg_info(recdict,
                     _('looks like the directory does not exist, creating it'))
            # TODO: This is just guessing; how do we know if the destination
            # is a directory or a file?
            if fcount > 1:
                dirname = destpath
            else:
                dirname = os.path.dirname(destpath)
            mkdircmd = ('ssh %s mkdir -p %s' % (tmach, dirname))
            if logged_system(recdict, mkdircmd) == 0:
                # Try copying again.
                ok = (logged_system(recdict, cmd) == 0)

    return ok

# Cached ftp connections.
ftp_conn = {}
did_set_exitfunc = 0
old_exitfunc = None

def ftpCloseAll():
    """
    Close all cached ftp connections.  Invoked on exit.
    """
    import ftplib

    if old_exitfunc:
        old_exitfunc()      # Invoke previous exit function.

    global ftp_conn
    for ftp in ftp_conn.values():
        try:
            ftp.quit()
        except ftplib.all_errors, e:
            msg_log({}, _("could not quit an ftp connection: %s") % e)
    ftp_conn = {}


# Cached passwords for ftp connections.
ftp_pass = {}

def ftpConnect(mach):
    """
    Connect to an ftp server "mach".
    If "mach" is in the form "user:passwd@ftp.com" use that.
    Otherwise consult the netrc file or ask the user.
    Returns a tuple.  The first item is an ftp object.
    If the connection was successful the second item is an empty string.
    If the connection failed the second item is an error message.
    """

    # Use a cached connection if there is one.
    if mach in ftp_conn.keys():
        return ftp_conn[mach], ''

    passwd = ''
    acct = ''
    key = None
    msg = ''

    # For "user:passwd@ftp.com" split at the @
    i = string.find(mach, '@')
    if i > 0:
        user = mach[:i]
        machname = mach[i+1:]
        i = string.find(user, ':')
        if i > 0:
            passwd = user[i+1:]
            user = user[:i]
        else:

            key = user + "@" + machname
            passwd = ftp_pass.get(key)
            if not passwd:
                prompt = (_('Enter password for user %s at %s: ')
                                                        % (user, machname))
                try:
                    import getpass
                    passwd = getpass.getpass(prompt)
                except:
                    # TODO: should display stars for typed chars
                    passwd = raw_input(prompt)
    else:
        user = ''
        machname = mach

        import aapnetrc
        # obtain the login name and password from the netrc file
        try:
            n = aapnetrc.netrc()
            res = n.authenticators(machname)
            if res is None:
                user = ''
            else:
                user, acct, passwd = res
        except (IOError, aapnetrc.NetrcParseError), e:
            pass

    # Try to open the connection to the ftp server.
    import ftplib
    try:
        ftp = ftplib.FTP()

        i = string.find(machname, ':')
        if i > 0:
            # Port number specified.
            ftp.connect(machname[0:i], machname[i+1:])
        else:
            # Use default port number.
            ftp.connect(machname)

        if user != '':
            if passwd != '':
                if acct != '':
                    ftp.login(user, passwd, acct)
                else:
                    ftp.login(user, passwd)
            else:
                ftp.login(user)
        else:
            ftp.login()
    except ftplib.all_errors, e:
        msg = (_('Cannot open connection to "%s": %s') % (mach, str(e)))
    else:
        if key and passwd:
            # Remember the typed password, so that the user doesn't have to
            # type it again.
            ftp_pass[key] = passwd

        # Cache the connection for another time.
        ftp_conn[mach] = ftp

        # Setup the exit function to close the connection.
        global did_set_exitfunc
        global old_exitfunc
        if not did_set_exitfunc:
            did_set_exitfunc = 1
            old_exitfunc = getattr(sys, 'exitfunc', None)
            sys.exitfunc = ftpCloseAll

    return ftp, msg


def remote_copy_move(rpstack, recdict, copy, from_items, to_item,
                                            optiondict, argexpand, errmsg = 0):
    """Copy or move command.  Copy when "copy" is non-zero.
       "from_items" is a dictlist of the files to copy or move from.
       "to_item" is the dictionary of the destination.
       "optiondict" is a dictionary with options:
            recursive: recursive copy.
            exist: skip if destination exists
            interactive: ask about overwriting.
            preserve: keep mode bits and timestamp
            quiet: don't report copied/moved files
            continue: ignore wildcards without matches
            mkdir: create directory when needed
       "argexpand" is non-zero when wildcards are to be expanded (called from
       ":copy" or ":move").
       When "errmsg" is non-zero, call recipe_error() for an error.
       Returns a list of filenames that failed."""
    from Remote import url_split3
    import glob
    failed = []

    # -l: change symlinks to real files
    #if optiondict.get("unlink"):
    #    keepsymlinks = 0
    #else:
    #    keepsymlinks = 1

    # Expand and check the "from" arguments.
    fromlist = []
    for a in from_items:
        fname = a["name"]
        fscheme, fmach, fpath = url_split3(fname)
        if fscheme == '':
            # It's a local file, may expand ~user and wildcards.
            f = os.path.expanduser(fname)
            if argexpand:
                fl = glob.glob(f)
                if len(fl) == 0:
                    if optiondict.get("continue") and has_wildcard(f):
                        continue
                    recipe_error(rpstack, _('No match for "%s"') % fname)
            else:
                fl = [ f ]
            # For copy without {r} sources can't be a directory.
            if copy and not optiondict.get("recursive"):
                for l in fl:
                    if os.path.isdir(l):
                        recipe_error(rpstack,
                             _('Copying a directory requires {r} flag: %s') % l)
            fromlist.extend(fl)

        else:
            # It's a URL, append without expanding.
            if not copy:
                recipe_error(rpstack, _('Cannot move from a URL yet: "%s"')
                                                                       % fname)
            if argexpand:
                # Do change [*] to *, we might support expanding later.
                f = expand_unescape(fname)
            else:
                f = fname
            fromlist.append(f)

    # Expand and check the "to" argument.
    tname = to_item["name"]
    tscheme, tmach, tpath = url_split3(tname)
    if tscheme == '':
        # For a local file expand "~/" and "~user/".  Don't use "tname", it may
        # start with "file:".
        tpath = os.path.expanduser(tpath)
    if argexpand:
        if tscheme == '':
            # For a local destination file expand wildcards.  If expanding
            # results in no matches use the name without expansion.
            l = glob.glob(tpath)
            if len(l) > 1:
                recipe_error(rpstack, _('More than one match for "%s"') % tname)
            if len(l) == 1:
                tpath = l[0]
        else:
            # Can't expand wildcards in a URL yet, at least remove escaped
            # wildcards.
            tpath = expand_unescape(tpath)

    if tscheme == '' and optiondict.get("mkdir"):
        # Create local target directory when it doesn't exist.
        if len(fromlist) > 1:
            dir = tpath;
        else:
            dir = os.path.dirname(tpath)
        may_create_dir(recdict, rpstack, dir, optiondict.get("quiet"))

    # If there is more than one source, target must be a directory.
    if tscheme == '' and len(fromlist) > 1 and not os.path.isdir(tpath):
        recipe_error(rpstack, _('Destination must be a directory: "%s"')
                                                                       % tname)

    msg = ''
    if tscheme == "ftp":
        #
        # Prepare for uploading through ftp.
        #
        ftp, msg = ftpConnect(tmach)

    elif tscheme in scheme_copy_names:
        #
        # TODO: Prepare for uploading through rcp/scp/rsync.
        #
        pass

    elif tscheme != '':
        msg = _('Can only upload to rcp://, scp://, rsync:// and ftp://')

    # If copy/move isn't possible either give an error message or return the
    # whole list of sources.
    if msg:
        if argexpand:
            msg_error(recdict, msg)
            return map(lambda x : x["name"], from_items)
        recipe_error(rpstack, msg)

    # files to be copied with scp or rsync later.
    scheme_files = {}
    scheme_filelist = {}
    for scheme in scheme_copy_names:
        scheme_files[scheme] = ''
        scheme_filelist[scheme] = []

    #
    # Loop over all "from" files.
    #
    for fname in fromlist:
        fscheme, fmach, fpath = url_split3(fname)

        # If the destination is a directory, append the source file name to the
        # destination directory.
        # "dest" includes the scheme (may be "file:" when "tscheme" is empty).
        if ((tscheme != '' and len(fromlist) > 1)
                                  or (tscheme == '' and os.path.isdir(tpath))):
            if optiondict.get("keepdir"):
                dest = os.path.join(tname, fpath)
                destpath = os.path.join(tpath, fpath)
                if tscheme == '' and os.path.dirname(fpath):
                    # Create the directory if it doesn't exist.
                    dir = os.path.dirname(destpath)
                    may_create_dir(recdict, rpstack, dir,
                                                       optiondict.get("quiet"))
            else:
                dest = os.path.join(tname, os.path.basename(fname))
                destpath = os.path.join(tpath, os.path.basename(fname))
        else:
            dest = tname
            destpath = tpath

        # If destination is a local file and exists:
        # - When {exist} option used, silently ignore.
        # - When {interactive} option used, ask for overwriting.  Use a special
        #   string to allow translating the response characters.
        if tscheme == '' and os.path.exists(destpath):
            if optiondict.get("exist"):
                msg = _('file already exists: "%s"') % dest
                msg_note(recdict, msg)
                failed.append(fname)
                continue
            if optiondict.get("interactive"):
                reply = raw_input(_('"%s" exists, overwrite? (y/n) ') % dest)
                if (len(reply) == 0 or not reply[0]
                              in _("yY   up to four chars that mean yes")[:4]):
                    if copy:
                        msg_note(recdict, _("file not copied"))
                    else:
                        msg_note(recdict, _("file not moved"))
                    failed.append(fname)
                    continue

        if fscheme == '' and tscheme == '':
            #
            # local file copy or move
            #

            if not copy:
                try:
                    os.rename(fpath, destpath)
                    done = 1
                except:
                    done = 0        # renaming failed, try copying

            if copy or not done:
                try:
                    import shutil

                    # TODO: handle symlinks and "keepsymlinks".
                    if os.path.isdir(fpath):
                        # Cannot use shutil.copytree here, it fails when the
                        # destination already exists.
                        if not os.path.exists(destpath):
                            os.mkdir(destpath)
                        # Call ourselves recursively
                        from Commands import dir_contents
                        flist = map(lambda x: {"name": x}, dir_contents(fpath))

                        if flist:
                            # Remove the "keepdir" attribute, the directory
                            # name is already in "destpath" now.
                            if optiondict.has_key("keepdir"):
                                opt = optiondict.copy()
                                del opt["keepdir"]
                            else:
                                opt = optiondict

                            f = remote_copy_move(rpstack, recdict, copy, flist,
                                         {"name": destpath}, opt, 0, errmsg)
                            if f:
                                failed.extend(f)
                    elif not copy or optiondict.get("preserve"):
                        shutil.copy2(fpath, destpath)
                    else:
                        shutil.copy(fpath, destpath)
                except (IOError, OSError), e:
                    msg = (_('Cannot copy "%s" to "%s": %s')
                                                       % (fname, dest, str(e)))
                    if not argexpand and not errmsg:
                        msg_note(recdict, msg)
                        failed.append(fname)
                    else:
                        recipe_error(rpstack, msg)
                    continue
                if not copy:
                    if os.path.isdir(fpath):
                        deltree(fpath)
                    else:
                        os.remove(fpath)

            if not optiondict.get("quiet"):
                f = shorten_name(fpath)
                if copy:
                    msg_info(recdict, _('Copied "%s" to "%s"') % (f, dest))
                else:
                    msg_info(recdict, _('Moved "%s" to "%s"') % (f, dest))

        else:
            if fscheme != '':
                # download to local file
                from Remote import url_download

                try:
                    if tscheme != '':
                        tmpfile, rtime = url_download(recdict, fname, '')
                    else:
                        tmpfile, rtime = url_download(recdict, fname, destpath)
                except IOError, e:
                    msg = (_('Cannot download "%s" to "%s": %s')
                                                       % (fname, dest, str(e)))
                    if not argexpand and not errmsg:
                        msg_note(recdict, msg)
                        failed.append(fname)
                    else:
                        recipe_error(rpstack, msg)
                    continue

            if tscheme != '':
                if fscheme != '':
                    # use temporary file
                    fromfile = tmpfile
                else:
                    fromfile = fpath

                postponed = 0
                if tscheme == 'ftp':
                    # No system command, give a message about what we're doing,
                    # it may take a while.
                    if fscheme != '':
                        msg_info(recdict, _('Uploading to "%s"' % dest))
                    else:
                        if not copy:
                            msg_info(recdict, _('Moving "%s" to "%s"'
                                                              % (fname, dest)))
                        else:
                            msg_info(recdict, _('Uploading "%s" to "%s"'
                                                              % (fname, dest)))

                    # FTP must use '/' path separator characters,
                    # os.path.join() may have used something else.
                    if os.sep != '/':
                        destpath = string.replace(destpath, os.sep, '/')

                    def store_file(ftp, fromfile, destpath):
                        import ftplib       # avoid warning message
                        error = None
                        f = open(fromfile, "rb")
                        try:
                            ftp.storbinary("STOR " + destpath, f, 8192)
                        except ftplib.all_errors, e:
                            error = e
                        f.close()
                        return error

                    error = store_file(ftp, fromfile, destpath)

                    # If it looks like the directory doesn't exist, try
                    # creating it.
                    if (error and string.find(str(error),
                                                  "No such file or dir") >= 0):
                        dir = os.path.dirname(destpath)
                        if dir:
                            import ftplib
                            msg_info(recdict, _('Directory does not appear to exist, creating it...'))
                            try:
                                ftp.mkd(dir)
                            except ftplib.all_errors, e:
                                msg_info(recdict, _('Could not create directory'))
                                pass
                            else:
                                msg_info(recdict, _('Directory created, attempt uploading again...'))
                                error = store_file(ftp, fromfile, destpath)

                    if error:
                        msg = (_('Cannot upload "%s" to "%s": %s')
                                                   % (fname, dest, str(error)))
                        if not argexpand and not errmsg:
                            msg_note(recdict, msg)
                            failed.append(fname)
                        else:
                            recipe_error(rpstack, msg)
                        if fscheme != '':
                            os.remove(tmpfile)      # delete temporary file
                        continue

                elif tscheme in scheme_copy_names:
                    if fscheme == '' and len(fromlist) > 1:
                        # do all local files at once below
                        scheme_files[tscheme] = (scheme_files[tscheme]
                                                       + "'" + fromfile + "' ")
                        scheme_filelist[tscheme].append(fromfile)
                        postponed = 1
                    elif not scheme_copy(rpstack, recdict, tscheme,
                                     "'" + fromfile + "'", tmach, destpath, 1):
                        msg = _('Cannot upload "%s" to "%s"') % (fname, dest)
                        if not argexpand and not errmsg:
                            msg_note(recdict, msg)
                            failed.append(fname)
                        else:
                            recipe_error(rpstack, msg)
                        if fscheme != '':
                            os.remove(tmpfile)      # delete temporary file
                        continue

                if fscheme != '':
                    os.remove(tmpfile)      # delete temporary file
                    msg_info(recdict, _('Uploaded to "%s"' % dest))
                else:
                    if not copy:
                        os.remove(fname)
                        msg_info(recdict, _('Moved "%s" to "%s"'
                                                % (shorten_name(fname), dest)))
                    elif not postponed:
                        msg_info(recdict, _('Uploaded "%s" to "%s"'
                                                % (shorten_name(fname), dest)))



    # End of loop over all "from" files.

    if fscheme != '':
        from Remote import url_cleanup
        url_cleanup(fscheme)

    # Copy collected files with rcp, scp and rsync.
    for scheme in scheme_copy_names:
        files = scheme_files[scheme]
        if files:
            filelist = scheme_filelist[scheme]

            # If there is only one file to copy, add its basename to the
            # destination path to avoid creating a file by the name of the
            # directory of that file.
            if len(filelist) == 1:
                tpath = os.path.join(tpath,
                        os.path.basename(filelist[0]))

            if scheme_copy(rpstack, recdict, scheme, files, tmach, tpath,
                                                                len(filelist)):
                dl = shorten_dictlist(str2dictlist(rpstack, files))
                flist = map(lambda x: x["name"], dl)
                msg_info(recdict, _('Uploaded "%s" to "%s"' % (flist, tname)))
            else:
                msg = _('Cannot upload "%s" to "%s"') % (files, tname)
                if not argexpand and not errmsg:
                    msg_note(recdict, msg)
                    failed.extend(filelist)
                else:
                    recipe_error(rpstack, msg)

    return failed


def may_create_dir(recdict, rpstack, dir, quiet):
    """
    Create directory "dir" if it does not exist.
    """
    if not os.path.exists(dir):
        try:
            os.makedirs(dir)
            if not quiet:
                msg_info(recdict, _('Created directory "%s"') % dir)
        except StandardError, e:
            recipe_error(rpstack, _('Destination directory "%s" cannot be created: %s') % (dir, str(e)))


# vim: set sw=4 et sts=4 tw=79 fo+=l: