#
# Copyright (C) 2005 Chris Halls <halls@debian.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of version 2.1 of the GNU Lesser General Public
# License as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
# -*- test-case-name: apt_proxy.test.test_cache -*-

"""
Cache management for apt-proxy

These classes implement functionality for managing apt-proxy's cache.  The most
important of these is CacheEntry, which manages the lifecycle of a file in ap's cache
"""

from twisted.internet import protocol, defer, reactor
from twisted.web import http
from twisted.protocols import basic
import os, re, stat, time, sys
from misc import log

class CacheEntry:
    """
    This class manages operations on a file in the cache.  Each physical
    file on the disk corresponds to one CacheEntry.  Normally a CacheEntry
    is created when the first Request for this file is received

    Active CacheEntries are managed in their corresponding Backend
    """

    # Define lifecyle of cache entry
    STATE_NEW = 1 # Entry is not yet being sent
    STATE_CONNECTING = 2 # Waiting for connection to download file
    STATE_DOWNLOAD = 3 # File is in process of downloading
    STATE_SENDFILE = 4 # File is being sent from cache
    STATE_SENT = 5 # Post download processing / waiting for clients to complete
    STATE_FAILED = 6 # Download failed
    

    bytesDownloaded = 0

    def __init__(self, backend, path):
        """
        Create a new cache entry
        @param backend Backend where this entry belongs
        @param path Path to file within backend directory
        """
        self.backend = backend
        self.factory = backend.factory
        self.requests = [] # Active client requests for this cache entry
        self.streamfile = None
        self.state = self.STATE_NEW

        # Path of file within backend e.g. 'dists/stable/Release.gpg'
        self.path = path 

        # Path of file within cache e.g. 'debian/dists/stable/Release.gpg'
        self.cache_path = backend.base + os.sep + path

        # File in cache '/var/cache/apt-proxy/debian/dists/stable/Release.gpg'
        self.file_path = (self.factory.config.cache_dir + os.sep + 
                          self.cache_path)

        # Directory of cache file '/var/cache/apt-proxy/debian/dists/stable'
        self.filedir = os.path.dirname(self.file_path)

        self.filetype = findFileType(path)
        self.filename = os.path.basename(path) # 'Release.gpg'

        # filebase='Release' fileext='gpg'
        (self.filebase, self.fileext) = os.path.splitext(self.filename)

        # self.create_directory()
        self.file_mtime = None # Modification time of cache file
        self.file_size = None # Size of cache file

        self.fetcher = None

    def add_request(self, request):
        """
        A new request has been received for this file
        """
        if request in self.requests:
            raise RuntimeError, \
                  'this request is already assigned to this CacheEntry'
        self.requests.append(request)
        if(len(self.requests)==1):
            # First request
            self.get()
        else:
            # Subsequent request - client must be brought up to date
            if self.state == self.STATE_DOWNLOAD:
                raise RuntimeError, \
                      'TODO: multiple clients not implemented yet'

    def remove_request(self,request):
        """
        Remove request, either because streaming is complete or
        the client has disconnected

        If parameter request is None, downloading has been aborted early
        """
        if request is not None and request in self.requests:
            self.requests.remove(request)
        if len(self.requests) != 0:
            return

        log.debug("Last request removed",'cacheEntry')
        self.backend.entry_done(self)

        # TODO - fixme
        #if (self.factory.config.complete_clientless_downloads == False
             #and self.state == self.STATE_DOWNLOAD
             #and self.fetcher is not None):
            ## Cancel download in progress
            #log.debug("cancelling download (set complete_clientless_downloads to continue)",'cacheEntry')
            #self.fetcher.cancel_download()

        #if self.streamfile is not None and self.state == self.STATE_DOWNLOAD:
            #name = self.streamfile.name
            #log.debug("Deleting temporary file (%s)" % name,'cacheEntry')
            #self.streamfile.close()
            #self.streamfile = None
            #os.remove(name)

    def start_request_stream(self, request):
        """
        Prepare a request for streaming
        """
        log.msg("start_request_stream:" + self.file_path, "CacheEntry")
        request.startStreaming(self.size, self.mtime)

        if self.streamfile.size() != 0:
            request.write(self.streamfile.read_from(start=0)) # TODO - is this efficient?


    def get(self):
        """
        Update current version of file in cache
        """
        if self.state == self.STATE_NEW:
            if os.path.exists(self.file_path):
                self.stat_file()
                if self.check_age():
                    self.verify()
                    return

        self.start_download()

    def verify(self):
        """
        check the existence and ask for the integrity of the requested file and
        return a Deferred to be trigered when we find out.
        """
        log.debug("check_cached: "+self.path, 'CacheEntry')
        verifier = FileVerifier(self.file_path, self.factory.config)
        d = verifier.verify()
        d.addCallback(self.send_cached_file)
        d.addErrback(self.verify_failed)

    def verify_failed(self, parm=None):
        self.file_mtime = None
        self.file_size = None
        self.start_download()

    def stat_file(self):
        """
        Read file age
        """
        stat_tuple = os.stat(self.file_path)

        self.file_mtime = stat_tuple[stat.ST_MTIME]
        self.file_size = stat_tuple[stat.ST_SIZE]
        log.debug("Modification time:" + 
                  time.asctime(time.localtime(self.file_mtime)), 
                  "CacheEntry")

    def check_age(self):
        """
        Read file age and check if file should be updated / refreshed

        @return True if file is still valid, False if file is out of date
        """

        update_times = self.factory.update_times

        if update_times.has_key(self.cache_path): 
            last_access = update_times[self.cache_path]
            log.debug("last_access from db: " + 
                      time.asctime(time.localtime(last_access)), 
                      "CacheEntry")
        else:
            last_access = self.file_mtime


        cur_time = time.time()
        min_time = cur_time - self.factory.config.min_refresh_delay

        if not self.filetype.mutable:
            log.debug("file is immutable: "+self.file_path, 'CacheEntry')
            return True
        elif last_access < min_time:
            log.debug("file is too old: "+self.file_path, 'CacheEntry')
            return False
        else:
            log.debug("file is ok: "+self.file_path, 'CacheEntry')
            return True

    def send_cached_file(self, unused=None):
        """
        File is up to date - send complete file from cache to clients
        """
        if self.file_mtime is not None:
            log.msg("sending file from cache:" + self.file_path, "CacheEntry")
            self.transfer_file(self.file_path)
        else:
            log.msg("sending hits to all clients (%s)" % (self.file_path), "CacheEntry")
            for req in self.requests:
                req.not_modified()

    def end_send_cached(self):
        """
        Processing continues here when the file has been sent from the cache
        """
        self.file_sent()

    def transfer_file(self, filename):
        """
        Send given file to clients
        """
        log.msg("transfer_file:" + filename, "CacheEntry")
        try:
            stat_tuple = os.stat(filename)
            mtime = stat_tuple[stat.ST_MTIME]
            size = stat_tuple[stat.ST_SIZE]
    
            self.state = self.STATE_SENDFILE
            if size > 0:
                log.debug("Sending file to clients:%s size:%s" % (filename, size), 'CacheEntry')
                self.streamfile = open(filename,'rb')
                #fcntl.lockf(file.fileno(), fcntl.LOCK_SH)

                for request in self.requests:
                    if request.start_streaming(size, mtime):
                        basic.FileSender().beginFileTransfer(self.streamfile, request) \
                                        .addBoth(self.file_transfer_complete, request, filename)
            else:
                log.debug("Sending empty file to clients:%s" % (filename), 'CacheEntry')
                for request in self.requests:
                    if request.start_streaming(size, mtime):
                        request.finish()
        except Exception, e:
            log.debug("Unexpected error: %s" % (e), 'CacheEntry')
            raise

    def file_transfer_complete(self, result, request, filename):
        log.debug("transfer complete: " + filename, 'CacheEntry')
        request.finish()
        if len(self.requests)==0:
            # Last file was sent
            self.file_sent()

    def create_directory(self):
        """
        Create directory for cache entry's file
        """
        if(not os.path.exists(self.filedir)):
            os.makedirs(self.filedir)

    def start_download(self):
        """
        Start file transfer from backend server
        """
        log.msg("start download:" + self.path, "CacheEntry")

        self.backend.start_download(self)

    def get_request_mtime(self):
        """
        Return the latest modification time for which a HIT can be given instead
        of downloading the file
        """
        backend_mtime = None # modification time to request from backend

        # Find the earliest if-modified-since of the requests
        for req in self.requests:
            if req.if_modified_since is not None:
                if backend_mtime is None:
                    backend_mtime = req.if_modified_since
                elif req.if_modified_since < backend_mtime:
                    backend_mtime = req.if_modified_since

        # If our cached file is newer, use that time instead
        if self.file_mtime is not None and backend_mtime < self.file_mtime:
            backend_mtime = self.file_mtime

        return backend_mtime

    def download_started(self, fetcher, size, mtime):
        """
        Callback from Fetcher
        A fetcher has begun streaming this file
        """
        log.msg("download started:" + self.file_path, "CacheEntry")
        self.state = self.STATE_DOWNLOAD
        self.create_directory()
        self.fetcher = fetcher
        self.file_mtime = mtime

        """
        Use post_convert and gzip_convert regular expresions of the Fetcher
        to gzip/gunzip file before and after download.
        """

        if self.filename == 'Packages.gz':
            log.msg('TODO postconvert Packages.gz',CacheEntry)
#             if (fetcher and fetcher.post_convert.search(req.uri)
#                 and not running.has_key(req.uri[:-3])):
#                 log.debug("post converting: "+req.uri,'convert')
#                 loop = LoopbackRequest(req)
#                 loop.uri = req.uri[:-3]
#                 loop.local_file = req.local_file[:-3]
#                 loop.process()
#                 loop.serve_if_cached=0
#                 #FetcherGzip will attach as a request of the
#                 #original Fetcher, efectively waiting for the
#                 #original file if needed
#                 gzip = FetcherGzip()
#                 gzip.activate(loop, postconverting=1)


        for req in self.requests:
            req.start_streaming(size, mtime)


    def download_data_received(self, data):
        """
        Callback from Fetcher
        A block of data has been received from the streaming backend server
        """
        #log.msg("download_data_received:" + self.file_path, "CacheEntry")
        for req in self.requests:
            req.write(data)

        if self.streamfile:
            # save to tempfile (if it in use)
            self.streamfile.append(data)

    def download_data_end(self):
        """
        Callback from Fetcher
        File streaming is complete
        """
        log.msg("download_data_end:" + self.file_path, "CacheEntry")
        self.state = self.STATE_SENT

        if self.streamfile is not None:
            # File was streamed to clients
            self.streamfile.close_and_rename(self.file_path)
            self.streamfile = None

            if self.file_mtime != None:
                os.utime(self.file_path, (time.time(), self.file_mtime))
            else:
                log.debug("no local time: "+self.file_path,'Fetcher')
                os.utime(self.file_path, (time.time(), 0))

        for req in self.requests:
            req.finish()

        self.file_sent()

    def download_failure(self, http_code, reason):
        """
        Download is not possible
        """
        log.msg("download_failure %s: (%s) %s"% (self.file_path, http_code, reason), "CacheEntry")

        for request in self.requests:
            request.finishCode(http_code, reason)
        self.state = self.STATE_FAILED
        ## Remove directory if file was not created
        #if not os.path.exists(self.file_path):
            #try:
                #os.removedirs(self.factory.config.cache_dir + os.sep + self.backend.base)
            #except:
                #pass


    def file_sent(self):
        """
        File has been sent successfully to at least one client
        Update databases with statistics for this file
        """
        log.msg("file_sent:" + self.file_path, "CacheEntry")

        self.state = self.STATE_SENT
        self.fetcher = None
        self.backend.file_served(self)
        self.factory.file_served(self.cache_path)
        self.factory.update_times[self.cache_path] = time.time()
        self.state = self.STATE_NEW

    def init_tempfile(self):
        #log.msg("init_tempfile:" + self.file_path, "CacheEntry")
        self.create_directory()
        self.streamFilename = self.file_path + ".apDownload"
        self.streamfile = StreamFile(self.streamFilename)

class FileType:
    """
    This is just a way to distinguish between different filetypes.

    self.regex: regular expression that files of this type should
    match. It could probably be replaced with something simpler,
    but... o well, it works.
    
    self.contype: mime string for the content-type http header.
    
    mutable: do the contents of this file ever change?  Files such as
    .deb and .dsc are never changed once they are created.
    
    """
    def __init__ (self, regex, contype, mutable):
        self.name = regex
        self.regex = re.compile(regex)
        self.contype = contype
        self.mutable = mutable

    def check (self, name):
        "Returns true if name is of this filetype"
        if self.regex.search(name):
            return 1
        else:
            return 0
    def __str__(self):
        return "FileType regex=%s mimetype=%s mutable=%s" % (self.name,self.contype,self.mutable)


# Set up the list of filetypes that we are prepared to deal with.
# If it is not in this list, then we will ignore the file and
# return an error.
filetypes = (
    FileType(r"\.u?deb$", "application/dpkg", 0),
    FileType(r"\.tar\.gz$", "application/x-gtar", 0),
    FileType(r"\.dsc$","text/plain", 0),
    FileType(r"\.diff\.gz$", "x-gzip", 0),
    FileType(r"\.bin$", "application/octet-stream", 0),
    FileType(r"\.tgz$", "application/x-gtar", 0),
    FileType(r"\.txt$", "text/plain", 1),
    FileType(r"\.html$", "text/html", 1),

    FileType(r"(?:^|/)(?:Packages|Release(?:\.gpg)?|Sources|(?:Contents|Translation)-[a-z0-9]+)"
                        r"(?:\.(?:gz|bz2))?$",
             "text/plain", 1),
    FileType(r"(?:^|/)(?:Packages|Sources|Contents-[a-z0-9]+)\.diff/Index$",
             "text/plain", 1),
    FileType(r"(?:^|/)(?:Packages|Sources|Contents-[a-z0-9]+)\.diff/[a-z0-9.-]+"
                        r"(?:\.(?:gz|bz2))?$",
             "text/plain", 0),

    FileType(r"\.rpm$", "application/rpm", 0),

    FileType(r"(?:^|/)(?:pkglist|release|srclist)(?:\.(?:\w|-)+)?"
                        r"(?:\.(?:gz|bz2))?$", 
             "text/plain", 1),
    FileType(r"\.gz$", "x-gzip", 1)
    )


def findFileType(name):
    "Look for the FileType of 'name'"
    for type in filetypes:
        if type.check(name):
            return type
    return None

class StreamFile:
    """
    A temporary file used to stream to during download
    """
    CHUNKSIZE = 16384
    def __init__(self, name, mode='w+b'):
        log.debug("Creating file: " + name, 'cache')
        self.file = file(name, mode, self.CHUNKSIZE)
        self.name = name
    def append(self, data):
        self.file.write(data)
    def size(self):
        return self.file.tell()
    def read_from(self, size=-1, start=None):
        if start != None:
            self.file.seek(start, SEEK_SET)
        data = self.file.read(self, size)
        self.file.seek(0, SEEK_END)
        return data
    def close(self):
        log.debug("Closing file: " + self.name, 'cache')
        self.file.close()
        self.file = None
    def close_and_rename(self, new_name):
        """
        File was successfully downloaded - close and rename to final destination
        """
        self.close()
        if self.name == new_name:
            return
        log.debug("renaming file: %s->%s " % (self.name, new_name), 'cache')
        os.rename(self.name, new_name)
        self.name = new_name

class FileVerifier:
    """
    Verifies the integrity of a cached file

    self.deferred: a deferred that will be triggered when the command
    completes, or if a timeout occurs.

    Sample:

    verifier = FileVerifier(self)
    verifier.deferred.addCallbacks(callback_if_ok, callback_if_fail)
    verifier.deferred.arm()

    then either callback_if_ok or callback_if_fail will be called
    when the subprocess finishes execution.

    Checkout twisted.internet.defer.Deferred on how to use self.deferred

    """
    def __init__(self, path, config):
        """
        Initialise verificatoin
        @param path: filename to be verified (absolute path)
        @param config apConfig configuration (timeout paramter defines max time)
        """
        self.path = path
        self.timeout = config.timeout
        self.deferred = defer.Deferred() # Deferred that passes status back

    def verify(self):
        if re.search(r"\.deb$", self.path):
            self.worker = FileVerifierProcess(self, '/usr/bin/dpkg', '--fsys-tarfile', self.path)
        elif re.search(r"\.gz$", self.path):
            self.worker = FileVerifierProcess(self, '/bin/gunzip', '-t', '-v', self.path)
        elif re.search(r"\.bz2$", self.path):
            self.worker = FileVerifierProcess(self, '/usr/bin/bunzip2', '--test', self.path)
        else:
            # Unknown file, just check it is not 0 size
            try:
                filesize = os.stat(self.path)[stat.ST_SIZE]
            except:
                filesize = 0

            if(os.stat(self.path)[stat.ST_SIZE]) < 1:
                self.failed("Zero length file")
            else:
                log.debug('Verification skipped for ' + self.path)
                self.deferred.callback(None)
        return self.deferred

    class VerificationFailure:
        def __init__(self, path, reason):
            self.path = path
            self.reason = reason
    def failed(self, reason):
        log.msg("cache file verification FAILED for %s: %s"%(self.path, reason), 'verify')
        os.unlink(self.path)
        self.deferred.errback(self.VerificationFailure(self.path, reason))

    def passed(self):
        log.debug("cache file verification passed: %s"%(self.path), 'verify')
        self.parent.deferred.callback(None)

class FileVerifierProcess(protocol.ProcessProtocol):
    """
    Verifies the integrity of a file by running an external command.
    """
    def __init__(self, verifier, *args):
        self.parent = verifier

        self.exe = args[0]
        log.debug("starting verification: " + self.exe + " " + str(args),'FileVerifierProcess',8)
	nullhandle = open("/dev/null", "w")
        self.process = reactor.spawnProcess(self, self.exe, args, childFDs = { 0:"w", 1:nullhandle.fileno(), 2:"r" })
        self.laterID = reactor.callLater(self.parent.timeout, self.timedout)

    def connectionMade(self):
        self.data = ''

    def outReceived(self, data):
        #we only care about errors
        pass

    def errReceived(self, data):
        self.data = self.data + data

    def timedout(self):
        """
        this should not happen, but if we timeout, we pretend that the
        operation failed.
        """
        self.laterID=None
        self.parent.failed("Verification process timed out")

    def processEnded(self, reason=None):
        """
        This get's automatically called when the process finishes, we check
        the status and report through the Deferred.
        """
        __pychecker__ = 'unusednames=reason'
        #log.debug("Process Status: %d" %(self.process.status),'verify')
        #log.debug(self.data, 'verify')
        if self.laterID:
            self.laterID.cancel()
            if self.process.status == 0:
                self.parent.deferred.callback(None)
            else:
                self.parent.failed(os.path.basename(self.exe)+ " failed")