#
# Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of version 2.1 of the GNU Lesser General Public
# License as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

import os, stat, signal, fcntl, exceptions
from os.path import dirname, basename
import tempfile, glob, re, urlparse, time
from twisted.internet import reactor
from twisted.python.failure import Failure
from twisted.internet import error, protocol
from twisted.web import http

import memleak
import fetchers, cache, packages
from misc import log, MirrorRecycler
import twisted_compat

#from posixfile import SEEK_SET, SEEK_CUR, SEEK_END
#since posixfile is considered obsolete I'll define the SEEK_* constants
#myself.
SEEK_SET = 0
SEEK_CUR = 1
SEEK_END = 2

from types import *

status_dir = '.apt-proxy'

class Backend:
    """
    A backend repository.  There is one Backend for each [...] section
    in apt-proxy.conf
    """

    "Sequence of BackendServers, in order of preference"
    uris = []

    "Hash of active cache entries"
    entries = {}

    "Packages database for this backend"
    packages = None
    name = None

    downloadQueuePerClient = True # Set to true if a download queue should be created per client

    def __init__(self, factory, config):
        log.debug("Creating Backend: " + config.name)
        self.factory = factory
        self.config = config # apBackendConfig configuration information
        self.base = config.name # Name of backend
        self.uris = [] # Sequence of BackendServers, in order of preference

        if self.downloadQueuePerClient:
            self.queue = fetchers.DownloadQueuePerClient()
        else:
            self.queue = fetchers.DownloadQueue()

        self.entries = {} # Hash of active cache entries
        self.packages = None # Packages database for this backend

        for uri in config.backends:
            self.addURI(uri)

        #self.get_packages_db().load()

    def addURI(self, uri):
        newBackend = BackendServer(self, uri)
        self.uris.append(newBackend)

    def get_first_server(self):
        "Provide first BackendServer for this Backend"
        return self.uris[0]

    def get_next_server(self, previous_server):
        "Return next server, or None if this is the last server"
        oldServerIdx = self.uris.index(previous_server)
        if(oldServerIdx+1 >= len(self.uris)):
            return None
        return self.uris[oldServerIdx+1]

    def __str__(self):
        return '('+self.base+')'+' servers:'+str(len(self.uris))

    def get_cache_entry(self, path):
        """
        Return CacheEntry for given path
        a new object is created if it does not already exist
        """
        if self.entries.has_key(path):
            log.debug("Cache entry exists: %s, %s entries" %(path,len(self.entries)))
            return self.entries[path]
        else:
            log.debug("New Cache entry: "+path)
            e = cache.CacheEntry(self, path)
            self.entries[path] = e
            return e
    def entry_done(self, entry):
        "A cache entry is finished and clients are disconnected"
        #if self.entries.has_key(entry.path):
        log.debug("entry_done: %s" %(entry.path), 'Backend')
        del self.entries[entry.path]

    def get_packages_db(self):
        "Return packages parser object for the backend, creating one if necessary"
        if self.packages == None:
            self.packages = packages.AptPackages(self.base, self.factory.config.cache_dir)
        return self.packages

    def get_path(self, path):
        """
        'path' is the original uri of the request.

        We return the path to be appended to the backend path to
        request the file from the backend server
        """
        return path[len(self.base)+2:]

    def file_served(self, entry):
        "A cache entry has served a file in this backend"
        self.get_packages_db().file_updated(entry)

    def start_download(self, entry):
        """
        A CacheEntry has requested that a file should be downloaded from the backend
        """
        self.queue.addFile(entry)

    def close(self):
        "Clean up backend and associated structures"
        if self.queue is not None:
            self.queue.stop()
            self.queue = None
        if self.packages is not None:
            del(self.packages)

class BackendServer:
    """
    A repository server.  A BackendServer is created for each URI defined in 'backends'
    for a Backend
    """
    
    backend = None        # Backend for this URI
    uri = None            # URI of server

    fetchers = {
        'http' : fetchers.HttpFetcher,
        'ftp'  : fetchers.FtpFetcher,
        'rsync': fetchers.RsyncFetcher,
        'file' : fetchers.FileFetcher,
        }
    ports = {
        'http' : 80,
        'ftp'  : 21,
        'rsync': 873,
        'file' : 0,
        }
    
    def __init__(self, backend, uri):
        self.backend = backend
        self.uri = uri
        log.debug("Created new BackendServer: " + uri)

        # hack because urlparse doesn't support rsync
        if uri[0:6] == 'rsync:':
            uri = 'http'+uri[5:]
            is_rsync=1
        else:
            is_rsync=0

        self.scheme, netloc, self.path, parameters, \
                     query, fragment = urlparse.urlparse(uri)
        if is_rsync:
            self.scheme = 'rsync'

        if '@' in netloc:
            auth = netloc[:netloc.rindex('@')]
            netloc = netloc[netloc.rindex('@'):]
            self.username, self.password = auth.split(':')
        else:
            self.username = None
            self.password = None
        if ':' in netloc:
            self.host, self.port = netloc.split(':')
        else:
            self.host = netloc
            self.port = self.ports[self.scheme]
        self.fetcher = self.fetchers[self.scheme]
        try:
            self.port = int(self.port)
        except ValueError:
            pass 

    def __str__(self):
        return ('(' + self.backend.base + ') ' + self.scheme + '://' +
               self.host + ':' + str(self.port))

class Request(http.Request):
    """
    Each new request from connected clients generates a new instance of this
    class, and process() is called.
    """
    if_modified_since = None
    local_size = None
    serve_if_cached = 1
    apFetcher = None
    uriIndex = 0             # Index of backend URI
    backend = None           # Backend for this request
    backendServer = None     # Current server to be tried
    cacheEntry = None        # Cache entry for file requested
    
    def __init__(self, channel, queued):
        log.debug("New Request, queued=%s" % (queued),'Request');
        self.factory=channel.factory
        http.Request.__init__(self, channel, queued)

    def process(self):
        """
        Each new request begins processing here
        """
        self.uri = self.clean_path(self.uri)

        if_modified_since = self.getHeader('if-modified-since')
        if if_modified_since != None:
            self.if_modified_since = http.stringToDatetime(
                    if_modified_since)

        if self.uri[0] != '/':
            log.debug("Request must include at least one '/'")
            self.finishCode(http.FORBIDDEN, "Request must include at least one '/'")
            return

        backendName = self.uri[1:].split('/')[0]
        log.debug("Request: %s %s backend=%s uri=%s"
                    % (self.method, self.uri, backendName, self.uri),'Request')

        if self.method != 'GET':
            #we currently only support GET
            log.debug("abort - method not implemented", 'Request')
            self.finishCode(http.NOT_IMPLEMENTED)
            return

        if re.search('/\.\./', self.uri):
            log.debug("/../ in simplified uri ("+self.uri+")", 'Request')
            self.finishCode(http.FORBIDDEN)
            return

        self.backend = self.factory.getBackend(backendName)
        if self.backend is None:
            self.finishCode(http.NOT_FOUND, "NON-EXISTENT BACKEND")
            return None

        log.debug("backend: %s %s" % (self.backend.base, self.backend.uris))

        backend_path = self.uri.split('/',2)[2]
        self.cacheEntry = self.backend.get_cache_entry(backend_path)

        if not self.cacheEntry.filetype:
            log.debug("abort - unknown extension for file %s" % (backend_path), 'Request')
            self.finishCode(http.FORBIDDEN, 'File not found - unknown extension')
            return

        self.setHeader('content-type', self.cacheEntry.filetype.contype)

        if os.path.isdir(self.cacheEntry.file_path):
            log.debug("abort - Directory listing not allowed", 'Request')
            self.finishCode(http.FORBIDDEN, 'Directory listing not permitted')
            return

        self.cacheEntry.add_request(self)

    def clean_path(self, uri):
        # Clean up URL given
        scheme, netloc, path, params, query, fragment = urlparse.urlparse(uri)
        return os.path.normpath(path)

    def not_modified(self):
        """
        File is not modified - send http hit
        """
        self.setHeader("content-length", 0)
        self.finishCode(http.NOT_MODIFIED, 'File is up to date')

    def start_streaming(self, size, mtime):
        """
        Prepare client to stream file
        Return false if streaming is not necessary (i.e. cache hit)
        """
        if self.if_modified_since is None or self.if_modified_since < mtime:
            log.debug("start_streaming size=%s mtime=%s if_modified_since=%s" % (size, mtime, self.if_modified_since) , 'Request')
            if mtime is not None:
                self.setHeader('last-modified', http.datetimeToString(mtime))
            if size is not None:
                self.setHeader('content-length', size)
            self.setResponseCode(http.OK, 'Streaming file')
            return True
        else:
            log.debug("file not modified: mtime=%s if_modified_since=%s" % (mtime, self.if_modified_since) , 'Request')
            self.not_modified()
            return False

    def finishCode(self, responseCode, message=None):
        "Finish the request with a status code and no streamed data"
        log.debug("finishCode: %s, %s" % (responseCode, message), 'Request')
        self.setResponseCode(responseCode, message)
        self.write("")
        self.finish()

    def finish(self):
        "Finish request after streaming"
        log.debug("finish. Queued: %s" % (self.queued) , 'Request')
        http.Request.finish(self)

        if self.cacheEntry:
            self.cacheEntry.remove_request(self)
            self.cacheEntry = None

    def connectionLost(self, reason=None):
        """
        The connection with the client was lost, remove this request from its
        Fetcher.
        """
        log.debug("connectionLost" , 'Request')
        if self.cacheEntry:
            self.cacheEntry.remove_request(self)
        #self.finish()

    def getFileno(self):
        """
        Get identifier which is unique per apt client
        """
        try:
            fileno = self.channel.transport.fileno()
        except:
            fileno = -1
            log.msg("could not get transport's file descriptor", 'Request')
        return fileno

class Channel(http.HTTPChannel):
    """
    This class encapsulates a channel (an HTTP socket connection with a single
    client).

    Each incoming request is passed to a new Request instance.
    """
    requestFactory = Request
    log_headers = None

    def headerReceived(self, line):
        "log and pass over to the base class"
        log.debug("Header: " + line)
        if self.log_headers == None:
            self.log_headers = line
        else:
            self.log_headers += ", " + line
        http.HTTPChannel.headerReceived(self, line)

    def allContentReceived(self):
        if self.log_headers != None:
            log.debug("Headers: " + self.log_headers)
            self.log_headers = None
        http.HTTPChannel.allContentReceived(self)

    def connectionLost(self, reason=None):
        "If the connection is lost, notify all my requests"
        __pychecker__ = 'unusednames=reason'
        log.debug("Client connection closed", 'Channel')
        http.HTTPChannel.connectionLost(self, reason)
        if log.isEnabled('memleak'):
            memleak.print_top_10()
        #reactor.stop()   # use for shutting down apt-proxy when a client disconnects

    #def requestDone(self, request):
        #log.debug("========Request Done=========", 'Channel')
        #http.HTTPChannel.requestDone(self, request)
        
class Factory(protocol.ServerFactory):
    """
    This is the center of apt-proxy, it holds all configuration and global data
    and gets attached everywhere.

    Factory receives incoming client connections and creates a Channel for
    each client request.

    interesting attributes:

    self.runningFetchers: a dictionary, uri/Fetcher pairs, that holds the
    active Fetcher for that uri if any. If there is an active Fetcher for
    a certain uri at a certain time the request is inserted into the Fetcher
    found here instead of instanciating a new one.

    Persisten dictionaries:
    self.update_times: last time we checked the freashness of a certain file.
    self.access_times: last time that a certain file was requested.
    self.packages: all versions of a certain package name.
    
    """



    def __init__ (self, config):
        self.runningFetchers = {}
        self.backends = {}
        self.config = config
        self.periodicCallback = None
        self.databases = databaseManager(self)
        self.recycler = None

    def __del__(self):
        pass
        #self.closeDatabases()

    def periodic(self):
        "Called periodically as configured mainly to do mirror maintanace."
        log.debug("Doing periodic cleaning up")
        self.periodicCallback = None
        self.clean_old_files()
        self.recycler.start()
        log.debug("Periodic cleaning done")
        self.startPeriodic()

    def startPeriodic(self):
        if (self.config.cleanup_freq != None and self.periodicCallback is None):
            log.debug("Will do periodic cleaup in %s sec" % (self.config.cleanup_freq))
            self.periodicCallback = reactor.callLater(self.config.cleanup_freq, self.periodic)

    def stopPeriodic(self):
        if self.periodicCallback is not None:
            self.periodicCallback.cancel()
            self.periodicCallback = None

    def __getattr__ (self, name):
        # Auto open database if requested
        if name in self.databases.table_names:
            db = self.databases.get(name)
            setattr(self, name, db)
            return db
        else:
            raise AttributeError(name)

    def startFactory(self):
        #start periodic updates
        self.configurationChanged()
        self.dumpdbs()
        self.recycler = MirrorRecycler(self, 1)
        #self.recycler.start()

    def configurationChanged(self, oldconfig = None):
        """
        Configuration has changed - update backends and background processes
        """
        if oldconfig is not None:
            for param in 'address', 'port', 'telnet_port', 'telnet_user', 'telnet_pass', 'cache_dir':
                if getattr(self.config, param) != getattr(oldconfig, param):
                    log.err('Configuration value %s has changed, ignored'%(param))
                    log.err('You must restart apt-proxy for the change to take effect')
                    setattr(self.config, param, getattr(oldconfig, param))

        if self.config.cleanup_freq != None and (oldconfig is None or oldconfig.cleanup_freq == None):
            self.startPeriodic()
        self.createBackends()

    def createBackends(self):
        self.backends = {}
        for name, bconf in self.config.backends.items():
            #print "[",name,"]"
            self.backends[name] = Backend(self, bconf)

    def getBackend(self, name):
        """
        Return backend with given name
        @param name Name of backend as specified in [backendName] section in config file
        @return Backend class, or None if not found
        """
        if self.backends.has_key(name):
            return self.backends[name]

        if not self.config.dynamic_backends:
            return None

        # We are using dynamic backends so we will use the name as
        # the hostname to get the files.
        backendServer = "http://" + name
        log.debug("Adding dynamic backend:" + name)
        backendConfig = self.config.addBackend(None, name, (backendServer,))
        backend = Backend(self, backendConfig)
        self.backends[name] = backend
        return backend

    def clean_versions(self, packages):
        """
        Remove entries for package versions which are not in cache, and delete
        some files if needed to respect the max_versions configuration.

        TODO: This must be properly done per distribution.
        """
        if self.config.max_versions == None:
            #max_versions is disabled
            return
        package_name = None
        cache_dir = self.config.cache_dir

        cached_packages = []   # all packages in cache directory
        current_packages = []  # packages referenced by Packages files

        import apt_pkg
        def reverse_compare(a, b):
            """ Compare package versions in reverse order """
            return apt_pkg.VersionCompare(b[0], a[0])

        if len(packages) <= self.config.max_versions:
            return

        from packages import AptDpkgInfo, get_mirror_versions
        for uri in packages[:]:
            if not os.path.exists(cache_dir +'/'+ uri):
                log.debug("clean_versions: file %s no longer exists"%(uri),
                        'versions')
                packages.remove(uri)
            else:
                try:
                    info = AptDpkgInfo(cache_dir +'/'+ uri)
                    cached_packages.append([info['Version'], uri])
                    package_name = info['Package']
                except SystemError:
                    log.msg("Found problems with %s, aborted cleaning"%(uri),
                            'versions')
                    return

        if len(cached_packages) > 0:
            import apt_pkg
            cached_packages.sort(reverse_compare)
            log.debug(str(cached_packages), 'versions')

            current_packages = get_mirror_versions(self, package_name)
            current_packages.sort(reverse_compare)
            log.debug("Current Versions: " + str(current_packages), 'versions')

            version_count = 0

            while len(cached_packages):
                #print 'current:',len(current_packages),'cached:',len(cached_packages), 'count:', version_count
                if len(current_packages):
                    compare_result = apt_pkg.VersionCompare(current_packages[0][0], cached_packages[0][0])
                    #print 'compare_result %s , %s = %s ' % (
                    #              current_packages[0][0], cached_packages[0][0], compare_result)
                else:
                    compare_result = -1

                if compare_result >= 0:
                    log.debug("reset at "+ current_packages[0][1], 'max_versions')
                    del current_packages[0]
                    version_count = 0
                else:
                    version_count += 1
                    if version_count > self.config.max_versions:
                        log.msg("Deleting " + cache_dir +'/'+ cached_packages[0][1], 'max_versions')
                        os.unlink(cache_dir +'/'+ cached_packages[0][1])
                        packages.remove(cached_packages[0][1])
                    del cached_packages[0]

    def clean_old_files(self):
        """
        Remove files which haven't been accessed for more than 'max_age' and
        all entries for files which are no longer there.
        """
        if self.config.max_age == None:
            #old file cleaning is disabled
            return
        cache_dir = self.config.cache_dir
        files = self.access_times.keys()
        min_time = time.time() - self.config.max_age

        for file in files:
            local_file = cache_dir + '/' + file
            if not os.path.exists(local_file):
                log.debug("old_file: non-existent "+file)
                del self.access_times[file]
            elif self.access_times[file] < min_time:
                log.debug("old_file: removing "+file)
                os.unlink(local_file)
                del self.access_times[file]

        #since we are at it, clear update times for non existent files
        files = self.update_times.keys()
        for file in files:
            if not os.path.exists(cache_dir+'/'+file):
                log.debug("old_file: non-existent "+file)
                del self.update_times[file]

    def file_served(self, cache_path):
        """
        Update the databases, this file has just been served.
        @param cache_path: path of file within cache e.g. debian/dists/stable/Release.gpg
        """
        log.debug("File served: %s" % (cache_path))
        path = os.sep + cache_path # Backwards compat
        #path = cache_path
        self.access_times[path]=time.time()
        if re.search("\.deb$", path):
            package = re.sub("^.*/", "", path)
            package = re.sub("_.*$", "", package)
            if not self.packages.has_key(package):
                packages = [path]
            else:
                packages = self.packages[package]
                if not path in packages:
                    packages.append(path)
                self.clean_versions(packages)
            self.packages[package] = packages
        self.dumpdbs()

    def closeDatabases(self):
        for db in self.databases.table_names:
            if getattr(self.databases, db) is not None:
                log.debug("closing " + db, 'db')
                getattr(self,db).close()
                delattr(self,db)
                setattr(self.databases, db, None)

    def stopFactory(self):
        log.debug('Main factory stop', 'factory')
        import packages
        # self.dumpdbs()
        
        # Stop all DownloadQueues and their fetchers
        for b in self.backends.values():
            b.close()
        self.backends = {}
        packages.cleanup(self)
        if self.recycler is not None:
            self.recycler.stop()
            self.recycler = None
        self.stopPeriodic()
        #self.closeDatabases()

    def dumpdbs (self):
        def dump_update(key, value):
            log.msg("%s: %s"%(key, time.ctime(value)),'db')
        def dump_access(key, value):
            log.msg("%s: %s"%(key, time.ctime(value)),'db')
        def dump_packages(key, list):
            log.msg("%s: "%(key),'db')
            for file in list:
                log.msg("\t%s"%(file),'db')
        def dump(db, func):
            keys = db.keys()
            for key in keys:
                func(key,db[key])
        if log.isEnabled('db'):
            log.msg("=========================",'db')
            log.msg("Dumping update times",'db')
            log.msg("=========================",'db')
            dump(self.update_times, dump_update)
            log.msg("=========================",'db')
            log.msg("Dumping access times",'db')
            log.msg("=========================",'db')
            dump(self.access_times, dump_access)
            log.msg("=========================",'db')
            log.msg("Dumping packages",'db')
            log.msg("=========================",'db')
            dump(self.packages, dump_packages)


    def buildProtocol(self, addr):
        __pychecker__ = 'unusednames=addr'
        proto = Channel()
        proto.factory = self;
        return proto

    def log(self, request):
        return

    def debug(self, message):
        log.debug(message)

class databaseManager:
    update_times = None
    access_times = None
    packages = None
    table_names=['update_times', 'access_times', 'packages']
    database_files=['update', 'access', 'packages']

    def __init__(self, factory):
        self.factory = factory

    def get(self, name):
        idx = self.table_names.index(name)
        db = getattr(self,name)
        if db is None:
            db = self.open_shelve(self.database_files[idx])
            setattr(self, name, db)
        return db

    def open_shelve(self, dbname):
        from bsddb import db,dbshelve

        shelve = dbshelve.DBShelf()
        db_dir = self.factory.config.cache_dir+'/'+status_dir+'/db'
        if not os.path.exists(db_dir):
            os.makedirs(db_dir)

        filename = db_dir + '/' + dbname + '.db'
        if os.path.exists(filename):
                try:
                    log.debug('Verifying database: ' + filename)
                    shelve.verify(filename)
                except:
                    os.rename(filename, filename+'.error')
                    log.msg(filename+' could not be opened, moved to '+filename+'.error','db', 1)
                    log.msg('Recreating '+ filename,'db', 1)
        try:
            log.debug('Opening database ' + filename)
            shelve = dbshelve.open(filename)

        # Handle upgrade to new format included on 1.9.20.
        except db.DBInvalidArgError:
            log.msg('Upgrading from previous database format: %s' % filename + '.previous')
            import bsddb.dbshelve
            os.rename(filename, filename + '.previous')
            previous_shelve = bsddb.dbshelve.open(filename + '.previous')
            shelve = dbshelve.open(filename)

            for k in previous_shelve.keys():
                shelve[k] = previous_shelve[k]
            log.msg('Upgrade complete')

        return shelve