#! /usr/bin/env python

  
#############################################################
#                                                           #
#   Author: Bertrand Neron, Sandrine Larroude               #
#   Organization:'Biological Software and Databases' Group, #
#                Institut Pasteur, Paris.                   #
#   Distributed under GPLv2 Licence. Please refer to the    #
#   COPYING.LIB document.                                   #
#                                                           #
#############################################################

"""
mobdeploy

This script is used to update the list of deployed and imported services
on the mobyle server, as well as the various index files.
"""

import sys, os, urllib2
import simplejson

MOBYLEHOME = None
if os.environ.has_key('MOBYLEHOME'):
    MOBYLEHOME = os.environ['MOBYLEHOME']
if not MOBYLEHOME:
    sys.exit('MOBYLEHOME must be defined in your environment')

if ( MOBYLEHOME ) not in sys.path:
    sys.path.append( MOBYLEHOME )
if ( os.path.join( MOBYLEHOME , 'Src' ) ) not in sys.path:    
    sys.path.append( os.path.join( MOBYLEHOME , 'Src' ) )

import shutil
from lxml import etree
from glob import glob
import logging

from Mobyle.ConfigManager import Config
_cfg = Config()
from Mobyle.Registry import Registry, ServerDef, ProgramDef, WorkflowDef, ViewerDef, TutorialDef,registry as registry_from_existing
from Mobyle import MobyleLogger, InterfacePreprocessor
from Mobyle.Validator import Validator
from Mobyle.SearchIndex import SearchIndex
from Mobyle.ClassificationIndex import ClassificationIndex
from Mobyle.DataInputsIndex import DataInputsIndex
from Mobyle.DescriptionsIndex import DescriptionsIndex
from Mobyle.DataTypeValidator import DataTypeValidator

net_enabled_parser = etree.XMLParser(no_network=False)

class ServicesDeployer(object):
    
    def __init__(self , verbosity = 0, useJing=False ):
        self.useJing = useJing
        self.tmp_dir = os.path.join( _cfg.repository_path() , 'tmp_deployment' )
        self.tmp_servers_dir = os.path.join( self.tmp_dir , 'servers' )
        self.tmp_indexes =  os.path.join( self.tmp_dir , 'index')
        MobyleLogger.MLogger()
        self.log = logging.getLogger('mobyle.registry' )
        console = logging.StreamHandler(sys.stderr)
        if verbosity == 0 :
            self.log.setLevel( logging.WARNING )
            formatter = logging.Formatter('%(levelname)-8s %(message)s')
        elif verbosity == 1:
            self.log.setLevel( logging.INFO )
            formatter = logging.Formatter('%(levelname)-8s %(message)s')
        else :
            self.log.setLevel( logging.DEBUG )
            formatter = logging.Formatter('L %(lineno)d : %(levelname)-8s %(message)s')
        console.setFormatter(formatter)
        self.log.addHandler(console)
        self.config_registry = self.get_registry_from_config()
        self.tpl_preprocessor=InterfacePreprocessor.InterfacePreprocessor()
        self.dtv = DataTypeValidator()
    
    def makeTmp( self ):
        """
        create a temporary architecture to build new services, viewers and indexes ...
        """
        self.log.info( "creating temporary architecture for services pre-deployment")
        if os.path.exists( self.tmp_dir ):
            self.log.critical("%s already exists. Reasons for this can be that \n(1) another mobdeploy is running,\n(2) a previous mobdeploy crashed.\nRemove it before to run mobdeploy again"% self.tmp_dir )
            sys.exit(1)
        os.mkdir( self.tmp_dir )
        os.mkdir( self.tmp_servers_dir )
        os.mkdir( self.tmp_indexes )
     
        
    def switchDeployement( self ):
        """
        replace deployed services, viewers, indexes by the new one 
        """
        self.log.info( "switching from pre-deployed to deployed" )
        effective_services_path = _cfg.services_path()
        old_deployement_path = os.path.join( _cfg.repository_path() , 'old_deployement' )
        if os.path.exists( old_deployement_path ) :
            shutil.rmtree( old_deployement_path )
        if os.path.exists( effective_services_path ):
            os.rename( effective_services_path , old_deployement_path )
        os.rename( self.tmp_dir , effective_services_path )
    
    
    def check_workflow_consistency( self , predeploy_registry ):
        """
        check if each task of local workflow is deployed
        """
        self.log.info( "check local workflows consitency" )
        if predeploy_registry.serversByName.get('local') is not None:
            local_server = predeploy_registry.serversByName[ 'local' ]
            for workflow_def in local_server.workflows :
                doc = etree.parse( workflow_def.path )
                tasks_node = doc.findall( 'flow/task')
                for task_node in tasks_node:
                    service_task = task_node.get( 'service')
                    server_task = task_node.get( 'server' , 'local' )
                    try:
                        predeploy_registry.serversByName[ server_task ].programsByName[ service_task ]
                        continue
                    except KeyError:
                        try:
                            predeploy_registry.serversByName[ server_task ].workflowsByName[ service_task ]
                            continue
                        except KeyError:
                            self.log.warning( "the service %s.%s required by workflow %s is missing." %(
                                                                                                          server_task ,
                                                                                                          service_task ,
                                                                                                          workflow_def.name
                                                                                                          ) )
                            self.log.warning( "workflow %s is removed until service %s.%s is deployed" %(workflow_def.name ,
                                                                                                               server_task ,
                                                                                                               service_task ,
                                                                                                               ) )
                            predeploy_registry.pruneService( workflow_def )
                            try:
                                os.unlink( workflow_def.path )
                            except Exception, err:
                                self.log.error( "unable to remove illegitimate workflow %s : %s" %(workflow_def.name , err ) )
                            
                        
    def doCommand( self , cmd , server_names , programs_name , workflows_names , viewers_name , tutorials_names, force = False ):
        """
    
        """
        self.makeTmp()
        if cmd != 'index':
            cmd_registry = self.get_registry_from_args( server_names , programs_name , workflows_names , viewers_name, tutorials_names)
            
            self._recoverFromExisting( cmd_registry, server_names, programs_name, workflows_names, viewers_name, tutorials_names)
            self._do_cmd( cmd , cmd_registry , force = force )
        idx_registry= Registry()
        idx_registry.servers_path = self.tmp_servers_dir
        idx_registry.load()
        self.check_workflow_consistency( idx_registry )
        self.make_indexes(idx_registry)
        self.switchDeployement()
        
    def get_registry_from_args(self, server_names, programs_names, workflows_names , viewers_names, tutorials_names):
        """
        build a registry based on the arguments 
        @param server_names: the portals name as defined in the configuration where the services can be found, 
          - 'local' mean this portal
          - 'all' mean all the server describe in the config
        @type server_names: list of string.
        @param programs_names: the names of the programs, 
          - 'all' mean all the programs for a server describe in the config 
        @type programs_names: list of string
        @param workflows_names: the names of the workflows, 
          - 'all' mean all the workflows for a server describe in the config 
        @type workflows_names: list of string
        @param viewers_names: the names of the viewers, 
          - 'all' mean all the viewers (only local server has viewers ). 
        @type viewers_names: list of string
        @param tutorials_names: the names of the tutorials, 
          - 'all' mean all the tutorials (only local server has tutorials ). 
        @type tutorials_names: list of string
        @return: a registry builded from the arguments
        @rtype: L{Mobyle.Registry} instance
        """
        args_registry = Registry()
        if server_names == ['all']:
            server_names = _cfg.portals().keys()
            server_names.append( 'local' )
        for server_name in server_names:
            try:
                server_from_conf = self.config_registry.serversByName[ server_name ]
            except KeyError:
                self.log.warning( "the server %s is not defined in the Mobyle Config, it will be skipped" %server_name )
                continue
            server = ServerDef( name = server_from_conf.name ,
                                url  =  server_from_conf.url ,
                                help = server_from_conf.help ,
                                repository = server_from_conf.repository ,
                                jobsBase = server_from_conf.jobsBase
                               )
            args_registry.addServer( server )
            if programs_names == ['all']:
                programs_names_for_this_server = [ p.name for p in server_from_conf.programs ]
            else:
                programs_names_for_this_server = programs_names
            for name in programs_names_for_this_server:
                if server.name == 'local':
                    try:
                        url = server_from_conf.programsByName[ name ].url
                        path = server_from_conf.programsByName[ name ].path
                    except KeyError:
                        custom_path = os.path.join( _cfg.mobylehome() , 'Local' , 'Services' , 'Programs' , name +'.xml' )
                        if os.path.exists( custom_path ):
                            url = "file://%s" %custom_path
                            path = custom_path
                        else:
                            #I put the public_path in service even this service is not exist
                            #this test is performed by is_available
                            #and if cmd is deploy a warnig is raised and service skipped
                            #    if cmd is removed the service will be removed form the next deployement
                            #fix bug #438
                            public_path = os.path.join( _cfg.mobylehome() , 'Services' , 'Programs' , name +'.xml' )
                            url = "file://%s" %public_path
                            path =  public_path
                else:
                    url = args_registry.getProgramUrl( name, server.name )
                    path = None
                service = ProgramDef( name = name,
                                        url = url,
                                        path = path ,
                                        server = server
                                    )
                args_registry.addProgram( service )
            
            if workflows_names == ['all']:
                workflows_names_for_this_server = [ s.name for s in server_from_conf.workflows ]
            else:
                workflows_names_for_this_server = workflows_names
            for name in workflows_names_for_this_server:
                if server.name == 'local':
                    try:
                        url = server_from_conf.workflowsByName[ name ].url
                        path = server_from_conf.workflowsByName[ name ].path
                    except KeyError:
                        custom_path = os.path.join( _cfg.mobylehome() , 'Local' , 'Services' , 'Workflows' , name +'.xml' )
                        if os.path.exists( custom_path ):
                            url = "file://%s" %custom_path
                            path = custom_path
                        else:
                            public_path = os.path.join( _cfg.mobylehome() , 'Services' , 'Workflows' , name +'.xml' )
                            url = "file://%s" %public_path
                            path =  public_path
                else:
                    url = args_registry.getWorkflowUrl( name, server.name)
                    path = None

                workflow = WorkflowDef( name = name,
                                        url = url,
                                        path = path,
                                        server = server
                                    )
                args_registry.addWorkflow( workflow )
            
            if server_name == 'local':
                if viewers_names == ['all']:
                    viewers_names = [ v.name for v in server_from_conf.viewers ]
                for name in viewers_names:
                    try:
                        url = server_from_conf.viewersByName[ name ].url
                        path = server_from_conf.viewersByName[ name ].path
                    except KeyError:
                        custom_path = os.path.join( _cfg.mobylehome() , 'Local' , 'Services' , 'Viewers' , name +'.xml' )
                        if os.path.exists( custom_path ):
                            url = "file://%s" %custom_path
                            path = custom_path
                        else:
                            public_path = os.path.join( _cfg.mobylehome() , 'Services' , 'Viewers' , name +'.xml' )
                            url = "file://%s" %public_path
                            path = public_path
                    viewer = ViewerDef( name = name ,
                                        url = url ,
                                        path = path ,
                                        server = server
                                        )
                    args_registry.addViewer( viewer ) 
                    
                if tutorials_names == ['all']:
                    tutorials_names = [ t.name for t in server_from_conf.tutorials ]
                for name in tutorials_names:
                    try:
                        url = server_from_conf.tutorialsByName[ name ].url
                        path = server_from_conf.tutorialsByName[ name ].path
                    except KeyError:
                        custom_path = os.path.join( _cfg.mobylehome() , 'Local' , 'Services' , 'Tutorials' , name +'.xml' )
                        if os.path.exists( custom_path ):
                            url = "file://%s" %custom_path
                            path = custom_path
                        else:
                            public_path = os.path.join( _cfg.mobylehome() , 'Services' , 'Tutorials' , name +'.xml' )
                            url = "file://%s" %public_path
                            path = public_path
                    tutorial = TutorialDef( name = name ,
                                        url = url ,
                                        path = path ,
                                        server = server
                                        )
                    args_registry.addTutorial( tutorial )   
        return args_registry


    def get_registry_from_config(self ):
        """
        build a registry based on the configuration ( as described by 
        LOCAL_DEPLOY_INCLUDE , LOCAL_DEPLOY_EXCLUDE and PORTALS )
        @return: a registry builded from the configuartion
        @rtype: L{Mobyle.Registry} instance
        """
        config_reg = Registry()
        customPrefix = os.path.join ( _cfg.mobylehome() , "Local" , "Services" )
        publicPrefix = os.path.join ( _cfg.mobylehome() , "Services" )
        
        def _include( services , service_type ):
            """
            @param services: a container to store service name and path
            @type services: dict
            @param service_type: the type of the service (programs, workflows, viewer ,... )
            @type service_type: string
            @return: the list of local (from this server) service file paths include in deploy
            @rtype: list of strings
            """
            for mask in _cfg.services_deployment_include( service_type ):
                for path in _uniq( glob( os.path.join( publicPrefix , service_type[0].upper() + service_type[1:] , mask + '.xml' )) , 
                                        glob( os.path.join( customPrefix , service_type[0].upper() + service_type[1:] ,mask + '.xml' ))
                                        ):
                    services[ os.path.basename( path )[:-4] ] = path
            return services
            
        def _exclude( services , service_type ):
            """
            @param services: a container to store service name and path
            @type services: dict
            @param service_type: the type of the service (programs, workflows, viewer ,... )
            @type service_type: string
            @return: the list of local (from this server) service file paths to exclude from deploy
            @rtype: list of strings
            """
            for mask in _cfg.services_deployment_exclude( service_type ):
                for path in _uniq( glob( os.path.join( publicPrefix , service_type[0].upper() + service_type[1:], mask + '.xml' )) ,
                                        glob( os.path.join( customPrefix , service_type[0].upper() + service_type[1:] , mask + '.xml' ))
                                        ):
                    try:
                        del( services[ os.path.basename( path )[:-4] ] )
                    except KeyError:
                        pass
            return services         
        
        def _uniq( publicPath , customPath ):
            """
            @param publicPath: the list of service definition in the MOBYLEHOME/Services/xx
            @type publicPath: list of strings
            @param customPath: the list of service definition in the MOBYLEHOME/Local/Services/xx
            @type customPath: list of strings
            @return: a list containing one xml uri per service with the priority to the custom xml   
            @rtype: list of string
            """
            result = {}
            for path in publicPath:
                result[ os.path.basename( path )[:-4] ] = path
            for path in customPath:
                result[ os.path.basename( path )[:-4] ] = path
            return result.values()
        
        local_server = ServerDef( name = 'local',
                                  url = _cfg.cgi_url() ,
                                  help = 'foo@bar',
                                  repository = _cfg.repository_url() ,
                                  jobsBase = _cfg.results_url()
                                  )
        config_reg.addServer(local_server)
        
        #for method in _cfg.services_deployment_order():
        #    getattr( self.get_registry_from_config , '_' + method )( local_services )
        #    eval( '_%s( local_services )' %method ) 
        
        ##############################
        #
        # local services
        #
        ################################
        local_programs = {}    
        local_programs = _include( local_programs , 'programs' )
        local_programs = _exclude( local_programs , 'programs' )
        #program    
        for program_name , program_path in local_programs.items():
            program = ProgramDef( name = program_name,
                                  url = "file://%s" %program_path ,
                                  path = program_path,
                                  server = local_server
                                )  
            config_reg.addProgram( program )    
        
        #workflow
        local_workflows = {}
        local_workflows = _include( local_workflows , 'workflows' )
        local_workflows = _exclude( local_workflows , 'workflows' )
        
        for workflow_name , workflow_path in local_workflows.items():
            workflow = WorkflowDef( name = workflow_name,
                                  url = "file://%s" %workflow_path ,
                                  path = workflow_path,
                                  server = local_server
                                )  
            config_reg.addWorkflow( workflow )   
        
        #viewer  
        local_viewers = {}
        local_viewers = _include( local_viewers , 'viewers' )
        local_viewers = _exclude( local_viewers , 'viewers' )
        for viewer_name , viewer_path in local_viewers.items():
            viewer = ViewerDef( name = viewer_name,
                                 url = "file://%s" %viewer_path ,
                                  path = viewer_path,
                                  server = local_server
                                )  
            config_reg.addViewer( viewer )   
       
        #tutorials  
        local_tutorials = {}
        local_tutorials = _include( local_tutorials , 'tutorials' )
        local_tutorials = _exclude( local_tutorials , 'tutorials' )
        for tutorial_name , tutorial_path in local_tutorials.items():
            tutorial = TutorialDef( name = tutorial_name,
                                  url = "file://%s" %tutorial_path ,
                                  path = tutorial_path,
                                  server = local_server
                                )  
            config_reg.addTutorial( tutorial ) 
        ##############################
        #
        # imported services
        #
        ################################    
        effective_servers_path =  _cfg.servers_path()   
        for server_name, properties in _cfg.portals().items():
            server = ServerDef( name = server_name,
                                url = properties['url'] ,
                                help = properties['help'],
                                repository = properties['repository'],
                                jobsBase = "%s/data/jobs" %properties['repository']
                               )
            config_reg.addServer(server)
            if 'services' in properties:
                #program
                if 'programs' in properties[ 'services' ]:
                    for prg_name in properties[ 'services' ][ 'programs' ]:
                        url = config_reg.getProgramUrl( prg_name, server.name )
                        program = ProgramDef( name = prg_name,
                                              url = url,
                                              path = os.path.join( effective_servers_path , server.name , ProgramDef.directory_basename , prg_name , '.xml') ,
                                              server = server
                                            )
                        config_reg.addProgram(program)
                #workflow    
                if 'workflows' in properties[ 'services' ]:
                    for wf_name in properties[ 'services' ][ 'workflows' ]:
                        url = config_reg.getWorkflowUrl( wf_name, server.name)
                        workflow = WorkflowDef( name = wf_name,
                                              url = url,
                                              path = os.path.join( effective_servers_path , server.name , WorkflowDef.directory_basename , wf_name , '.xml') ,
                                              server = server
                                            )
                        config_reg.addWorkflow( workflow )
        return config_reg
    
                   
        
    def _recoverFromExisting(self, args_registry, server_names, programs_name, workflows_names, viewers_name, tutorials_names):
        """
        based on the informations from the registry_from_existing recover the services definitions
        which are not specified on the command line and recover for the new deployement.
        @param args_registry: the registry build from the command line options.
        @type args_registry: a L{Mobyle.Registry} instance
        """
        for server in registry_from_existing.servers if not 'all' in server_names else ():
            if server.programs or server.workflows or server.viewers or server.tutorials :
                tmp_server_path = os.path.join( self.tmp_servers_dir , server.name )
                try:
                    if os.path.isdir( tmp_server_path ):
                        shutil.rmtree( tmp_server_path )
                    os.mkdir( tmp_server_path , 0775 )
                    if server.programs:
                        os.mkdir( os.path.join( tmp_server_path , ProgramDef.directory_basename ) , 0775 ) 
                    if server.workflows:
                        os.mkdir( os.path.join( tmp_server_path , WorkflowDef.directory_basename ) , 0775 ) 
                    if server.viewers:
                        os.mkdir( os.path.join( tmp_server_path , ViewerDef.directory_basename ) , 0775 ) 
                    if server.tutorials:
                        os.mkdir( os.path.join( tmp_server_path , TutorialDef.directory_basename ) , 0775 ) 
                    
                except (OSError , IOError ), err :
                    self.log.critical( "cannot create temporary directories : %s" %err )
                    sys.exit(1)
                if server.name in server_names:
                    programs_2_recover = server.programs if not 'all' in programs_name else ()
                else:
                    programs_2_recover = server.programs
                for program in programs_2_recover:
                    if not program in args_registry.programs:
                        try:
                            self._getOldXmlAsIs( program )
                        except Exception , err:
                            self.log.error("cannot retrieve xml corresponding to program %s.%s from the deployed version : %s" %(server.name,
                                                                                                                     program.name , 
                                                                                                                      err,
                                                                                                                      ) ,
                                                                                                                      exc_info = True
                            )
                if server.name in server_names:
                    workflows_2_recover = server.workflows if not 'all' in workflows_names else ()
                else:
                    workflows_2_recover = server.workflows
                for workflow in workflows_2_recover:
                    if not workflow in args_registry.workflows :
                        try:
                            self._getOldXmlAsIs( workflow )
                        except Exception , err:
                            self.log.error("cannot retrieve xml corresponding to workflow %s.%s from the deployed version  : %s" %(server.name,
                                                                                                                      workflow.name ,
                                                                                                                      err
                                                                                                                      ))
                if server.name in server_names:
                    viewers_2_recover = server.viewers if not 'all' in viewers_name else ()
                else:
                    viewers_2_recover = server.viewers
                for viewer in viewers_2_recover:            
                    if not viewer in args_registry.viewers :
                        try:
                            self._getOldXmlAsIs( viewer )
                        except Exception , err:
                            self.log.error("cannot retrieve xml corresponding to viewer %s.%s from the deployed version  : %s" %(server.name,
                                                                                                                      viewer.name ,
                                                                                                                      err
                                                                                                                      ))
                if server.name in server_names:
                    tutorials_2_recover = server.tutorials if not 'all' in tutorials_names else ()
                else:
                    tutorials_2_recover = server.tutorials
                for tutorial in tutorials_2_recover:            
                    if not tutorial in args_registry.tutorials :
                        try:
                            self._getOldXmlAsIs( tutorial )
                        except Exception , err:
                            self.log.error("cannot retrieve xml corresponding to tutorial %s.%s from the deployed version  : %s" %(server.name,
                                                                                                                      tutorial.name ,
                                                                                                                      err
                                                                                                                      ))
                
    def _do_cmd( self , cmd , cmd_registry , force = False ):
        """
        execute the command cmd on the services defined in the cmd_registry
        @param cmd: the command to execute on the services to prepare a new deployment:
          -deploy to deploy a service
          -clean to remove a service
        @type cmd: string 
        @param cmd_registry: the registry build based on the arguments specified on the command line
        @type cmd_registry: a L{Mobyle.Registry} instance
        @param force: force the execution of the command even the service  is not exported
        """
        for server in cmd_registry.servers:
            if server.name!='local':
                try:
                    req = urllib2.Request(server.url+"/net_services.py")
                    handle = urllib2.urlopen(req)
                    resp = handle.read()
                    server.services_availability_properties =  simplejson.loads(resp)
                except Exception, e:
                    self.log.error( "error retrieving server properties for server %s, no service will be imported from it" % server.name )
                    continue
            tmp_server_path = os.path.join( self.tmp_servers_dir , server.name )
            if not os.path.isdir( tmp_server_path ):
                try:
                    os.mkdir( tmp_server_path , 0775 )
                except (OSError , IOError ), err :
                    self.log.critical( "cannot create temporary directories %s : %s" %( tmp_server_path , err) )
                    sys.exit(1)
            if server.programs:    
                tmp_programs_path = os.path.join( tmp_server_path , ProgramDef.directory_basename )
                if not os.path.isdir( tmp_programs_path ):
                    try:
                        os.mkdir( tmp_programs_path , 0775 )
                    except (OSError , IOError ), err :
                        self.log.critical( "cannot create temporary directories %s : %s" %( tmp_programs_path , err) )
                        sys.exit(1)
            if server.workflows:    
                tmp_workflows_path = os.path.join( tmp_server_path , WorkflowDef.directory_basename )
                if not os.path.isdir( tmp_workflows_path ):
                    try:
                        os.mkdir( tmp_workflows_path , 0775 )
                    except (OSError , IOError ), err :
                        self.log.critical( "cannot create temporary directories %s : %s" %( tmp_workflows_path , err) )
                        sys.exit(1)  
            if server.viewers:    
                tmp_viewers_path = os.path.join( tmp_server_path , ViewerDef.directory_basename )
                if not os.path.isdir( tmp_viewers_path ):
                    try:
                        os.mkdir( tmp_viewers_path , 0775 ) 
                    except (OSError , IOError ), err :
                        self.log.critical( "cannot create temporary directories %s : %s" %( tmp_viewers_path , err) )
                        sys.exit(1)           

            if server.tutorials:    
                tmp_tutorials_path = os.path.join( tmp_server_path , TutorialDef.directory_basename )
                if not os.path.isdir( tmp_tutorials_path ):
                    try:
                        os.mkdir( tmp_tutorials_path , 0775 ) 
                    except (OSError , IOError ), err :
                        self.log.critical( "cannot create temporary directories %s : %s" %( tmp_tutorials_path , err) )
                        sys.exit(1) 
            for service in server.programs + server.workflows + server.viewers + server.tutorials  :
                if cmd == 'deploy':
                    tmp_service_path = os.path.join( tmp_server_path , service.directory_basename , service.name + ".xml" )
                    if not self.is_available( service ):
                        if server.name == 'local':
                            self.log.warning( "the server local does not have %s %s.  This service will not be deployed" %(service.type, service.name) )
                            continue
                        else:
                            if force:
                                self.log.warning( "the server %s does not export %s %s." %(server.name , service.type, service.name ) )
                            else:
                                self.log.warning( "the server %s does not export %s %s. This service will not be deployed" %(service.type,
                                                                                                                             server.name ,
                                                                                                                             service.name 
                                                                                                                             ) )
                                continue
                    if not self.config_registry.has_service(service) :
                        self.log.warning( "The %s %s.%s is not in the Config. When you want to clean this service, you must do it explicitly or add it to the Config." %( service.type,
                                                                                                                                                                          server.name ,
                                                                                                                                                                          service.name
                                                                                                                                                                       ))
                    try:
                        self.log.debug("copying %s to %s" % (service.url, tmp_service_path))
                        self.getXml( service.url , tmp_service_path )
                    except Exception , err :
                        self.log.error("the %s %s.%s cannot be retrieved: %s"%( service.type, server.name , service.name , err ), exc_info=True)
                        if registry_from_existing.has_service( service ):
                            try:
                                self._getOldXmlAsIs( registry_from_existing.serversByName[ service.server.name ].programsByName[ service.name ] )
                                self.log.error( "the %s %s.%s has been recovered from the already deployed version"%( service.type, server.name, service.name ))
                                continue
                            except Exception, err:
                                self.log.error( str(err) ,exc_info = True )
                                continue
                        else:
                            cmd_registry.pruneService( service )
                            self.log.error( "the %s %s.%s cannot be deployed "%( service.type, server.name, service.name ) ) 
                            continue 
                    
                    #validate the xml
                    try:
                        validate = self._validateOrDie( service )
                        if not(validate):
                            raise Exception("service validation failed")
                        if isinstance( service , ViewerDef ) or isinstance( service , TutorialDef ):
                            try:
                                src = service.path[:-4]
                                dst = os.path.join( tmp_server_path , service.directory_basename , service.name )
                                if os.path.exists(src):
                                    shutil.copytree( src , dst)
                                else:
                                    if isinstance( service , ViewerDef ):
                                        raise Exception( "there is no directory corresponding to viewer %s "%service.name )
                            except Exception , err:
                                self.log.error( "the archive corresponding to the %s %s cannot be retrieved : %s" %( service.type ,
                                                                                                                     service.name , 
                                                                                                                       err ))
                                #remove the new xml before to recover the old one
                                #because the xml is 0444 and it cannot be replace by the old one
                                os.unlink( tmp_service_path ) 
                                try:
                                    self._getOldXmlAsIs( registry_from_existing.serversByName[ service.server.name ].programsByName[ service.name ] )
                                    self.log.warning( "the %s %s.%s has been recovered from the already deployed version"%( service.type, server.name , service.name ),
                                                    exc_info = True )
                                    continue #the xml has been already processed
                                except Exception, err:
                                    #the message is already logged in _getOldXmlAsIs
                                    try:
                                        os.unlink( tmp_service_path )
                                    except:
                                        pass
                                    try:
                                        os.unlink( dst )
                                    except :
                                        pass
                                    continue
                    except Exception, err:
                        self.log.error( "the service %s.%s does not validate : %s"%( server.name , service.name , err ))
                        self.log.error( "the service %s.%s will not be deployed : %s"%( server.name , service.name , err ))
                        os.unlink( tmp_service_path )
                        if isinstance( service , ViewerDef ):
                            path = os.path.join( tmp_server_path , service.directory_basename , service.name )
                            shutil.rmtree( path )
                        if registry_from_existing.has_service( service ):
                            try:
                                self._getOldXmlAsIs( registry_from_existing.serversByName[ service.server.name ].programsByName[ service.name ] )
                                self.log.error( "the service %s.%s has been recovered from the already deployed version"%( server.name , service.name ))
                                continue
                            except Exception, err:
                                self.log.error( err )
                                continue
                        else:
                            cmd_registry.pruneService( service )
                            self.log.error( "the service %s.%s cannot be deployed "%( server.name , service.name ) ) 
                            continue 
                          
                    #process the xml
                    try:
                        self.log.debug('processing service at url %s' % service.url)
                        self.process_service( tmp_service_path )                        
                    except Exception , err :
                        self.log.error( "the %s %s.%s cannot be transformed : %s"%( service.type, server.name , service.name , err ), exc_info=True)
                        os.unlink( tmp_service_path )
                        if isinstance( service , ViewerDef ):
                            path = os.path.join( tmp_server_path , service.directory_basename , service.name )
                            shutil.rmtree( path )
                        if registry_from_existing.has_service(service):
                            try:
                                self._getOldXmlAsIs( registry_from_existing.serversByName[ service.server.name ].programsByName[ service.name ] )
                                self.log.error( "the %s %s.%s has been recovered from the already deployed version"%( service.type, server.name, service.name ))
                                continue
                            except Exception, err:
                                cmd_registry.pruneService( service )
                                self.log.error( "the %s %s.%s cannot be deployed "%( service.type, server.name , service.name ) ) 
                                continue
                        else:
                            cmd_registry.pruneService( service )
                            self.log.error( "the %s %s.%s cannot be deployed "%( service.type, server.name , service.name ) ) 
                            continue   
                                       
                elif cmd == 'clean':
                    cmd_registry.pruneService( service )
                    if self.config_registry.has_service( service ) :
                        self.log.warning( "the %s %s.%s is in the Config. If you want to delete  it permanently from your portal remove it from the Config." %( service.type, server.name , service.name) )
                        continue
                    
                    
            ###############################
            # clean the empty directories #
            ###############################
            tmp_viewers_path = os.path.join( tmp_server_path , ViewerDef.directory_basename )
            if os.path.isdir( tmp_viewers_path ) and not os.listdir( tmp_viewers_path ):
                try:
                    self.log.debug( "cleaning %s.viewers dir" %server.name )
                    os.rmdir( tmp_viewers_path )
                except (OSError , IOError ), err :
                    self.log.debug( "ERROR during cleaning %s.viewers : %s "%(server.name , err) )
            
            tmp_tutorials_path = os.path.join( tmp_server_path , ViewerDef.directory_basename )
            if os.path.isdir( tmp_tutorials_path ) and not os.listdir( tmp_tutorials_path ):
                try:
                    self.log.debug( "cleaning %s.tutorials dir" %server.name )
                    os.rmdir( tmp_tutorials_path )
                except (OSError , IOError ), err :
                    self.log.debug( "ERROR during cleaning %s.tutorials : %s "%(server.name , err) )        
            
            tmp_workflows_path = os.path.join( tmp_server_path , WorkflowDef.directory_basename )
            if os.path.isdir( tmp_workflows_path ) and not os.listdir( tmp_workflows_path ) :
                try:
                    self.log.debug( "cleaning %s.workflows dir" %server.name )
                    os.rmdir( tmp_workflows_path )
                except (OSError , IOError ), err :
                    self.log.debug( "ERROR during cleaning %s.workflows : %s "%(server.name , err) )
            
            tmp_programs_path = os.path.join( tmp_server_path , ProgramDef.directory_basename )
            if os.path.isdir( tmp_programs_path ) and not os.listdir( tmp_programs_path ) :
                try:
                    self.log.debug( "cleaning %s.programs dir" %server.name )
                    os.rmdir( tmp_programs_path )
                except (OSError , IOError ), err :
                    self.log.debug( "ERROR during cleaning %s.programs : %s "%(server.name , err) )
            
            if os.path.isdir( tmp_server_path ) and not os.listdir( tmp_server_path ) :
                try:
                    self.log.debug( "cleaning %s dir" %server.name )
                    os.rmdir( tmp_server_path )
                except (OSError , IOError ), err :
                    self.log.debug( "ERROR during cleaning %s : %s "%(server.name , err) )
                    
                    
                    
    def is_available(self , service ):
        """
        @param service:
        @type service:
        @return: 
        @rtype: boolean
        """
        if service.server.name == 'local':
            if os.path.exists( service.path ):
                return True
            else:
                return False
        else:
            ap = service.server.services_availability_properties.get(service.name)
            av = True
            if ap is None:
                self.log.error( "import error: service %s is not present on server %s, import cancelled."%(service.name , service.server.name) )
                av = False
            else:
                #remote service exists, continue
                if ap['disabled']:
                    self.log.warning( "import error: service %s is disabled on server %s, import cancelled."%(service.name , service.server.name) )
                    av = False
                #remote service enabled, continue
                if not(ap['authorized']):
                    self.log.warning( "import error: service %s is not authorized for you on server %s, import cancelled."%(service.name , service.server.name) )
                    av = False
                if not(ap['exported']):
                    self.log.warning( "import error: service %s is not exported on server %s, import cancelled."%(service.name , service.server.name) )
                    av = False
            return av
        
    def _getOldXmlAsIs(self , service ):
        """
        get the already deployed xml and copy it (hard link) in tmp directory 
        @param service: the service 
        @type service: a Registry.ProgramDef or Registry.WorkflowDef instance
        """
        self.log.info( "> RECOVER %s.%s definition from the deployed version" %( service.server.name , service.name ) )
        tmp_server_path = os.path.join( self.tmp_servers_dir , service.server.name )
        try:
            os.link(  service.path , os.path.join( tmp_server_path , service.directory_basename , service.name + ".xml" ) )
        except Exception, err:
            #if the src and dest are not on the same device
            #an OSError: [Errno 18] Invalid cross-device link , is raised
            self.log.debug("_getOldXmlAsIs os.link( %s , %s ) failed : %s"%( service.path ,
                                                                             os.path.join( tmp_server_path , service.directory_basename , service.name + ".xml" ),
                                                                             err))
            try:
                shutil.copy( service.path , os.path.join( tmp_server_path , service.directory_basename , service.name + ".xml" ) )
            except IOError ,err:
                msg = "can't copy service definition from %s to %s : %s" %( service.path ,
                                                              os.path.join( tmp_server_path , service.directory_basename , service.name + ".xml" ) ,
                                                              err )
                raise IOError( msg )
        
        if isinstance( service , ViewerDef ):
            src = service.path[:-4]
            dst = os.path.join( tmp_server_path , service.directory_basename , service.name )
            try:    
                shutil.copytree( src , dst)
            except Exception , err:
                try:
                    os.unlink( os.path.join( tmp_server_path , service.directory_basename , service.name + ".xml" ) )
                except:
                    pass
                raise Exception( "can't copy viewer directory from %s to %s: %s"%( src ,
                                                                                       dst ,
                                                                                       err))
                
        if isinstance( service , TutorialDef ):
            src = service.path[:-4]
            dst = os.path.join( tmp_server_path , service.directory_basename , service.name )
            if os.path.exists(src):
                try:    
                    shutil.copytree( src , dst)
                except Exception , err:
                    try:
                        os.unlink( os.path.join( tmp_server_path , service.directory_basename , service.name + ".xml" ) )
                    except:
                        pass
                    raise Exception( "can't copy viewer directory from %s to %s: %s"%( src ,
                                                                                           dst ,
                                                                                           err))
                                    
         
        
    def getXml(self , uri , dest_path ):
        """
        get the XML corresponding to uri , resolve the Xinclude and write the 
        resulting XML to dest_path
        @param uri: the uri of the xml 
        @type uri: string
        @param src_path: the path to write the xml after Xinclude resolution
        @type src_path: string
        """
        doc_tree = etree.parse( uri, parser=net_enabled_parser)
        doc_tree.xinclude()
        dest_file = open( dest_path, "w" )
        dest_file.write( etree.tostring( doc_tree ) )
        dest_file.close()
            
    def template(self, programpath):
        """
        Pre-process the xsl on program xml definitions
        @param programpath : xml path file to process
        @type programpath : string
        @return: if the processing run well or not
        @rtype: boolean
        """        
        doc = etree.parse(programpath)
        if doc.getroot().tag=='workflow':
            from Mobyle.Workflow import Parser
            from Mobyle.WorkflowLayout import layout
            p = Parser()
            w = p.parse(programpath)
            i_l_s = layout(w) # graph layout as an svg string
            i_l_e = etree.XML(i_l_s) # graph layout as an element
            g_i = w.find("head/interface[@type='graph']")
            if g_i is None:
                g_i = etree.SubElement(w.find('head'),'interface') # graph interface container element
            g_i.clear()
            g_i.set("type","graph")
            g_i.append(i_l_e)
            fileResult = open(programpath,"w")
            fileResult.write(etree.tostring(w, pretty_print=True))
            fileResult.close()
            doc = etree.parse(programpath)

        for style, params in self.XslPipe:
            for p in params.keys():
                if p == 'programUri':
                    params[p] = "'"+programpath+"'"
                    self.log.debug('programUri=%s' % programpath)
            result = style(doc, **params)
            doc = result

        #write the result on the xml itself
        try:
            fileResult = open(programpath,"w")
            fileResult.write(str(doc))
            fileResult.close()
            return True
        except IOError, ex:
            self.log.error("Problem with the html generation: %s." % ex)
            return False  
          
    def process_service(self , tmp_service_path ):
        """
        @param tmp_service_path: the absolute path to find the new service definition
        @type tmp_service_path:  ServiceDef instance
        """
        self.log.info( "processing %s" %tmp_service_path )
        self.tpl_preprocessor.process_interface(tmp_service_path)
        #self.template(tmp_service_path)
        os.chmod( tmp_service_path , 0444 )
    
    def _validateOrDie(self, service ):    
        """
        checks if the program on the specified path validates.
        logs validation errors
        if it does not validate, the file is removed
        Warning: validation should be done in the target path, to avoid file loss.
        @param path: path to the program file
        @type path: string
        @return: True if it validates, False otherwise
        @rtype: list of strings
        """
        self.log.info( "validating %s.%s (published on %s" % (service.server.name , service.name, service.url) )
        try:
            if service.path is not None:
                docPath=service.path
            else:
                docPath=service.url
            val = Validator(type = service.type, docPath = docPath, publicURI = service.url,runRNG_Jing=self.useJing)
            val.run()
            if not(val.valOk):
                self.log.error("Testing : Deployment of %s.%s ABORTED, it does NOT validate. If it exists, the previous xml version is temporary kept. Details follow:" % (service.server.name , 
                                                                                                                                                                           service.name) )
                if val.runRNG and val.rngOk == False:
                    self.log.error("* relax ng validation failed - errors detail:")
                    for re in val.rngErrors:
                        self.log.error(" - %s" % re)
                if val.runRNG_Jing and not(val.rng_JingOk):
                    self.log.error("* relax ng JING validation failed - errors detail:")
                    for line in val.rng_JingErrors:
                         self.log.error("- %s" % (line))
                if val.runSCH and len(val.schErrors) > 0:
                    self.log.error("* schematron validation failed - errors detail:")
                    for se in val.schErrors:
                        self.log.error(" - %s" % se)       
                return False
            else:
                # if it validates w.r.t. Validator rules, then try validating w.r.t the datatypes
                errors = self.dtv.validateDataTypes(docPath=docPath)
                #if len(errors)!=0:
                #    self.log.error("* datatypes validation failed - errors detail:")
                #    for error in errors:
                #        self.log.error(" - %s" % error)
                #    return False
            return True
        except Exception, err:
            self.log.error("the service %s on server %s does not validate(url=%s): %s" % ( service.name, service.server.name, service.url, err ), exc_info=True)
            return False    

    
    def make_indexes( self, registry ):
        """
        attention pour le moment j'ai enleve l'argument kind. a voir 
        dans quelles conditions on fait quoi 
        workflow, program , viewer ...
        generate index files corresponding to the deployed service
        @param kind: le type d'objet sur lequel faire les index ????
        @type kind: string
        """
        self.log.info("Regenerating indexes:")
        for kind in ["program","workflow","viewer", "tutorial"]:
            self.log.info("Regenerating search index for %s..." % kind)
            SearchIndex.generate(type=kind, dir=self.tmp_indexes, registry=registry)
            self.log.info("Regenerating classification index for %s..." % kind)
            ClassificationIndex.generate(type=kind, dir=self.tmp_indexes, registry=registry)
            self.log.info("Regenerating descriptions index for %s..." % kind)
            DescriptionsIndex.generate(type=kind, dir=self.tmp_indexes, registry=registry)
            self.log.info("Regenerating inputs index for %s..." % kind)
            DataInputsIndex.generate(type=kind, dir=self.tmp_indexes, registry=registry)
             
 
if __name__ == '__main__':
    from optparse import OptionParser, OptionGroup
    servers   = None
    programs  = None
    workflows = None
    viewers   = None
    tutorials = None
    
    usage = """usage: %prog [options] cmd
cmds:
    deploy   : deploy services
    clean    : remove services
    index    : build the indexes (deploy and clean imply index)

If no options are provided the command is applied 
on all services from all servers and all viewers.
    """
    parser = OptionParser( usage= usage )

    service_group = OptionGroup(parser, "services related options" )

    service_group.add_option("-p", "--programs",
                      action="store", 
                      type = 'string', 
                      dest="programs",
                      help="comma separated list of programs on which the command will be applied. The keyword 'all' can be used to install all programs defined in the Mobyle configuration. ")
    
    service_group.add_option("-w", "--workflows",
                      action="store", 
                      type = 'string', 
                      dest="workflows",
                      help="comma separated list of workflows on which the command will be applied. The keyword 'all' can be used to install all workflows defined in the Mobyle configuration. ")
    
    service_group.add_option("-s", "--servers",
                      action="store", 
                      type = 'string', 
                      dest="servers",
                      help="comma separated list of SERVERS on which the command will be applied. This option is meaningless with options -v -t. The keyword 'all' can be used to install all programs defined in the Mobyle configuration.")
    
    service_group.add_option("-f", "--force",
                      action="store_true", 
                      dest="force",
                      default= False ,
                      help="force the deployement of a service even the server does not export it. This option is reserved for debugging purpose.")
    parser.add_option_group(service_group)

    service_group.add_option("-j", "--jing",
                      action="store_true", 
                      dest="useJing",
                      default= False ,
                      help="Also validate using Jing, which should generate more helpful error messages. This option is reserved for debugging purpose.")
    parser.add_option_group(service_group)
    
    viewer_group = OptionGroup(parser, "viewers related options" )
    
    viewer_group.add_option("-v", "--viewers",
                      action="store", 
                      type = 'string', 
                      dest="viewers",
                      help="comma separated list of VIEWERS on which the command will be applied. The keyword 'all' can be used to install all viewers defined in the Mobyle configuration.")
    parser.add_option_group(viewer_group)

    tutorial_group = OptionGroup(parser, "tutorials related options" )
    service_group.add_option("-t", "--tutorials",
                      action="store", 
                      type = 'string', 
                      dest="tutorials",
                      help="comma separated list of TUTORIALS on which the command will be applied. The keyword 'all' can be used to install all viewers defined in the Mobyle configuration.")

    general_group = OptionGroup(parser, "general options" )
    general_group.add_option("-V", "--verbose",
                      action="count", 
                      dest="verbosity", 
                      default= 0,
                      help="increase the verbosity level. There is 3 levels: Warning (default) show Warning and Error messages , Info (-V) and Debug.(-VV)") 
    parser.add_option_group(general_group)     
    options, args = parser.parse_args()
    if len( args ) != 1:
        parser.print_help( sys.stderr )
        sys.exit(1)
    command = args[0]    
    
    
    if options.servers is None :
        options.servers   = 'all'
    if options.programs is None and options.workflows is None and options.viewers is None and options.tutorials is None:
        options.programs  = 'all'
        options.viewers   = 'all'
        options.workflows = 'all'
        options.tutorials = 'all'
        
    if options.servers is not None:    
        servers = options.servers.split( ',' )
        if 'all' in servers:
            servers = [ 'all' ]
    if options.programs is not None:    
        programs = options.programs.split( ',' )
        if 'all' in programs:
            programs =[ 'all' ]
    else:
        programs = []
    if options.workflows is not None:    
        workflows = options.workflows.split( ',' )
        if 'all' in workflows:
            workflows =[ 'all' ]  
    else:
        workflows = []
    if options.viewers is not None:
        viewers = options.viewers.split( ',' )
        if 'all' in viewers:
            viewers = [ 'all' ]
    else:
        viewers = []
        
    if options.tutorials is not None:
        tutorials = options.tutorials.split( ',' )
        if 'all' in tutorials:
            tutorials = [ 'all' ]
    else:
        tutorials = []
    deployer = ServicesDeployer( verbosity = options.verbosity, useJing=options.useJing )          
    deployer.doCommand(command , servers, programs, workflows, viewers, tutorials, force= options.force )