#!/usr/bin/env python """ test2.py [--log_file PATH] [--verbose] """ ################################################################################ # # test2 # # # Copyright (c) 7/16/2010 Leo Goodstadt # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. ################################################################################# import sys, os # add self to search path for testing if __name__ == '__main__': exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0] module_name = os.path.split(sys.argv[0])[1] module_name = os.path.splitext(module_name)[0]; else: module_name = __name__ # Use import path from <<../python_modules>> if __name__ == '__main__': sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"../.."))) #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 # options #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 if __name__ == '__main__': from optparse import OptionParser import StringIO parser = OptionParser(version="%prog 1.0", usage = "\n\n %progs [options]") parser.add_option("-i", "--input_file", dest="input_file", metavar="FILE", type="string", help="Name and path of input file. " "Defaults to reading from STDIN.") # # general options: verbosity / logging # parser.add_option("-v", "--verbose", dest = "verbose", action="count", default=0, help="Print more verbose messages for each additional verbose level.") parser.add_option("-L", "--log_file", dest="log_file", metavar="FILE", type="string", help="Name and path of log file") parser.add_option("--skip_parameter_logging", dest="skip_parameter_logging", action="store_true", default=False, help="Do not print program parameters to log.") parser.add_option("--debug", dest="debug", action="count", default=0, help="Set default program parameters in debugging mode.") # # pipeline # parser.add_option("-t", "--target_tasks", dest="target_tasks", action="append", default = list(), metavar="JOBNAME", type="string", help="Target task(s) of pipeline.") parser.add_option("-j", "--jobs", dest="jobs", default=1, metavar="N", type="int", help="Allow N jobs (commands) to run simultaneously.") parser.add_option("-n", "--just_print", dest="just_print", action="store_true", default=False, help="Don't actually run any commands; just print the pipeline.") parser.add_option("--flowchart", dest="flowchart", metavar="FILE", type="string", help="Don't actually run any commands; just print the pipeline " "as a flowchart.") # # Less common pipeline options # parser.add_option("--key_legend_in_graph", dest="key_legend_in_graph", action="store_true", default=False, help="Print out legend and key for dependency graph.") parser.add_option("--draw_graph_horizontally", dest="draw_horizontally", action="store_true", default=False, help="Draw horizontal dependency graph.") parser.add_option("--forced_tasks", dest="forced_tasks", action="append", default = list(), metavar="JOBNAME", type="string", help="Pipeline task(s) which will be included even if they are up to date.") # get help string f =StringIO.StringIO() parser.print_help(f) helpstr = f.getvalue() original_args = " ".join(sys.argv) (options, remaining_args) = parser.parse_args() #vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv # # # Debug: Change these # # # #^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ options.flowchart = "front_page_flowchart.png" options.key_legend_in_graph = True if options.debug: options.log_file = os.path.join("test2.log") options.verbose = 5 options.log_parameters = True #vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv # # # Debug: Change these # # # #^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # mandatory options # mandatory_options = [] def check_mandatory_options (options, mandatory_options, helpstr): """ Check if specified mandatory options have b een defined """ missing_options = [] for o in mandatory_options: if not getattr(options, o): missing_options.append("--" + o) if not len(missing_options): return raise Exception("Missing mandatory parameter%s: %s.\n\n%s\n\n" % ("s" if len(missing_options) > 1 else "", ", ".join(missing_options), helpstr)) check_mandatory_options (options, mandatory_options, helpstr) #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 # imports #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 from ruffus import * from ruffus.ruffus_exceptions import JobSignalledBreak #from json import dumps #from collections import defaultdict #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 # Functions #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 # Logger #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 if __name__ == '__main__': import logging import logging.handlers MESSAGE = 15 logging.addLevelName(MESSAGE, "MESSAGE") def setup_std_logging (logger, log_file, verbose): """ set up logging using programme options """ class debug_filter(logging.Filter): """ Ignore INFO messages """ def filter(self, record): return logging.INFO != record.levelno class NullHandler(logging.Handler): """ for when there is no logging """ def emit(self, record): pass # We are interesting in all messages logger.setLevel(logging.DEBUG) has_handler = False # log to file if that is specified if log_file: handler = logging.FileHandler(log_file, delay=False) handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)6s - %(message)s")) handler.setLevel(MESSAGE) logger.addHandler(handler) has_handler = True # log to stderr if verbose if verbose: stderrhandler = logging.StreamHandler(sys.stderr) stderrhandler.setFormatter(logging.Formatter(" %(message)s")) stderrhandler.setLevel(logging.DEBUG) if log_file: stderrhandler.addFilter(debug_filter()) logger.addHandler(stderrhandler) has_handler = True # no logging if not has_handler: logger.addHandler(NullHandler()) # # set up log # logger = logging.getLogger(module_name) setup_std_logging(logger, options.log_file, options.verbose) # # Allow logging across Ruffus pipeline # def get_logger (logger_name, args): return logger from ruffus.proxy_logger import * (logger_proxy, logging_mutex) = make_shared_logger_and_proxy (get_logger, module_name, {}) # # log programme parameters # if not options.skip_parameter_logging: programme_name = os.path.split(sys.argv[0])[1] logger.info("%s %s" % (programme_name, original_args)) #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 # Pipeline #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 @files(None, "a.1") def task1(input_file, output_file): open(output_file, "w") @transform(task1, suffix("1"), "2") def task2(input_file, output_file): open(output_file, "w") @transform(task2, suffix("2"), "3") def task3(input_file, output_file): open(output_file, "w") @transform(task3, suffix("3"), "4") def task4(input_file, output_file): open(output_file, "w") #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 # Main logic #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 import time open("a.2", "w") time.sleep(1) open("a.1", "w") time.sleep(1) open("a.3", "w") pipeline_printout_graph ( open(options.flowchart, "w"), os.path.splitext(options.flowchart)[1][1:], [task4], no_key_legend = not options.key_legend_in_graph, user_colour_scheme = {"colour_scheme_index":0}, pipeline_name = "Pipeline Flowchart:", size = (6,5), dpi = 72, ) os.unlink("a.1") os.unlink("a.2") os.unlink("a.3") #pipeline_run(options.target_tasks, options.forced_tasks, # multiprocess = options.jobs, # logger = stderr_logger, # verbose = options.verbose)