File: ruth.py

package info (click to toggle)
sanlock 2.2-2
  • links: PTS
  • area: main
  • in suites: jessie, jessie-kfreebsd, wheezy
  • size: 1,124 kB
  • ctags: 1,917
  • sloc: ansic: 12,971; python: 896; sh: 440; makefile: 227
file content (359 lines) | stat: -rwxr-xr-x 11,824 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
#!/usr/bin/python
# Copyright 2009 Red Hat, Inc. and/or its affiliates.
#
# Licensed to you under the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version. See the files README and
# LICENSE_GPL_v2 which accompany this distribution.
#

import sys
from optparse import OptionParser
import os
from ConfigParser import ConfigParser
import logging
from copy import deepcopy
import unittest as ut

import confUtils
from confUtils import Validate
from testRunner import TestRunner
#To use the same instance as everyone else
import ruth

LOG_LEVELS = {'d': logging.DEBUG,
              'i': logging.INFO,
              'w': logging.WARNING,
              'e': logging.ERROR,
              'c': logging.CRITICAL,
              'debug': logging.DEBUG,
              'info': logging.INFO,
              'warning': logging.WARNING,
              'error': logging.ERROR,
              'critical': logging.CRITICAL}

DEFAULT_CONFIG_PATH = "~/.ruthrc"

USAGE = '''usage: %%prog [options] [conf1 [conf2 [...] ] ]
Loads the configuration from '%s', unless other 'config' files are specified
in command line. For more help read \"README.1st\".''' % (DEFAULT_CONFIG_PATH)

CONFIG_TEMPLATE = {
    "global" : {
        "verbose" : {"default" : 1, "validator" : Validate.int},
        "modules" : {"validator" : Validate.list},
    }
}

def generateTemplateFromSuite(suite):
    """
    Generate a config template from a test suit.
    """
    templates = []
    for testCase in suite:
        if hasattr(testCase, "getConfigTemplate"):
            templates.append(testCase.getConfigTemplate())

    return confUtils.mergeTemplates(templates)

def validateSuiteConfig(suite, cfg):
    """
    Validate a config against a suite. Validates that all the test cases in the suite have
    all the options they need in the configuration file.

    To be used by most ruth modules.

    :returns: a touple of ``(result, message)``.
    """
    masterTemplate = generateTemplateFromSuite(suite)
    cfg = _expandGlobalCfg(masterTemplate, cfg)
    return confUtils.validateConfigFile(masterTemplate, cfg)

def _expandGlobalCfg(template, cfg):
    """
    Distribute the options defined in 'global' to all the sections.

    :returns: a new config file with the global section distributed
              to all the sections defined in the template.
    """
    #Work on a copy
    cfg = deepcopy(cfg)

    #Do we even need to work
    if not cfg.has_section("global"):
        return cfg

    for section in template:
        if not cfg.has_section(section):
            cfg.add_section(section)
        for option in template[section]:
            if not cfg.has_option("global", option):
                continue

            globalValue = cfg.get("global", option)
            cfg.set(section, option, globalValue)

    return cfg

class RuthTestCase(ut.TestCase):
    mycfg = property(lambda self: self.cfg[self.__class__.__name__])
    def _getConfig(self):
        """
        Manages the configuration wrapper of a test case
        """
        a = self.__class__
        if not hasattr(self, "_confDict"):
            template = self.getConfigTemplate()
            expandedCfg = _expandGlobalCfg(template, self._cfg)
            confDict = confUtils.conf2dict(template, expandedCfg)
            setattr(self, "_confDict", confDict)
        return self._confDict

    def _getLog(self):
        if (not hasattr(self, "_log")) or self._log == None:
            self._log = logging.getLogger("test." + self.id())

        return self._log

    def _setLog(self, value):
        self._log = value

    log = property(lambda self: self._getLog(), lambda self, value: self._setLog(value))

    #Dynamic because the base class get the conf directly from Ruth.
    #If it were static everyone would have the basic classes wrapper.
    cfg = property(lambda self : self._getConfig())

    @classmethod
    def getConfigTemplate(cls):
        """
        Returns a config template that announces what the
        test case expects from the config file.

        .. note::
            Should be overrided by subclasses.
        """
        return {}

    @classmethod
    def setConfig(cls, cfg):
        cls._cfg = cfg

    def setUp(self):
        pass

    def tearDown(self):
        pass


def parseArguments():
    """
    Prepares the options parser and parses the cmd line args.
    """
    usage = USAGE
    parser = OptionParser(usage=usage)

    #Prepare generation configuration option
    parser.add_option("-g", "--generate-configuration",
        action="store", dest="moduleToGenerate", type="string", metavar="MODULE", default=None,
        help="Instead of running the suite. Creates a sample configuration from MODULE)")

    #prepare quiet option
    parser.add_option("-q", "--quiet",
        action="store_true", dest="quiet", default=False,
        help="Should I bother you with unnecessary niceties. (Hello message and end quote).")

    #prepare verbose option
    parser.add_option("-v",
        action="count", dest="verbosity", default=0,
        help="Override configurations' verbose level.")

   #prepare filter option
    parser.add_option("-f", "--filter-tests",
        action="store", dest="filter", default="*", metavar="GLOB",
        help="Only tests that match this glob filter will run." + \
             "Using '^' in the beginning of the glob means: Match opposite of GLOB.")

    #prepare debug option
    parser.add_option("-d", "--debug",
        action="store_true", dest="debug", default=False,
        help="Should I print a lot of output in case of internal errors.")

    #prepare log option
    parser.add_option("-l", "--logging",
        action="store", dest="logLevel", default=None, metavar="LEVEL",
        help="Turn on test logging of the level LEVEL")

    #parse args
    options, args = parser.parse_args()
    if len(args) == 0:
        args = [DEFAULT_CONFIG_PATH]

    return (options, args)

def generateSampleConfigFile(defaultModule, suite, targetFile):
    #Generate template
    template = generateTemplateFromSuite(suite)

    #Add default module
    if not "global" in template:
        template["global"] = {}
    globalSection = template["global"]

    if not "modules" in globalSection:
        globalSection["modules"] = {}

    template["global"]["modules"]["default"] = defaultModule

    #Write it all to disk
    confUtils.generateSampleConfigFile(template, targetFile)

def handleSampleConfigFileGeneration(moduleToGenerate, targetFile):
    """
    Takes care of sample config generation.

    :param moduleToGenerate: The name of the python module.
                             Should be the same as in and :keyword:`import` statement.
    :param targetFile: The path to where to sample config file will be generated.

    :returns: **0** if successful, **400** on import error or **500** on config generation error.
    """
    #Import module
    try:
        print "Importing module '%s'..." % (moduleToGenerate)
        suiteModule = __import__(moduleToGenerate)
    except Exception, ex:
        print "Could not import module '%s'. (%s)" % (moduleToGenerate, ex)
        return 400

    #Get suite and generate config file
    try:
        print "Generating sample config file at '%s'..." % (targetFile)
        generateSampleConfigFile(moduleToGenerate, suiteModule.suite(), targetFile)
    except Exception, ex:
        print "Could not generate sample config file from module '%s'. (%s: %s)" % (moduleToGenerate, ex.__class__.__name__, ex)
        return 500

    return 0

def _printHeader(header, marker="="):
    sep = marker * len(header)
    print sep
    print header
    print sep

def runBatch(confFile, options):
    """
    Run a batch test as stated in a config file.
    """
    # Try to load config file
    mycfg = {}
    batchcfg = {}
    output = sys.stdout
    try:
        output.write("Validating configuration file '%s'.\n" % (os.path.split(confFile)[1]))
        output.flush()
        confUtils.validateConfigFile(CONFIG_TEMPLATE, confFile)
        output.write("Loading RUTH configuration.\n")
        batchcfg = ConfigParser()
        batchcfg.read(confFile)
        mycfg = confUtils.conf2dict(CONFIG_TEMPLATE, batchcfg)
    except Exception, ex:
        raise Exception("Could not load config file '%s'. Bailing out from batch. (%s: %s)" % (confFile, ex.__class__.__name__, ex))

    #Get modules to test
    modules = mycfg["global"]["modules"]
    output.write("Running tests from modules: %s.\n" % (", ".join(modules)))
    output.flush()

    #test modules
    batch = {}
    for mod in modules[:]:
        #import module
        imported_module = __import__(mod)

        try:
            if hasattr(imported_module, "validateConfig"):
                imported_module.validateConfig(batchcfg)
            else:
                validateSuiteConfig(imported_module.suite(), batchcfg)
            batch[mod] = imported_module
            output.write("Module '%s' is READY\n" % (mod))
        except Exception, ex:
            output.write("Module '%s' is NOT READY (%s: %s)\n" % (mod, ex.__class__.__name__, ex))
            modules.remove(mod)
    #set configuration
    ruth.RuthTestCase.setConfig(batchcfg)

    results = []
    #run tests
    for mod in batch:
        output.write("Exercising module '%s'\n" % mod)
        output.flush()
        suite = batch[mod].suite()
        verbose = mycfg["global"]["verbose"]

        if options.verbosity > 0:
            verbose = options.verbosity

        logging = True
        if options.logLevel is None:
            logging = False

        results.append(TestRunner(verbosity=verbose, stream=output, filter=options.filter, logging=logging).run(suite))

    return results

def main():
    hello = """
    Hello, nice to meet you. I am RUTH - "Regression and Unit Test Harness".
    I am going to run a comprehensive test suite in order to validate vdsm
    functionality. However, I may require some assistance from you in order
    to correctly bootstrap the whole procedure.
    Use --help to see what you can do with me.
    """

    options, args = parseArguments()
    if options.logLevel is None:
        logging.basicConfig(filename='/dev/null')
    else:
        if not options.logLevel in LOG_LEVELS:
            print "Invalid logging level, possible values are %s." % ", ".join(options.keys())
            return

        logging.basicConfig(filename='/dev/stdout', filemode='w+',level=LOG_LEVELS[options.logLevel],
            format="\t\t%(asctime)s %(levelname)-8s%(message)s", datefmt='%H:%M:%S')

    if not options.quiet:
        print hello

    if options.moduleToGenerate:
        return handleSampleConfigFileGeneration(options.moduleToGenerate, args[0])

    #iterate config files and run their tests
    configFiles = args
    i = 0
    results = []
    isMultipleConfigMode = len(configFiles) > 1
    for confFile in configFiles:
        i += 1
        if isMultipleConfigMode:
            _printHeader("Processing batch %d of %d. Configuration is '%s'." % (i, len(configFiles), os.path.split(confFile)[1]))
        try:
            results.extend(runBatch(os.path.expanduser(confFile), options))
        except Exception, ex:
            if options.debug:
                import traceback
                print traceback.format_exc()
            print ex

    if isMultipleConfigMode:
        totalFailures = sum([len(result.failures) for result in results])
        totalErrors = sum([len(result.errors) for result in results])
        _printHeader("Totals: Failures %d, Errors %d." % (totalFailures, totalErrors))

    if not options.quiet:
        print 'All Done!\nremember:\n\t"To Err is Human, To Test is Divine!"'

if __name__ == '__main__':
    main()