# -*- python -*-
# ex: set syntax=python:

from buildbot.buildslave import BuildSlave
from buildbot.changes.pb import PBChangeSource
from buildbot.scheduler import AnyBranchScheduler, Triggerable
from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter
from buildbot.schedulers.filter import ChangeFilter
from buildbot.status import html
from buildbot.status.web.authz import Authz
from buildbot.process import buildstep, factory, properties
from buildbot.steps import master, shell, source, transfer, trigger
from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED

from twisted.internet import defer

import os
import re
import json
import operator
import urllib

from committer_auth import CommitterAuth
import wkbuild


c = BuildmasterConfig = {}

c['change_source'] = PBChangeSource()

# permissions for WebStatus
authz = Authz(
    auth=CommitterAuth('auth.json'),
    forceBuild='auth',
    forceAllBuilds='auth',
    pingBuilder=True,
    gracefulShutdown=False,
    stopBuild='auth',
    stopAllBuilds='auth',
    cancelPendingBuild='auth',
    stopChange=True,
    cleanShutdown=False)

c['status'] = []
c['status'].append(html.WebStatus(http_port=8710,
                                  revlink="http://trac.webkit.org/changeset/%s", 
                                  changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
                                  authz=authz))

c['slavePortnum'] = 17000
c['projectName'] = "WebKit"
c['projectURL'] = "http://webkit.org"
c['buildbotURL'] = "http://build.webkit.org/"

c['buildHorizon'] = 1000
c['logHorizon'] = 500
c['eventHorizon'] = 200
c['buildCacheSize'] = 60

WithProperties = properties.WithProperties


class TestWithFailureCount(shell.Test):
    failedTestsFormatString = "%d tests failed"

    def countFailures(self, cmd):
        return 0

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)
        self.failedTestCount = self.countFailures(cmd)

    def evaluateCommand(self, cmd):
        if self.failedTestCount:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.failedTestCount:
            return [self.failedTestsFormatString % self.failedTestCount]

        return [self.name]


class ConfigureBuild(buildstep.BuildStep):
    name = "configure build"
    description = ["configuring build"]
    descriptionDone = ["configured build"]
    def __init__(self, platform, configuration, architecture, buildOnly, SVNMirror, *args, **kwargs):
        buildstep.BuildStep.__init__(self, *args, **kwargs)
        self.platform = platform.split('-', 1)[0]
        self.fullPlatform = platform
        self.configuration = configuration
        self.architecture = architecture
        self.buildOnly = buildOnly
        self.SVNMirror = SVNMirror
        self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, SVNMirror=SVNMirror)

    def start(self):
        self.setProperty("platform", self.platform)
        self.setProperty("fullPlatform", self.fullPlatform)
        self.setProperty("configuration", self.configuration)
        self.setProperty("architecture", self.architecture)
        self.setProperty("buildOnly", self.buildOnly)
        self.setProperty("shouldAbortEarly", True)
        self.setProperty("SVNMirror", self.SVNMirror)
        self.finished(SUCCESS)
        return defer.succeed(None)


class CheckOutSource(source.SVN):
    mode = "update"
    def __init__(self, SVNMirror, **kwargs):
        kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
        kwargs['defaultBranch'] = "trunk"
        kwargs['mode'] = self.mode
        source.SVN.__init__(self, **kwargs)
        self.addFactoryArguments(SVNMirror=SVNMirror)

class WaitForSVNServer(shell.ShellCommand):
    name = "wait-for-svn-server"
    command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
    description = ["waiting for SVN server"]
    descriptionDone = ["SVN server is ready"]
    haltOnFailure = True

class InstallWin32Dependencies(shell.Compile):
    description = ["installing dependencies"]
    descriptionDone = ["installed dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]

class KillOldProcesses(shell.Compile):
    name = "kill old processes"
    description = ["killing old processes"]
    descriptionDone = ["killed old processes"]
    command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]

class DeleteStaleBuildFiles(shell.Compile):
    name = "delete stale build files"
    description = ["deleting stale build files"]
    descriptionDone = ["delete stale build files"]
    command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]

class InstallEflDependencies(shell.ShellCommand):
    name = "jhbuild"
    description = ["updating efl dependencies"]
    descriptionDone = ["updated efl dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkitefl-libs"]
    haltOnFailure = True

class InstallGtkDependencies(shell.ShellCommand):
    name = "jhbuild"
    description = ["updating gtk dependencies"]
    descriptionDone = ["updated gtk dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
    haltOnFailure = True

def appendCustomBuildFlags(step, platform, fullPlatform=""): 
    if platform in ('efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'): 
        step.setCommand(step.command + ['--' + platform])

class CompileWebKit(shell.Compile):
    command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
    env = {'MFLAGS':''}
    name = "compile-webkit"
    description = ["compiling"]
    descriptionDone = ["compiled"]
    warningPattern = ".*arning: .*"

    def start(self):
        platform = self.getProperty('platform')
        buildOnly = self.getProperty('buildOnly')
        architecture = self.getProperty('architecture')
        if platform == 'mac' and architecture == 'i386':
            self.setCommand(self.command + ['--32-bit'])
        if platform == 'mac' and buildOnly:
            self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])

        appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))

        return shell.Compile.start(self)


class CompileWebKit1Only(CompileWebKit):
    command = ["perl", "./Tools/Scripts/build-webkit", "--no-webkit2", WithProperties("--%(configuration)s")]


class CompileWebKit2Only(CompileWebKit):
    command = ["perl", "./Tools/Scripts/build-webkit", "--no-webkit1", WithProperties("--%(configuration)s")]


class ArchiveBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
               WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
    name = "archive-built-product"
    description = ["archiving built product"]
    descriptionDone = ["archived built product"]
    haltOnFailure = True


class ExtractBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
               WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
    name = "extract-built-product"
    description = ["extracting built product"]
    descriptionDone = ["extracted built product"]
    haltOnFailure = True


class UploadBuiltProduct(transfer.FileUpload):
    slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
    masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
    haltOnFailure = True

    def __init__(self, **kwargs):
        kwargs['slavesrc'] = self.slavesrc
        kwargs['masterdest'] = self.masterdest
        kwargs['mode'] = 0644
        transfer.FileUpload.__init__(self, **kwargs)


class DownloadBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
        WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
        WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
    name = "download-built-product"
    description = ["downloading built product"]
    descriptionDone = ["downloaded built product"]
    haltOnFailure = True
    flunkOnFailure = True


class RunJavaScriptCoreTests(shell.Test):
    name = "jscore-test"
    description = ["jscore-tests running"]
    descriptionDone = ["jscore-tests"]
    command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
    logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}

    def __init__(self, buildJSCTool=True, *args, **kwargs):
        self.buildJSCTool = buildJSCTool
        shell.Test.__init__(self, *args, **kwargs)
        self.addFactoryArguments(buildJSCTool=buildJSCTool)

    def start(self):
        appendCustomBuildFlags(self, self.getProperty('platform'))
        if not self.buildJSCTool:
            self.setCommand(self.command + ['--no-build'])
        return shell.Test.start(self)

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
        if statusLines and statusLines[0].split()[0] != '0':
            self.regressionLine = statusLines[0]
        else:
            self.regressionLine = None

        if 'actual.html (source)' in cmd.logs:
            self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())

    def evaluateCommand(self, cmd):
        if self.regressionLine:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.regressionLine:
            return [self.name, self.regressionLine]

        return [self.name]


class RunWebKitTests(shell.Test):
    name = "layout-test"
    description = ["layout-tests running"]
    descriptionDone = ["layout-tests"]
    command = ["perl", "./Tools/Scripts/run-webkit-tests",
               "--no-show-results",
               "--no-new-test-results",
               "--no-sample-on-timeout",
               "--results-directory", "layout-test-results",
               "--builder-name", WithProperties("%(buildername)s"),
               "--build-number", WithProperties("%(buildnumber)s"),
               "--master-name", "webkit.org",
               "--test-results-server", "webkit-test-results.appspot.com",
               WithProperties("--%(configuration)s")]

    def __init__(self, buildJSCTool=True, *args, **kwargs):
        self.buildJSCTool = buildJSCTool
        shell.Test.__init__(self, *args, **kwargs)
        self.addFactoryArguments(buildJSCTool=buildJSCTool)

    def start(self):
        platform = self.getProperty('platform')
        shouldAbortEarly = self.getProperty('shouldAbortEarly')
        appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
        if platform.startswith('mac'):
            self.setCommand(self.command + ['--no-build'])
        if shouldAbortEarly:
            self.setCommand(self.command + ["--exit-after-n-crashes-or-timeouts", "50", "--exit-after-n-failures", "500"])
        if platform == "win":
            rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")]
            self.setCommand(self.command + ['--no-build'])
        else:
            rootArgument = ['--root=WebKitBuild/bin']
        if not self.buildJSCTool:
            self.setCommand(self.command + rootArgument)
        return shell.Test.start(self)

    def _parseOldRunWebKitTestsOutput(self, logText):
        incorrectLayoutLines = []
        for line in logText.splitlines():
            if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
                incorrectLayoutLines.append(line)
            elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
                incorrectLayoutLines.append(line)
            elif line.startswith("WARNING:") and line.find(' leak') >= 0:
                incorrectLayoutLines.append(line.replace('WARNING: ', ''))
            elif line.find('Exiting early') >= 0:
                incorrectLayoutLines.append(line)

            # FIXME: Detect and summarize leaks of RefCounted objects

        self.incorrectLayoutLines = incorrectLayoutLines

    # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
    nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')

    def _strip_python_logging_prefix(self, line):
        match_object = self.nrwt_log_message_regexp.match(line)
        if match_object:
            return match_object.group('message')
        return line

    def _parseNewRunWebKitTestsOutput(self, logText):
        incorrectLayoutLines = []
        expressions = [
            ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
            ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
            ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
            ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
        ]
        testFailures = {}

        for line in logText.splitlines():
            if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
                incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
                continue
            for name, expression in expressions:
                match = expression.search(line)

                if match:
                    testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
                    break

                # FIXME: Parse file names and put them in results

        for name in testFailures:
            incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)

        self.incorrectLayoutLines = incorrectLayoutLines

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        if logText.find("Collecting tests ...") >= 0:
            self._parseNewRunWebKitTestsOutput(logText)
        else:
            self._parseOldRunWebKitTestsOutput(logText)

    def evaluateCommand(self, cmd):
        result = SUCCESS

        if self.incorrectLayoutLines:
            if len(self.incorrectLayoutLines) == 1:
                line = self.incorrectLayoutLines[0]
                if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
                    return WARNINGS

            for line in self.incorrectLayoutLines:
                if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
                    result = WARNINGS
                else:
                    return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return result

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.incorrectLayoutLines:
            return self.incorrectLayoutLines

        return [self.name]


class RunUnitTests(TestWithFailureCount):
    name = "run-api-tests"
    description = ["unit tests running"]
    descriptionDone = ["unit-tests"]
    command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
    failedTestsFormatString = "%d unit tests failed or timed out"

    def start(self):
        platform = self.getProperty('fullPlatform')
        if platform.startswith('win'):
            self.setCommand(self.command + ['--no-build'])
        if platform.startswith('mac'):
            self.setCommand(self.command + ['--no-build'])
        return shell.Test.start(self)

    def countFailures(self, cmd):
        log_text = cmd.logs['stdio'].getText()
        count = 0

        split = re.split(r'\sTests that timed out:\s', log_text)
        if len(split) > 1:
            count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))

        split = re.split(r'\sTests that failed:\s', split[0])
        if len(split) > 1:
            count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))

        return count


class RunPythonTests(TestWithFailureCount):
    name = "webkitpy-test"
    description = ["python-tests running"]
    descriptionDone = ["python-tests"]
    command = ["python", "./Tools/Scripts/test-webkitpy"]
    failedTestsFormatString = "%d python tests failed"

    def start(self):
        platform = self.getProperty('platform')
        # Python tests are flaky on the GTK builders, running them serially
        # helps and does not significantly prolong the cycle time.
        if platform == 'gtk':
            self.setCommand(self.command + ['--child-processes', '1'])
        # Python tests fail on windows bots when running more than one child process
        # https://bugs.webkit.org/show_bug.cgi?id=97465
        if platform == 'win':
            self.setCommand(self.command + ['--child-processes', '1'])
        return shell.Test.start(self)

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
        regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
        return 0


class RunPerlTests(TestWithFailureCount):
    name = "webkitperl-test"
    description = ["perl-tests running"]
    descriptionDone = ["perl-tests"]
    command = ["perl", "./Tools/Scripts/test-webkitperl"]
    failedTestsFormatString = "%d perl tests failed"

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
        regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return int(match.group('count'))
        return 0


class RunBindingsTests(shell.Test):
    name = "bindings-generation-tests"
    description = ["bindings-tests running"]
    descriptionDone = ["bindings-tests"]
    command = ["python", "./Tools/Scripts/run-bindings-tests"]


class RunEflAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]
    command = ["perl", "./Tools/Scripts/run-efl-tests", WithProperties("--%(configuration)s")]


class RunGtkAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]
    command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()

        self.incorrectTests = 0
        self.timedOutTests = 0
        self.skippedTests = 0
        self.statusLine = []

        foundItems = re.findall("Tests failed \((\d+)\):", logText)
        if (foundItems):
            self.incorrectTests = int(foundItems[0])

        foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
        if (foundItems):
            self.timedOutTests = int(foundItems[0])

        foundItems = re.findall("Tests skipped \((\d+)\):", logText)
        if (foundItems):
            self.skippedTests = int(foundItems[0])

        self.totalFailedTests = self.incorrectTests + self.timedOutTests

        if self.totalFailedTests > 0:
            self.statusLine = [
                "%d API tests failed, %d timed out, %d skipped" %
                (self.incorrectTests, self.timedOutTests, self.skippedTests)
            ]

    def evaluateCommand(self, cmd):
        if self.totalFailedTests > 0:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.totalFailedTests > 0:
            return self.statusLine

        return [self.name]

class RunQtAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]

    def start(self):
        self.setCommand( ["python", "./Tools/Scripts/run-qtwebkit-tests", "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120", "--%s" % self.getProperty('configuration')] )
        if self.getProperty('fullPlatform').endswith("-wk2"):
            self.setCommand(self.command + ['--webkit2'])
        return shell.Test.start(self)

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped, (?P<crashed>\d+) crashed", logText)

        self.incorrectTests = 0
        self.crashedTests = 0
        self.statusLine = []

        if foundItems:
            self.incorrectTests = int(foundItems[0][1])
            self.crashedTests = int(foundItems[0][3])

            if self.incorrectTests > 0 or self.crashedTests > 0:
                self.statusLine = [
                    "%s passed, %s failed, %s skipped, %s crashed" % (foundItems[0][0], foundItems[0][1], foundItems[0][2], foundItems[0][3])
                ]

    def evaluateCommand(self, cmd):
        if self.crashedTests:
            return FAILURE

        if re.findall("Timeout, process", cmd.logs['stdio'].getText()):
            self.statusLine = ["Failure: timeout occured during testing"]
            return FAILURE

        if self.incorrectTests:
            return WARNINGS

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.incorrectTests:
            return self.statusLine

        return [self.name]

class RunWebKitLeakTests(RunWebKitTests):
    warnOnWarnings = True
    def start(self):
        self.setCommand(self.command + ["--leaks"])
        return RunWebKitTests.start(self)


class RunWebKit2Tests(RunWebKitTests):
    def start(self):
        self.setProperty("shouldAbortEarly", False)
        self.setCommand(self.command + ["--webkit-test-runner"])
        if self.getProperty('buildername') == "Qt Mountain Lion Release":
            self.setCommand(self.command + ["-p"])

        return RunWebKitTests.start(self)


class RunAndUploadPerfTests(shell.Test):
   name = "perf-test"
   description = ["perf-tests running"]
   descriptionDone = ["perf-tests"]
   command = ["python", "./Tools/Scripts/run-perf-tests",
              "--output-json-path", "perf-test-results.json",
              "--slave-config-json-path", "../../perf-test-config.json",
              "--no-show-results",
              "--reset-results",
              "--test-results-server", "perf.webkit.org",
              "--builder-name", WithProperties("%(buildername)s"),
              "--build-number", WithProperties("%(buildnumber)s"),
              "--platform", WithProperties("%(fullPlatform)s"),
              WithProperties("--%(configuration)s")]

   def start(self):
       self.setCommand(self.command)
       return shell.Test.start(self)


class RunAndUploadPerfTestsWebKit2(RunAndUploadPerfTests):
    def start(self):
        self.setCommand(self.command + ["--webkit-test-runner"])
        return RunAndUploadPerfTests.start(self)


class ArchiveTestResults(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
    name = "archive-test-results"
    description = ["archiving test results"]
    descriptionDone = ["archived test results"]
    haltOnFailure = True


class UploadTestResults(transfer.FileUpload):
    slavesrc = "layout-test-results.zip"
    masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")

    def __init__(self, **kwargs):
        kwargs['slavesrc'] = self.slavesrc
        kwargs['masterdest'] = self.masterdest
        kwargs['mode'] = 0644
        transfer.FileUpload.__init__(self, **kwargs)


class ExtractTestResults(master.MasterShellCommand):
    zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
    resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
    descriptionDone = ["uploaded results"]

    def __init__(self, **kwargs):
        kwargs['command'] = ""
        master.MasterShellCommand.__init__(self, **kwargs)

    def resultDirectoryURL(self):
        return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"

    def start(self):
        self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
        return master.MasterShellCommand.start(self)

    def addCustomURLs(self):
        url = self.resultDirectoryURL() + "results.html"
        self.addURL("view results", url)

    def finished(self, result):
        self.addCustomURLs()
        return master.MasterShellCommand.finished(self, result)


class ExtractTestResultsAndLeaks(ExtractTestResults):
    def addCustomURLs(self):
        ExtractTestResults.addCustomURLs(self)
        url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
        self.addURL("view leaks", url)


class Factory(factory.BuildFactory):
    def __init__(self, platform, configuration, architectures, buildOnly, SVNMirror):
        factory.BuildFactory.__init__(self)
        self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, SVNMirror=SVNMirror))
        if SVNMirror:
            self.addStep(WaitForSVNServer())
        self.addStep(CheckOutSource(SVNMirror=SVNMirror))
        # There are multiple Qt slaves running on same machines, so buildslaves shouldn't kill the processes of other slaves.
        if not platform.startswith("qt"):
            self.addStep(KillOldProcesses())
        self.addStep(DeleteStaleBuildFiles())
        if platform == "win":
            self.addStep(InstallWin32Dependencies())
        if platform == "gtk":
            self.addStep(InstallGtkDependencies())
        if platform == "efl":
            self.addStep(InstallEflDependencies())


class BuildFactory(Factory):
    def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, True, SVNMirror)
        self.addStep(CompileWebKit())
        if triggers:
            self.addStep(ArchiveBuiltProduct())
            self.addStep(UploadBuiltProduct())
            self.addStep(trigger.Trigger(schedulerNames=triggers))

class BuildAndAPITestFactory(Factory):
    def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, True, SVNMirror)
        self.addStep(CompileWebKit())
        if triggers:
            self.addStep(ArchiveBuiltProduct())
            self.addStep(UploadBuiltProduct())
            self.addStep(trigger.Trigger(schedulerNames=triggers))
        if platform == "efl":
            self.addStep(RunEflAPITests)
        if platform == "gtk":
            self.addStep(RunGtkAPITests())
        if platform.startswith("qt"):
            self.addStep(RunQtAPITests)

def unitTestsSupported(configuration, platform):
    if platform.startswith('mac') and configuration == "release":
        return False; # https://bugs.webkit.org/show_bug.cgi?id=82652
    return platform == 'win' or platform.startswith('mac')

def pickLatestBuild(builder, requests):
    return max(requests, key=operator.attrgetter("submittedAt"))

class TestFactory(Factory):
    TestClass = RunWebKitTests
    ExtractTestResultsClass = ExtractTestResults
    def __init__(self, platform, configuration, architectures, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror)
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        self.addStep(RunJavaScriptCoreTests(buildJSCTool=False))
        self.addStep(self.TestClass(buildJSCTool=(platform != 'win')))

        if unitTestsSupported(configuration, platform): 
            self.addStep(RunUnitTests())
        self.addStep(RunPythonTests())
        self.addStep(RunPerlTests())
        self.addStep(RunBindingsTests())
        self.addStep(ArchiveTestResults())
        self.addStep(UploadTestResults())
        self.addStep(self.ExtractTestResultsClass())
        if platform == "efl":
            self.addStep(RunEflAPITests)
        if platform == "gtk":
            self.addStep(RunGtkAPITests())
        if platform.startswith("qt"):
            self.addStep(RunQtAPITests)

class BuildAndTestFactory(Factory):
    CompileClass = CompileWebKit
    TestClass = RunWebKitTests
    ExtractTestResultsClass = ExtractTestResults
    def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        self.addStep(self.CompileClass())
        self.addStep(RunJavaScriptCoreTests())
        self.addStep(self.TestClass())
        if unitTestsSupported(configuration, platform): 
            self.addStep(RunUnitTests())
        self.addStep(RunPythonTests())
        self.addStep(RunPerlTests())
        self.addStep(RunBindingsTests())
        self.addStep(ArchiveTestResults())
        self.addStep(UploadTestResults())
        self.addStep(self.ExtractTestResultsClass())
        if platform == "efl":
            self.addStep(RunEflAPITests())
        if platform == "gtk":
            self.addStep(RunGtkAPITests())
        if platform.startswith("qt"):
            self.addStep(RunQtAPITests())
        if triggers:
            self.addStep(ArchiveBuiltProduct())
            self.addStep(UploadBuiltProduct())
            self.addStep(trigger.Trigger(schedulerNames=triggers))

class BuildAndTestWebKit2Factory(BuildAndTestFactory):
    CompileClass = CompileWebKit
    TestClass = RunWebKit2Tests

class BuildAndTestWebKit1OnlyFactory(BuildAndTestFactory):
    CompileClass = CompileWebKit1Only

class BuildAndTestWebKit2OnlyFactory(BuildAndTestFactory):
    CompileClass = CompileWebKit2Only
    TestClass = RunWebKit2Tests

class TestLeaksFactory(TestFactory):
    TestClass = RunWebKitLeakTests
    ExtractTestResultsClass = ExtractTestResultsAndLeaks


class TestWebKit2Factory(TestFactory):
    TestClass = RunWebKit2Tests

class BuildAndPerfTestFactory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        self.addStep(CompileWebKit())
        self.addStep(RunAndUploadPerfTests())

class BuildAndPerfTestWebKit2Factory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        self.addStep(CompileWebKit())
        self.addStep(RunAndUploadPerfTestsWebKit2())

class DownloadAndPerfTestFactory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        self.addStep(RunAndUploadPerfTests())

class DownloadAndPerfTestWebKit2Factory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        self.addStep(RunAndUploadPerfTestsWebKit2())

class PlatformSpecificScheduler(AnyBranchScheduler):
    def __init__(self, platform, branch, **kwargs):
        self.platform = platform
        filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
        AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)

    def filter(self, change):
        return wkbuild.should_build(self.platform, change.files)

trunk_filter = ChangeFilter(branch=["trunk", None])

def loadBuilderConfig(c):
    # FIXME: These file handles are leaked.
    passwords = json.load(open('passwords.json'))
    config = json.load(open('config.json'))

    c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]

    c['schedulers'] = []
    for scheduler in config['schedulers']:
        if "change_filter" in scheduler:
            scheduler["change_filter"] = globals()[scheduler["change_filter"]]
        kls = globals()[scheduler.pop('type')]
        # Python 2.6 can't handle unicode keys as keyword arguments:
        # http://bugs.python.org/issue2646.  Modern versions of json return
        # unicode strings from json.load, so we map all keys to str objects.
        scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))

        c['schedulers'].append(kls(**scheduler))

    forceScheduler = ForceScheduler(
        name="force",
        builderNames=[builder['name'] for builder in config['builders']],
        reason=StringParameter(name="reason", default="", size=40),

        # Validate SVN revision: number or emtpy string
        revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),

        # Disable default enabled input fields: branch, repository, project, additional properties
        branch=FixedParameter(name="branch"),
        repository=FixedParameter(name="repository"),
        project=FixedParameter(name="project"),
        properties=[]
    )
    c['schedulers'].append(forceScheduler)

    c['builders'] = []
    for builder in config['builders']:
        for slaveName in builder['slavenames']:
            for slave in config['slaves']:
                if slave['name'] != slaveName or slave['platform'] == '*':
                    continue

                if slave['platform'] != builder['platform']:
                    raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])

                break

        platform = builder['platform']

        builderType = builder.pop('type')
        factory = globals()["%sFactory" % builderType]
        factorykwargs = {}
        for key in "platform", "configuration", "architectures", "triggers", "SVNMirror":
            value = builder.pop(key, None)
            if value:
                factorykwargs[key] = value

        builder["factory"] = factory(**factorykwargs)

        if platform.startswith('mac'):
            builder["category"] = 'AppleMac'
        elif platform == 'win':
            builder["category"] = 'AppleWin'
        elif platform.startswith('gtk'):
            builder["category"] = 'GTK'
        elif platform.startswith('qt'):
            builder["category"] = 'Qt'
        elif platform.startswith('efl'):
            builder["category"] = "EFL"
        else:
            builder["category"] = 'misc'

        if (builder['category'] == 'AppleMac' or builder['category'] == 'AppleWin') and builderType != 'Build':
            builder['nextBuild'] = pickLatestBuild

        c['builders'].append(builder)

loadBuilderConfig(c)