File: talos_from_code.py

package info (click to toggle)
firefox-esr 60.9.0esr-1~deb9u1
  • links: PTS, VCS
  • area: main
  • in suites: stretch
  • size: 2,721,864 kB
  • sloc: cpp: 5,255,014; ansic: 2,334,861; python: 586,390; asm: 342,462; java: 329,373; xml: 194,330; sh: 83,420; makefile: 22,793; perl: 16,226; cs: 4,725; objc: 3,768; yacc: 1,766; ada: 1,681; lex: 1,364; pascal: 1,264; exp: 527; php: 436; lisp: 258; ruby: 157; awk: 152; sed: 53; csh: 27
file content (131 lines) | stat: -rw-r--r-- 4,523 bytes parent folder | download | duplicates (4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
#! /usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

#
# Script name:   talos_from_code.py
# Purpose:       Read from a talos.json file the different files to download for a talos job
# Author(s):     Zambrano Gasparnian, Armen <armenzg@mozilla.com>
# Target:        Python 2.5
#
from __future__ import absolute_import, print_function

import json
import os
import re
import sys
import urllib2
import urlparse
from optparse import OptionParser


def main():
    '''
    This script downloads a talos.json file which indicates which files to download
    for a talos job.
    See a talos.json file for a better understand:
    http://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
    '''
    parser = OptionParser()
    parser.add_option("--talos-json-url", dest="talos_json_url", type="string",
                      help="It indicates from where to download the talos.json file.")
    (options, args) = parser.parse_args()

    # 1) check that the url was passed
    if options.talos_json_url is None:
        print("You need to specify --talos-json-url.")
        sys.exit(1)

    # 2) try to download the talos.json file
    try:
        jsonFilename = download_file(options.talos_json_url)
    except Exception as e:
        print("ERROR: We tried to download the talos.json file but something failed.")
        print("ERROR: %s" % str(e))
        sys.exit(1)

    # 3) download the necessary files
    print("INFO: talos.json URL: %s" % options.talos_json_url)
    try:
        key = 'talos.zip'
        entity = get_value(jsonFilename, key)
        if passesRestrictions(options.talos_json_url, entity["url"]):
            # the key is at the same time the filename e.g. talos.zip
            print("INFO: Downloading %s as %s" %
                  (entity["url"], os.path.join(entity["path"], key)))
            download_file(entity["url"], entity["path"], key)
        else:
            print("ERROR: You have tried to download a file " +
                  "from: %s " % entity["url"] +
                  "which is a location different than http://talos-bundles.pvt.build.mozilla.org/")
            print("ERROR: This is only allowed for the certain branches.")
            sys.exit(1)
    except Exception as e:
        print("ERROR: %s" % str(e))
        sys.exit(1)


def passesRestrictions(talosJsonUrl, fileUrl):
    '''
    Only certain branches are exempted from having to host their downloadable files
    in talos-bundles.pvt.build.mozilla.org
    '''
    if talosJsonUrl.startswith("http://hg.mozilla.org/try/") or \
       talosJsonUrl.startswith("https://hg.mozilla.org/try/") or \
       talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/") or \
       talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/") or \
       talosJsonUrl.startswith("http://hg.mozilla.org/projects/ash/") or \
       talosJsonUrl.startswith("https://hg.mozilla.org/projects/ash/"):
        return True
    else:
        p = re.compile('^http://talos-bundles.pvt.build.mozilla.org/')
        m = p.match(fileUrl)
        if m is None:
            return False
        return True


def get_filename_from_url(url):
    '''
    This returns the filename of the file we're trying to download
    '''
    parsed = urlparse.urlsplit(url.rstrip('/'))
    if parsed.path != '':
        return parsed.path.rsplit('/', 1)[-1]
    else:
        print("ERROR: We were trying to download a file from %s " +
              "but the URL seems to be incorrect.")
        sys.exit(1)


def download_file(url, path="", saveAs=None):
    '''
    It downloads a file from URL to the indicated path
    '''
    req = urllib2.Request(url)
    f = urllib2.urlopen(req)
    if path != "" and not os.path.isdir(path):
        try:
            os.makedirs(path)
            print("INFO: directory %s created" % path)
        except Exception as e:
            print("ERROR: %s" % str(e))
            sys.exit(1)
    filename = saveAs if saveAs else get_filename_from_url(url)
    local_file = open(os.path.join(path, filename), 'wb')
    local_file.write(f.read())
    local_file.close()
    return filename


def get_value(json_filename, key):
    '''
    It loads up a JSON file and returns the value for the given string
    '''
    f = open(json_filename, 'r')
    return json.load(f)[key]


if __name__ == '__main__':
    main()