1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
#!/usr/bin/env vpython3
# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import os
import sys
from core import path_util
path_util.AddTelemetryToPath()
from core import benchmark_finders
from telemetry.core import optparse_argparse_migration as oam
from py_utils import cloud_storage
path_util.AddAndroidPylibToPath()
def GetAllStorySets():
story_sets = []
benchmarks_to_skip = [
'analysis_metrics_ct',
'skpicture_printer_ct',
'screenshot_ct',
'rendering.cluster_telemetry',
'repaint_ct',
'rasterize_and_record_micro_ct',
'multipage_skpicture_printer_ct',
'loading.cluster_telemetry',
'v8.loading.cluster_telemetry',
'v8.loading_runtime_stats.cluster_telemetry',
'memory.cluster_telemetry',
'skpicture_printer',
'multipage_skpicture_printer',
'leak_detection.cluster_telemetry',
'generic_trace_ct',
'ad_tagging.cluster_telemetry',
'layout_shift.cluster_telemetry',
]
for benchmark in benchmark_finders.GetAllBenchmarks():
if benchmark.Name() in benchmarks_to_skip:
continue
parser = oam.CreateFromOptparseInputs()
benchmark.AddBenchmarkCommandLineArgs(parser)
options, _ = parser.parse_args([])
story_sets.append(benchmark().CreateStorySet(options))
return story_sets
def NormalizedPath(p):
return os.path.normpath(os.path.abspath(p))
def GetMissingArchivesInCloudStorage(archive_infos, wpr_sha_files):
if wpr_sha_files:
abs_wpr_sha_files_path = []
for f in wpr_sha_files:
assert os.path.exists(f), '% does not exist' % f
abs_wpr_sha_files_path.append(NormalizedPath(f))
wpr_sha_files = abs_wpr_sha_files_path
cloud_storage_paths = set()
missing_sha_files = set()
for wpr_archive_info in archive_infos:
bucket = wpr_archive_info._bucket
story_archives = wpr_archive_info._data['archives']
for story in story_archives:
for _, archive_path in story_archives[story].items():
archive_path = os.path.join(wpr_archive_info._base_dir,
archive_path)
hash_path = NormalizedPath(archive_path + '.sha1')
if not os.path.exists(hash_path):
missing_sha_files.add(hash_path)
continue
if hash_path not in wpr_sha_files:
continue
remote_path = cloud_storage.ReadHash(hash_path)
cloud_storage_paths.add((bucket, remote_path, archive_path))
buckets = {}
# Create a dictionary of all of the buckets.
for (bucket, _, _) in cloud_storage_paths:
buckets[bucket] = {}
# Some buckets contain a modest (< 10,000) number of entries and can be
# processed more efficiently by listing the contents once rather than calling
# .Exists() for each file. Other buckets contain 600,000+ (chromium-telemetry)
# or far more files (chrome-telemetry) so .Exists() must be used for them.
fast_buckets = ['chrome-partner-telemetry']
# Iterate through all of the fast buckets getting a list of all files in those
# buckets.
for bucket in buckets:
if bucket in fast_buckets:
buckets[bucket] = set(cloud_storage.ListFiles(bucket))
missing_archives = set()
for (bucket, remote_path, archive_path) in cloud_storage_paths:
found = False
if bucket in fast_buckets:
if '/' + remote_path in buckets[bucket]:
found = True
elif cloud_storage.Exists(bucket, remote_path):
found = True
if not found:
missing_archives.add((archive_path, bucket))
return missing_archives, missing_sha_files
def main(args):
parser = argparse.ArgumentParser(
'Validate whether WPR archives are properly stored in CloudStorage.')
parser.add_argument('wpr_sha_files', nargs='*')
options = parser.parse_args(args)
archive_infos = []
for s in GetAllStorySets():
if not s.wpr_archive_info:
continue
archive_infos.append(s.wpr_archive_info)
missing_archives, missing_sha_files = GetMissingArchivesInCloudStorage(
archive_infos, options.wpr_sha_files)
assert not missing_archives, (
'Archives not checked in cloud storage properly:\n%s' %
'\n'.join('%s (expected bucket: %s)' % (p, b) for p, b in missing_archives))
assert not missing_sha_files, (
'These SHA files are missing. Did you forget to check them in?\n%s' %
'\n'.join(missing_sha_files))
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
|
