1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
#!/usr/bin/python
"""Diff a repo (downstream) and its upstream.
This script:
1. Downloads a repo source tree with specified manifest URL, branch
and release tag.
2. Retrieves the BUILD_ID from $downstream/build/core/build_id.mk.
3. Downloads the upstream using the BUILD_ID.
4. Diffs each project in these two repos.
"""
import argparse
import datetime
import os
import subprocess
import repo_diff_trees
HELP_MSG = "Diff a repo (downstream) and its upstream"
DOWNSTREAM_WORKSPACE = "downstream"
UPSTREAM_WORKSPACE = "upstream"
DEFAULT_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
DEFAULT_MANIFEST_BRANCH = "android-8.0.0_r10"
DEFAULT_UPSTREAM_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
DEFAULT_UPSTREAM_MANIFEST_BRANCH = "android-8.0.0_r1"
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "android_exclusions.txt")
def parse_args():
"""Parse args."""
parser = argparse.ArgumentParser(description=HELP_MSG)
parser.add_argument("-u", "--manifest-url",
help="manifest url",
default=DEFAULT_MANIFEST_URL)
parser.add_argument("-b", "--manifest-branch",
help="manifest branch",
default=DEFAULT_MANIFEST_BRANCH)
parser.add_argument("-r", "--upstream-manifest-url",
help="upstream manifest url",
default=DEFAULT_UPSTREAM_MANIFEST_URL)
parser.add_argument("-a", "--upstream-manifest-branch",
help="upstream manifest branch",
default=DEFAULT_UPSTREAM_MANIFEST_BRANCH)
parser.add_argument("-e", "--exclusions-file",
help="exclusions file",
default=DEFAULT_EXCLUSIONS_FILE)
parser.add_argument("-t", "--tag",
help="release tag (optional). If not set then will "
"sync the latest in the branch.")
parser.add_argument("-i", "--ignore_error_during_sync",
action="store_true",
help="repo sync might fail due to varios reasons. "
"Ignore these errors and move on. Use with caution.")
return parser.parse_args()
def repo_init(url, rev, workspace):
"""Repo init with specific url and rev.
Args:
url: manifest url
rev: manifest branch, or rev
workspace: the folder to init and sync code
"""
try:
subprocess.check_output("repo", stderr=subprocess.PIPE,
cwd=os.path.dirname(workspace), shell=True)
except subprocess.CalledProcessError:
pass
else:
raise ValueError("cannot repo-init workspace (%s), workspace is within an "
"existing tree" % workspace)
print("repo init:\n url: %s\n rev: %s\n workspace: %s" %
(url, rev, workspace))
subprocess.check_output("repo init --manifest-url=%s --manifest-branch=%s" %
(url, rev), cwd=workspace, shell=True)
def repo_sync(workspace, ignore_error, retry=5):
"""Repo sync."""
count = 0
while count < retry:
count += 1
print("repo sync (retry=%d/%d):\n workspace: %s" %
(count, retry, workspace))
try:
command = "repo sync --jobs=24 --current-branch --quiet"
command += " --no-tags --no-clone-bundle"
if ignore_error:
command += " --force-broken"
subprocess.check_output(command, cwd=workspace, shell=True)
except subprocess.CalledProcessError as e:
print "Error: %s" % e.output
if count == retry and not ignore_error:
raise e
# Stop retrying if the repo sync was successful
else:
break
def get_commit_with_keyword(project_path, keyword):
"""Get the latest commit in $project_path with the specific keyword."""
return subprocess.check_output(("git -C %s "
"rev-list --max-count=1 --grep=\"%s\" "
"HEAD") %
(project_path, keyword), shell=True).rstrip()
def get_build_id(workspace):
"""Get BUILD_ID defined in $workspace/build/core/build_id.mk."""
path = os.path.join(workspace, "build", "core", "build_id.mk")
return subprocess.check_output("source %s && echo $BUILD_ID" % path,
shell=True).rstrip()
def repo_sync_specific_release(url, branch, tag, workspace, ignore_error):
"""Repo sync source with the specific release tag."""
if not os.path.exists(workspace):
os.makedirs(workspace)
manifest_path = os.path.join(workspace, ".repo", "manifests")
repo_init(url, branch, workspace)
if tag:
rev = get_commit_with_keyword(manifest_path, tag)
if not rev:
raise(ValueError("could not find a manifest revision for tag " + tag))
repo_init(url, rev, workspace)
repo_sync(workspace, ignore_error)
def diff(manifest_url, manifest_branch, tag,
upstream_manifest_url, upstream_manifest_branch,
exclusions_file, ignore_error_during_sync):
"""Syncs and diffs an Android workspace against an upstream workspace."""
workspace = os.path.abspath(DOWNSTREAM_WORKSPACE)
upstream_workspace = os.path.abspath(UPSTREAM_WORKSPACE)
# repo sync downstream source tree
repo_sync_specific_release(
manifest_url,
manifest_branch,
tag,
workspace,
ignore_error_during_sync)
build_id = None
if tag:
# get the build_id so that we know which rev of upstream we need
build_id = get_build_id(workspace)
if not build_id:
raise(ValueError("Error: could not find the Build ID of " + workspace))
# repo sync upstream source tree
repo_sync_specific_release(
upstream_manifest_url,
upstream_manifest_branch,
build_id,
upstream_workspace,
ignore_error_during_sync)
# make output folder
if tag:
output_folder = os.path.abspath(tag.replace(" ", "_"))
else:
current_time = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
output_folder = os.path.abspath(current_time)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# do the comparison
repo_diff_trees.diff(
upstream_workspace,
workspace,
os.path.join(output_folder, "project.csv"),
os.path.join(output_folder, "commit.csv"),
os.path.abspath(exclusions_file),
)
def main():
args = parse_args()
diff(args.manifest_url,
args.manifest_branch,
args.tag,
args.upstream_manifest_url,
args.upstream_manifest_branch,
args.exclusions_file,
args.ignore_error_during_sync)
if __name__ == "__main__":
main()
|
