1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
import argparse
import json
import os
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Dict, List
from tools.stats.upload_stats_lib import (
download_gha_artifacts,
download_s3_artifacts,
unzip,
upload_to_rockset,
)
def get_sccache_stats(
workflow_run_id: int, workflow_run_attempt: int
) -> List[Dict[str, Any]]:
with TemporaryDirectory() as temp_dir:
print("Using temporary directory:", temp_dir)
os.chdir(temp_dir)
# Download and extract all the reports (both GHA and S3)
download_s3_artifacts("sccache-stats", workflow_run_id, workflow_run_attempt)
artifact_paths = download_gha_artifacts(
"sccache-stats", workflow_run_id, workflow_run_attempt
)
for path in artifact_paths:
unzip(path)
stats_jsons = []
for json_file in Path(".").glob("**/*.json"):
with open(json_file) as f:
stats_jsons.append(json.load(f))
return stats_jsons
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Upload test stats to Rockset")
parser.add_argument(
"--workflow-run-id",
type=int,
required=True,
help="id of the workflow to get artifacts from",
)
parser.add_argument(
"--workflow-run-attempt",
type=int,
required=True,
help="which retry of the workflow this is",
)
args = parser.parse_args()
stats = get_sccache_stats(args.workflow_run_id, args.workflow_run_attempt)
upload_to_rockset("sccache_stats", stats)
|
