1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
|
#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Firebase Test Lab (Flank) test runner script for Taskcluster
# This script is used to run UI tests on Firebase Test Lab using Flank
# It requires a service account key file to authenticate with Firebase Test Lab
# It also requires the `gcloud` command line tool to be installed and configured
# Lastly it requires the `flank.jar` file to be present in the `test-tools` directory set up in the task definition
# The service account key file is stored in the `secrets` section of the task definition
# Flank: https://flank.github.io/flank/
import argparse
import logging
import os
import subprocess
import sys
from enum import Enum
from pathlib import Path
from typing import Optional, Union
from urllib.parse import urlparse
# Worker paths and binaries
class Worker(Enum):
JAVA_BIN = "/usr/bin/java"
FLANK_BIN = "/builds/worker/test-tools/flank.jar"
RESULTS_DIR = "/builds/worker/artifacts/results"
# Locate other scripts and configs relative to this script. The actual
# invocation of Flank will be relative to ANDROID_TEST path below.
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TOPSRCDIR = os.path.join(SCRIPT_DIR, "../../..")
ANDROID_TEST = os.path.join(TOPSRCDIR, "mobile/android/test_infra")
def setup_logging():
"""Configure logging for the script."""
log_format = "%(message)s"
logging.basicConfig(level=logging.INFO, format=log_format)
def run_command(
command: list[Union[str, bytes]], log_path: Optional[str] = None
) -> int:
"""Execute a command, log its output, and check for errors.
Args:
command: The command to execute
log_path: The path to a log file to write the command output to
Returns:
int: The exit code of the command
"""
with subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
cwd=ANDROID_TEST,
) as process:
if log_path:
with open(log_path, "a") as log_file:
for line in process.stdout:
sys.stdout.write(line)
log_file.write(line)
else:
for line in process.stdout:
sys.stdout.write(line)
process.wait()
sys.stdout.flush()
if process.returncode != 0:
error_message = f"Command {' '.join(command)} failed with exit code {process.returncode}"
logging.error(error_message)
return process.returncode
def setup_environment():
"""Configure Google Cloud project and authenticate with the service account."""
project_id = os.getenv("GOOGLE_PROJECT")
credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
if not project_id or not credentials_file:
logging.error(
"Error: GOOGLE_PROJECT and GOOGLE_APPLICATION_CREDENTIALS environment variables must be set."
)
sys.exit(1)
run_command(["gcloud", "config", "set", "project", project_id])
run_command(
["gcloud", "auth", "activate-service-account", "--key-file", credentials_file]
)
def execute_tests(
flank_config: str, apk_app: Path, apk_test: Optional[Path] = None
) -> int:
"""Run UI tests on Firebase Test Lab using Flank.
Args:
flank_config: The YML configuration for Flank to use e.g, automation/taskcluster/androidTest/flank-<config>.yml
apk_app: Absolute path to a Android APK application package (optional) for robo test or instrumentation test
apk_test: Absolute path to a Android APK androidTest package
Returns:
int: The exit code of the command
"""
run_command([Worker.JAVA_BIN.value, "-jar", Worker.FLANK_BIN.value, "--version"])
flank_command = [
Worker.JAVA_BIN.value,
"-jar",
Worker.FLANK_BIN.value,
"android",
"run",
"--config",
f"{ANDROID_TEST}/flank-configs/{flank_config}",
"--app",
str(apk_app),
"--local-result-dir",
Worker.RESULTS_DIR.value,
"--project",
os.environ.get("GOOGLE_PROJECT"),
]
# Add a client details parameter using the repository name
matrixLabel = os.environ.get("GECKO_HEAD_REPOSITORY")
geckoRev = os.environ.get("GECKO_HEAD_REV")
if matrixLabel is not None and geckoRev is not None:
flank_command.extend(
[
"--client-details",
f"matrixLabel={urlparse(matrixLabel).path.rpartition('/')[-1]},geckoRev={geckoRev}",
]
)
# Add androidTest APK if provided (optional) as robo test or instrumentation test
if apk_test:
flank_command.extend(["--test", str(apk_test)])
exit_code = run_command(flank_command, "flank.log")
if exit_code == 0:
logging.info("All UI test(s) have passed!")
return exit_code
def process_results(flank_config: str, test_type: str = "instrumentation") -> None:
"""Process and parse test results.
Args:
flank_config: The YML configuration for Flank to use e.g, automation/taskcluster/androidTest/flank-<config>.yml
"""
parse_junit_results_artifact = os.path.join(SCRIPT_DIR, "parse-junit-results.py")
copy_robo_crash_artifacts_script = os.path.join(
SCRIPT_DIR, "copy-artifacts-from-ftl.py"
)
os.chmod(parse_junit_results_artifact, 0o755)
os.chmod(copy_robo_crash_artifacts_script, 0o755)
# Process the results differently based on the test type: instrumentation or robo
#
# Instrumentation (i.e, Android UI Tests): parse the JUnit results for CI logging
# Robo Test (i.e, self-crawling): copy crash artifacts from Google Cloud Storage over
if test_type == "instrumentation":
run_command(
[parse_junit_results_artifact, "--results", Worker.RESULTS_DIR.value],
"flank.log",
)
if test_type == "robo":
run_command([copy_robo_crash_artifacts_script, "crash_log"])
def main():
"""Parse command line arguments and execute the test runner."""
parser = argparse.ArgumentParser(
description="Run UI tests on Firebase Test Lab using Flank as a test runner"
)
parser.add_argument(
"flank_config",
help="The YML configuration for Flank to use e.g, 'fenix/flank-arm-debug.yml'."
+ " This is relative to 'mobile/android/test_infra/flank-configs'.",
)
parser.add_argument(
"apk_app", help="Absolute path to a Android APK application package"
)
parser.add_argument(
"--apk_test",
help="Absolute path to a Android APK androidTest package",
default=None,
)
args = parser.parse_args()
setup_environment()
# Only resolve apk_test if it is provided
apk_test_path = Path(args.apk_test).resolve() if args.apk_test else None
exit_code = execute_tests(
flank_config=args.flank_config,
apk_app=Path(args.apk_app).resolve(),
apk_test=apk_test_path,
)
# Determine the instrumentation type to process the results differently
instrumentation_type = "instrumentation" if args.apk_test else "robo"
process_results(flank_config=args.flank_config, test_type=instrumentation_type)
sys.exit(exit_code)
if __name__ == "__main__":
setup_logging()
main()
|