File: proxy_startup.py

package info (click to toggle)
python-azure 20230112%2Bgit-1
  • links: PTS, VCS
  • area: main
  • in suites: bookworm
  • size: 749,544 kB
  • sloc: python: 6,815,827; javascript: 287; makefile: 195; xml: 109; sh: 105
file content (222 lines) | stat: -rw-r--r-- 8,674 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------

import os
import logging
import requests
import shlex
import sys
import time
import signal

import pytest
import subprocess

from .config import PROXY_URL
from .helpers import is_live_and_not_recording
from .sanitizers import add_remove_header_sanitizer, set_custom_default_matcher


_LOGGER = logging.getLogger()

CONTAINER_NAME = "ambitious_azsdk_test_proxy"
LINUX_IMAGE_SOURCE_PREFIX = "azsdkengsys.azurecr.io/engsys/testproxy-lin"
WINDOWS_IMAGE_SOURCE_PREFIX = "azsdkengsys.azurecr.io/engsys/testproxy-win"
CONTAINER_STARTUP_TIMEOUT = 60
PROXY_MANUALLY_STARTED = os.getenv("PROXY_MANUAL_START", False)

PROXY_CHECK_URL = PROXY_URL + "/Info/Available"
TOOL_ENV_VAR = "PROXY_PID"

discovered_roots = []

def get_image_tag(repo_root: str) -> str:
    """Gets the test proxy Docker image tag from the target_version.txt file in /eng/common/testproxy"""
    version_file_location = os.path.relpath("eng/common/testproxy/target_version.txt")
    version_file_location_from_root = os.path.abspath(os.path.join(repo_root, version_file_location))

    with open(version_file_location_from_root, "r") as f:
        image_tag = f.read().strip()

    return image_tag


def ascend_to_root(start_dir_or_file: str) -> str:
    """
    Given a path, ascend until encountering a folder with a `.git` folder present within it. Return that directory.

    :param str start_dir_or_file: The starting directory or file. Either is acceptable.
    """
    if os.path.isfile(start_dir_or_file):
        current_dir = os.path.dirname(start_dir_or_file)
    else:
        current_dir = start_dir_or_file

    while current_dir is not None and not (os.path.dirname(current_dir) == current_dir):
        possible_root = os.path.join(current_dir, ".git")

        # we need the git check to prevent ascending out of the repo
        if os.path.exists(possible_root):
            if current_dir not in discovered_roots:
                discovered_roots.append(current_dir)
            return current_dir
        else:
            current_dir = os.path.dirname(current_dir)

    raise Exception(f'Requested target "{start_dir_or_file}" does not exist within a git repo.')


def delete_container() -> None:
    """Delete container if it remained"""
    proc = subprocess.Popen(
        shlex.split(f"docker rm -f {CONTAINER_NAME}"),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stdin=subprocess.DEVNULL,
    )
    output, stderr = proc.communicate(timeout=10)
    return None


def check_availability() -> None:
    """Attempts request to /Info/Available. If a test-proxy instance is responding, we should get a response."""
    try:
        response = requests.get(PROXY_CHECK_URL, timeout=10)
        return response.status_code
    # We get an SSLError if the container is started but the endpoint isn't available yet
    except requests.exceptions.SSLError as sslError:
        _LOGGER.debug(sslError)
        return 404
    except Exception as e:
        _LOGGER.debug(e)
        return 404


def check_system_proxy_availability() -> None:
    """Checks for SSL_CERT_DIR and REQUESTS_CA_BUNDLE environment variables."""
    ssl_cert = "SSL_CERT_DIR"
    ca_bundle = "REQUESTS_CA_BUNDLE"

    if PROXY_URL.startswith("https") and not os.environ.get(ssl_cert):
        _LOGGER.error(f"Please ensure the '{ssl_cert}' environment variable is correctly set in your test environment")
    if PROXY_URL.startswith("https") and not os.environ.get(ca_bundle):
        _LOGGER.error(f"Please ensure the '{ca_bundle}' environment variable is correctly set in your test environment")


def check_proxy_availability() -> None:
    """Waits for the availability of the test-proxy."""
    start = time.time()
    now = time.time()
    status_code = 0
    while now - start < CONTAINER_STARTUP_TIMEOUT and status_code != 200:
        status_code = check_availability()
        now = time.time()


def create_container(repo_root: str) -> None:
    """Creates the test proxy Docker container"""
    # Most of the time, running this script on a Windows machine will work just fine, as Docker defaults to Linux
    # containers. However, in CI, Windows images default to _Windows_ containers. We cannot swap them. We can tell
    # if we're in a CI build by checking for the environment variable TF_BUILD.
    delete_container()

    if sys.platform.startswith("win") and os.environ.get("TF_BUILD"):
        image_prefix = WINDOWS_IMAGE_SOURCE_PREFIX
        path_prefix = "C:"
        linux_container_args = ""
    else:
        image_prefix = LINUX_IMAGE_SOURCE_PREFIX
        path_prefix = ""
        linux_container_args = "--add-host=host.docker.internal:host-gateway"

    image_tag = get_image_tag(repo_root)
    subprocess.Popen(
        shlex.split(
            f"docker run --rm --name {CONTAINER_NAME} -v '{repo_root}:{path_prefix}/srv/testproxy' "
            f"{linux_container_args} -p 5001:5001 -p 5000:5000 {image_prefix}:{image_tag}"
        ),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stdin=subprocess.DEVNULL,
    )


def start_test_proxy(request) -> None:
    """Starts the test proxy and returns when the proxy server is ready to receive requests. In regular use
    cases, this will auto-start the test-proxy docker container. In CI, or when environment variable TF_BUILD is set, this
    function will start the test-proxy .NET tool."""

    repo_root = ascend_to_root(request.node.items[0].module.__file__)
    check_system_proxy_availability()

    if not PROXY_MANUALLY_STARTED:
        if os.getenv("TF_BUILD"):
            _LOGGER.info("Starting the test proxy tool...")
            if check_availability() == 200:
                _LOGGER.debug("Tool is responding, exiting...")
            else:
                envname = os.getenv("TOX_ENV_NAME", "default")
                root = os.getenv("BUILD_SOURCESDIRECTORY", repo_root)
                log = open(os.path.join(root, "_proxy_log_{}.log".format(envname)), "a")

                _LOGGER.info("{} is calculated repo root".format(root))

                os.environ["PROXY_ASSETS_FOLDER"] = os.path.join(root, "l", envname)
                if not os.path.exists(os.environ["PROXY_ASSETS_FOLDER"]):
                    os.makedirs(os.environ["PROXY_ASSETS_FOLDER"])

                proc = subprocess.Popen(
                    shlex.split('test-proxy start --storage-location="{}" -- --urls "{}"'.format(root, PROXY_URL)),
                    stdout=log,
                    stderr=log
                )
                os.environ[TOOL_ENV_VAR] = str(proc.pid)
        else:
            _LOGGER.info("Starting the test proxy container...")
            create_container(repo_root)

    # Wait for the proxy server to become available
    check_proxy_availability()
    # remove headers from recordings if we don't need them, and ignore them if present
    # Authorization, for example, can contain sensitive info and can cause matching failures during challenge auth
    headers_to_ignore = "Authorization, x-ms-client-request-id, x-ms-request-id"
    add_remove_header_sanitizer(headers=headers_to_ignore)
    set_custom_default_matcher(excluded_headers=headers_to_ignore)


def stop_test_proxy() -> None:
    """Stops any running instance of the test proxy"""

    if not PROXY_MANUALLY_STARTED:
        if os.getenv("TF_BUILD"):
            _LOGGER.info("Stopping the test proxy tool...")

            try:
                os.kill(int(os.getenv(TOOL_ENV_VAR)), signal.SIGTERM)
            except:
                _LOGGER.debug("Unable to kill running test-proxy process.")

        else:
            _LOGGER.info("Stopping the test proxy container...")
            subprocess.Popen(
                shlex.split(f"docker stop {CONTAINER_NAME}"),
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                stdin=subprocess.DEVNULL,
            )


@pytest.fixture(scope="session")
def test_proxy(request) -> None:
    """Pytest fixture to be used before running any tests that are recorded with the test proxy"""
    if is_live_and_not_recording():
        yield
    else:
        start_test_proxy(request)
        # Everything before this yield will be run before fixtures that invoke this one are run
        # Everything after it will be run after invoking fixtures are done executing
        yield
        stop_test_proxy()