File: test_export_tasks.py

package info (click to toggle)
python-moto 5.1.18-3
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 116,520 kB
  • sloc: python: 636,725; javascript: 181; makefile: 39; sh: 3
file content (292 lines) | stat: -rw-r--r-- 10,187 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
import copy
import json
from datetime import datetime, timedelta
from uuid import UUID, uuid4

import boto3
import pytest
from botocore.exceptions import ClientError

from moto import mock_aws
from moto.core.utils import unix_time_millis
from tests import allow_aws_request

S3_POLICY = {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": "s3:GetBucketAcl",
            "Effect": "Allow",
            "Resource": "arn:aws:s3:::{BUCKET_NAME}",
            "Principal": {"Service": "logs.us-east-1.amazonaws.com"},
            "Condition": {
                "StringEquals": {"aws:SourceAccount": ["AccountId1"]},
                "ArnLike": {"aws:SourceArn": ["..."]},
            },
        },
        {
            "Action": "s3:PutObject",
            "Effect": "Allow",
            "Resource": "arn:aws:s3:::{BUCKET_NAME}/*",
            "Principal": {"Service": "logs.us-east-1.amazonaws.com"},
            "Condition": {
                "StringEquals": {
                    "s3:x-amz-acl": "bucket-owner-full-control",
                    "aws:SourceAccount": ["AccountId1"],
                },
                "ArnLike": {"aws:SourceArn": ["..."]},
            },
        },
    ],
}


@pytest.fixture()
def logs():
    if allow_aws_request():
        yield boto3.client("logs", region_name="us-east-1")
    else:
        with mock_aws():
            yield boto3.client("logs", region_name="us-east-1")


@pytest.fixture()
def s3():
    if allow_aws_request:
        yield boto3.client("s3", region_name="us-east-1")
    else:
        with mock_aws():
            yield boto3.client("s3", region_name="us-east-1")


@pytest.fixture(scope="function")
def log_group_name(logs):
    name = "/moto/logs_test/" + str(uuid4())[0:5]
    logs.create_log_group(logGroupName=name)
    yield name
    logs.delete_log_group(logGroupName=name)


@pytest.fixture()
def bucket_name(s3, account_id):
    name = f"moto-logs-test-{str(uuid4())[0:6]}"
    s3.create_bucket(Bucket=name)
    policy = copy.copy(S3_POLICY)
    policy["Statement"][0]["Resource"] = f"arn:aws:s3:::{name}"
    policy["Statement"][0]["Condition"]["StringEquals"]["aws:SourceAccount"] = [
        account_id
    ]
    policy["Statement"][0]["Condition"]["ArnLike"]["aws:SourceArn"] = [
        f"arn:aws:logs:us-east-1:{account_id}:log-group:*"
    ]
    policy["Statement"][1]["Resource"] = f"arn:aws:s3:::{name}/*"
    policy["Statement"][1]["Condition"]["StringEquals"]["aws:SourceAccount"] = [
        account_id
    ]
    policy["Statement"][1]["Condition"]["ArnLike"]["aws:SourceArn"] = [
        f"arn:aws:logs:us-east-1:{account_id}:log-group:*"
    ]

    s3.put_bucket_policy(Bucket=name, Policy=json.dumps(policy))
    yield name

    # Delete any files left behind
    versions = s3.list_object_versions(Bucket=name).get("Versions", [])
    for key in versions:
        s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId"))
    delete_markers = s3.list_object_versions(Bucket=name).get("DeleteMarkers", [])
    for key in delete_markers:
        s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId"))

    # Delete bucket itself
    s3.delete_bucket(Bucket=name)


@pytest.mark.aws_verified
def test_create_export_task_happy_path(logs, s3, log_group_name, bucket_name):
    fromTime = 1611316574
    to = 1642852574
    resp = logs.create_export_task(
        logGroupName=log_group_name,
        fromTime=fromTime,
        to=to,
        destination=bucket_name,
    )
    task_id = resp["taskId"]
    # taskId resembles a valid UUID (i.e. a string of 32 hexadecimal digits)
    assert UUID(task_id)
    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200

    # s3 bucket contains indication that permissions were successful
    resp = s3.get_object(Bucket=bucket_name, Key="aws-logs-write-test")
    assert resp["Body"].read() == b"Permission Check Successful"

    try:
        # ExportTask's can take a long time to succeed
        # From the docs: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/S3ExportTasks.html
        #     > the export task might take anywhere from a few seconds to a few hours
        #
        # There can be only one ExportTask active at any point in time
        # Cancelling this one ensures that there's no Task active
        # And subsequent tests can create Export Tasks without running into a LimitExceededException
        logs.cancel_export_task(taskId=task_id)
    except ClientError as exc:
        # Because there are no logs, the export task in AWS usually finishes very quickly
        # Which is fine - we can just ignore that
        assert (
            exc.response["Error"]["Message"]
            == "The specified export task has already finished"
        )


@pytest.mark.aws_verified
def test_cancel_unknown_export_task(logs):
    with pytest.raises(ClientError) as exc:
        logs.cancel_export_task(taskId=str(uuid4()))
    err = exc.value.response["Error"]
    assert err["Code"] == "ResourceNotFoundException"
    assert err["Message"] == "The specified export task does not exist."


@pytest.mark.aws_verified
def test_create_export_task_raises_ClientError_when_bucket_not_found(
    logs,
    log_group_name,
):
    destination = "368a7022dea3dd621"
    fromTime = 1611316574
    to = 1642852574
    with pytest.raises(ClientError) as exc:
        logs.create_export_task(
            logGroupName=log_group_name,
            fromTime=fromTime,
            to=to,
            destination=destination,
        )
    err = exc.value.response["Error"]
    assert err["Code"] == "InvalidParameterException"
    assert (
        err["Message"]
        == "The given bucket does not exist. Please make sure the bucket is valid."
    )


@pytest.mark.aws_verified
def test_create_export_raises_ResourceNotFoundException_log_group_not_found(
    logs,
    bucket_name,
):
    with pytest.raises(logs.exceptions.ResourceNotFoundException) as exc:
        logs.create_export_task(
            logGroupName=f"/aws/nonexisting/{str(uuid4())[0:6]}",
            fromTime=1611316574,
            to=1642852574,
            destination=bucket_name,
        )
    err = exc.value.response["Error"]
    assert err["Code"] == "ResourceNotFoundException"
    assert err["Message"] == "The specified log group does not exist."


@pytest.mark.aws_verified
def test_create_export_executes_export_task(logs, s3, log_group_name, bucket_name):
    fromTime = int(unix_time_millis(datetime.now() - timedelta(days=1)))
    to = int(unix_time_millis(datetime.now() + timedelta(days=1)))

    logs.create_log_stream(logGroupName=log_group_name, logStreamName="/some/stream")

    messages = [
        {"timestamp": int(unix_time_millis()), "message": f"hello_{i}"}
        for i in range(10)
    ]
    logs.put_log_events(
        logGroupName=log_group_name, logStreamName="/some/stream", logEvents=messages
    )

    task_id = logs.create_export_task(
        logGroupName=log_group_name,
        fromTime=fromTime,
        to=to,
        destination=bucket_name,
    )["taskId"]

    task = logs.describe_export_tasks(taskId=task_id)["exportTasks"][0]
    assert task["status"]["code"] in ["COMPLETED", "RUNNING", "PENDING"]
    assert task["logGroupName"] == log_group_name
    assert task["destination"] == bucket_name
    assert task["destinationPrefix"] == "exportedlogs"
    assert task["from"] == fromTime
    assert task["to"] == to

    objects = s3.list_objects(Bucket=bucket_name)["Contents"]
    key_names = [o["Key"] for o in objects]
    assert "aws-logs-write-test" in key_names


def test_describe_export_tasks_happy_path(logs, s3, log_group_name):
    destination = "mybucket"
    fromTime = 1611316574
    to = 1642852574
    s3.create_bucket(Bucket=destination)
    logs.create_export_task(
        logGroupName=log_group_name,
        fromTime=fromTime,
        to=to,
        destination=destination,
    )
    resp = logs.describe_export_tasks()
    assert len(resp["exportTasks"]) == 1
    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
    assert resp["exportTasks"][0]["destination"] == destination
    assert resp["exportTasks"][0]["from"] == fromTime
    assert resp["exportTasks"][0]["to"] == to
    assert resp["exportTasks"][0]["status"]["code"] == "COMPLETED"
    assert resp["exportTasks"][0]["status"]["message"] == "Completed successfully"


def test_describe_export_tasks_out_of_order_timestamps(logs, s3, log_group_name):
    destination = "mybucket"
    fromTime = 1000
    to = 500
    s3.create_bucket(Bucket=destination)
    logs.create_export_task(
        logGroupName=log_group_name,
        fromTime=fromTime,
        to=to,
        destination=destination,
    )
    resp = logs.describe_export_tasks()
    assert len(resp["exportTasks"]) == 1
    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
    assert resp["exportTasks"][0]["destination"] == destination
    assert resp["exportTasks"][0]["from"] == fromTime
    assert resp["exportTasks"][0]["to"] == to
    assert resp["exportTasks"][0]["status"]["code"] == "active"
    assert resp["exportTasks"][0]["status"]["message"] == "Task is active"


def test_describe_export_tasks_task_id(logs, log_group_name, bucket_name):
    fromTime = 1611316574
    to = 1642852574
    resp = logs.create_export_task(
        logGroupName=log_group_name,
        fromTime=fromTime,
        to=to,
        destination=bucket_name,
    )
    taskId = resp["taskId"]
    resp = logs.describe_export_tasks(taskId=taskId)
    assert len(resp["exportTasks"]) == 1
    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
    assert resp["exportTasks"][0]["destination"] == bucket_name
    assert resp["exportTasks"][0]["from"] == fromTime
    assert resp["exportTasks"][0]["to"] == to
    assert resp["exportTasks"][0]["status"]["code"] == "COMPLETED"
    assert resp["exportTasks"][0]["status"]["message"] == "Completed successfully"


def test_describe_export_tasks_raises_ResourceNotFoundException_task_id_not_found(
    logs,
):
    with pytest.raises(logs.exceptions.ResourceNotFoundException):
        logs.describe_export_tasks(taskId="368a7022dea3dd621")