1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
|
# Python wrapper script for collecting Canary metrics, setting-up/tearing-down alarms, reporting metrics to Cloudwatch,
# checking the alarms to ensure everything is correct at the end of the run, and pushing the log to S3 if successful.
# Needs to be installed prior to running
# Part of standard packages in Python 3.4+
import argparse
import time
import datetime
# Dependencies in project folder
from CanaryWrapper_Classes import *
from CanaryWrapper_MetricFunctions import *
# Code for command line argument parsing
# ================================================================================
command_parser = argparse.ArgumentParser("CanaryWrapper")
command_parser.add_argument("--canary_executable", type=str, required=True,
help="The path to the canary executable (or program - like 'python3')")
command_parser.add_argument("--canary_arguments", type=str, default="",
help="The arguments to pass/launch the canary executable with")
command_parser.add_argument("--git_hash", type=str, required=True,
help="The Git commit hash that we are running the canary with")
command_parser.add_argument("--git_repo_name", type=str, required=True,
help="The name of the Git repository")
command_parser.add_argument("--git_hash_as_namespace", type=bool, default=False,
help="(OPTIONAL, default=False) If true, the git hash will be used as the name of the Cloudwatch namespace")
command_parser.add_argument("--output_log_filepath", type=str, default="output.log",
help="(OPTIONAL, default=output.log) The file to output log info to. Set to 'None' to disable")
command_parser.add_argument("--output_to_console", type=bool, default=True,
help="(OPTIONAL, default=True) If true, info will be output to the console")
command_parser.add_argument("--cloudwatch_region", type=str, default="us-east-1",
help="(OPTIONAL, default=us-east-1) The AWS region for Cloudwatch")
command_parser.add_argument("--s3_bucket_name", type=str, default="canary-wrapper-folder",
help="(OPTIONAL, default=canary-wrapper-folder) The name of the S3 bucket where success logs will be stored")
command_parser.add_argument("--snapshot_wait_time", type=int, default=600,
help="(OPTIONAL, default=600) The number of seconds between gathering and sending snapshot reports")
command_parser.add_argument("--ticket_category", type=str, default="AWS",
help="(OPTIONAL, default=AWS) The category to register the ticket under")
command_parser.add_argument("--ticket_type", type=str, default="SDKs and Tools",
help="(OPTIONAL, default='SDKs and Tools') The type to register the ticket under")
command_parser.add_argument("--ticket_item", type=str, default="IoT SDK for CPP",
help="(OPTIONAL, default='IoT SDK for CPP') The item to register the ticket under")
command_parser.add_argument("--ticket_group", type=str, default="AWS IoT Device SDK",
help="(OPTIONAL, default='AWS IoT Device SDK') The group to register the ticket under")
command_parser.add_argument("--dependencies", type=str, default="",
help="(OPTIONAL, default='') Any dependencies and their commit hashes. \
Current expected format is '(name or path);(hash);(next name or path);(hash);(etc...)'.")
command_parser.add_argument("--lambda_name", type=str, default="iot-send-email-lambda",
help="(OPTIONAL, default='CanarySendEmailLambda') The name of the Lambda used to send emails")
command_parser.add_argument("--codebuild_log_path", type=str, default="",
help="The CODEBUILD_LOG_PATH environment variable. Leave blank to ignore")
command_parser_arguments = command_parser.parse_args()
if (command_parser_arguments.output_log_filepath == "None"):
command_parser_arguments.output_log_filepath = None
if (command_parser_arguments.snapshot_wait_time <= 0):
command_parser_arguments.snapshot_wait_time = 60
# Deal with possibly empty values in semi-critical commands/arguments
if (command_parser_arguments.canary_executable == ""):
print ("ERROR - required canary_executable is empty!", flush=True)
exit (1) # cannot run without a canary executable
if (command_parser_arguments.git_hash == ""):
print ("ERROR - required git_hash is empty!", flush=True)
exit (1) # cannot run without git hash
if (command_parser_arguments.git_repo_name == ""):
print ("ERROR - required git_repo_name is empty!", flush=True)
exit (1) # cannot run without git repo name
if (command_parser_arguments.git_hash_as_namespace is not True and command_parser_arguments.git_hash_as_namespace is not False):
command_parser_arguments.git_hash_as_namespace = False
if (command_parser_arguments.output_log_filepath == ""):
command_parser_arguments.output_log_filepath = None
if (command_parser_arguments.output_to_console != True and command_parser_arguments.output_to_console != False):
command_parser_arguments.output_to_console = True
if (command_parser_arguments.cloudwatch_region == ""):
command_parser_arguments.cloudwatch_region = "us-east-1"
if (command_parser_arguments.s3_bucket_name == ""):
command_parser_arguments.s3_bucket_name = "canary-wrapper-folder"
if (command_parser_arguments.ticket_category == ""):
command_parser_arguments.ticket_category = "AWS"
if (command_parser_arguments.ticket_type == ""):
command_parser_arguments.ticket_type = "SDKs and Tools"
if (command_parser_arguments.ticket_item == ""):
command_parser_arguments.ticket_item = "IoT SDK for CPP"
if (command_parser_arguments.ticket_group == ""):
command_parser_arguments.ticket_group = "AWS IoT Device SDK"
# ================================================================================
datetime_now = datetime.datetime.now()
datetime_string = datetime_now.strftime("%d-%m-%Y/%H-%M-%S")
print("Datetime string is: " + datetime_string, flush=True)
# Make the snapshot class
data_snapshot = DataSnapshot(
git_hash=command_parser_arguments.git_hash,
git_repo_name=command_parser_arguments.git_repo_name,
datetime_string=datetime_string,
git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace,
git_fixed_namespace_text="mqtt5_canary",
output_log_filepath="output.txt",
output_to_console=command_parser_arguments.output_to_console,
cloudwatch_region="us-east-1",
cloudwatch_make_dashboard=False,
cloudwatch_teardown_alarms_on_complete=True,
cloudwatch_teardown_dashboard_on_complete=True,
s3_bucket_name=command_parser_arguments.s3_bucket_name,
s3_bucket_upload_on_complete=True,
lambda_name=command_parser_arguments.lambda_name,
metric_frequency=command_parser_arguments.snapshot_wait_time)
# Make sure nothing failed
if (data_snapshot.abort_due_to_internal_error == True):
print ("INFO - Stopping application due to error caused by credentials")
print ("Please fix your credentials and then restart this application again", flush=True)
exit(0)
# Register metrics
data_snapshot.register_metric(
new_metric_name="total_cpu_usage",
new_metric_function=get_metric_total_cpu_usage,
new_metric_unit="Percent",
new_metric_alarm_threshold=70,
new_metric_reports_to_skip=1,
new_metric_alarm_severity=5,
is_percent=True)
data_snapshot.register_metric(
new_metric_name="total_memory_usage_value",
new_metric_function=get_metric_total_memory_usage_value,
new_metric_unit="Bytes")
data_snapshot.register_metric(
new_metric_name="total_memory_usage_percent",
new_metric_function=get_metric_total_memory_usage_percent,
new_metric_unit="Percent",
new_metric_alarm_threshold=70,
new_metric_reports_to_skip=0,
new_metric_alarm_severity=5,
is_percent=True)
# Print diagnosis information
data_snapshot.output_diagnosis_information(command_parser_arguments.dependencies)
# Make the snapshot (metrics) monitor
snapshot_monitor = SnapshotMonitor(
wrapper_data_snapshot=data_snapshot,
wrapper_metrics_wait_time=command_parser_arguments.snapshot_wait_time)
# Make sure nothing failed
if (snapshot_monitor.had_internal_error == True):
print ("INFO - Stopping application due to error caused by credentials")
print ("Please fix your credentials and then restart this application again", flush=True)
exit(0)
# Make the application monitor
application_monitor = ApplicationMonitor(
wrapper_application_path=command_parser_arguments.canary_executable,
wrapper_application_arguments=command_parser_arguments.canary_arguments,
wrapper_application_restart_on_finish=False,
data_snapshot=data_snapshot # pass the data_snapshot for printing to the log
)
# Make sure nothing failed
if (application_monitor.error_has_occurred == True):
print ("INFO - Stopping application due to error caused by credentials")
print ("Please fix your credentials and then restart this application again", flush=True)
exit(0)
# For tracking if we stopped due to a metric alarm
stopped_due_to_metric_alarm = False
execution_sleep_time = 30
def execution_loop():
while True:
snapshot_monitor.monitor_loop_function(
time_passed=execution_sleep_time, psutil_process=application_monitor.application_process_psutil)
application_monitor.monitor_loop_function(
time_passed=execution_sleep_time)
# Did a metric go into alarm?
if (snapshot_monitor.has_cut_ticket == True):
# Set that we had an 'internal error' so we go down the right code path
snapshot_monitor.had_internal_error = True
break
# If an error has occurred or otherwise this thread needs to stop, then break the loop
if (application_monitor.error_has_occurred == True or snapshot_monitor.had_internal_error == True):
break
time.sleep(execution_sleep_time)
def application_thread():
start_email_body = "MQTT5 Short Running Canary Wrapper has started for "
start_email_body += "\"" + command_parser_arguments.git_repo_name + "\" commit \"" + command_parser_arguments.git_hash + "\""
start_email_body += "\nThe wrapper will run for the length the MQTT5 Canary application is set to run for, which is determined by "
start_email_body += "the arguments set. The arguments used for this run are listed below:"
start_email_body += "\n Arguments: " + command_parser_arguments.canary_arguments
snapshot_monitor.send_email(email_body=start_email_body, email_subject_text_append="Started")
# Start the application going
snapshot_monitor.start_monitoring()
application_monitor.start_monitoring()
# Allow the snapshot monitor to cut tickets
snapshot_monitor.can_cut_ticket = True
# Start the execution loop
execution_loop()
# Make sure everything is stopped
snapshot_monitor.stop_monitoring()
application_monitor.stop_monitoring()
# Track whether this counts as an error (and therefore we should cleanup accordingly) or not
wrapper_error_occurred = False
# Finished Email
send_finished_email = True
finished_email_body = "MQTT5 Short Running Canary Wrapper has stopped."
finished_email_body += "\n\n"
try:
# Find out why we stopped
if (snapshot_monitor.had_internal_error == True):
if (snapshot_monitor.has_cut_ticket == True):
# We do not need to cut a ticket here - it's cut by the snapshot monitor!
print ("ERROR - Snapshot monitor stopped due to metric in alarm!", flush=True)
finished_email_body += "Failure due to required metrics being in alarm! A new ticket should have been cut!"
finished_email_body += "\nMetrics in Alarm: " + str(snapshot_monitor.cloudwatch_current_alarms_triggered)
wrapper_error_occurred = True
else:
print ("ERROR - Snapshot monitor stopped due to internal error!", flush=True)
cut_ticket_using_cloudwatch(
git_repo_name=command_parser_arguments.git_repo_name,
git_hash=command_parser_arguments.git_hash,
git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace,
git_fixed_namespace_text="mqtt5_canary",
cloudwatch_region="us-east-1",
ticket_description="Snapshot monitor stopped due to internal error! Reason info: " + snapshot_monitor.internal_error_reason,
ticket_reason="Snapshot monitor stopped due to internal error",
ticket_allow_duplicates=True,
ticket_category=command_parser_arguments.ticket_category,
ticket_item=command_parser_arguments.ticket_item,
ticket_group=command_parser_arguments.ticket_group,
ticket_type=command_parser_arguments.ticket_type,
ticket_severity=4)
wrapper_error_occurred = True
finished_email_body += "Failure due to Snapshot monitor stopping due to an internal error."
finished_email_body += " Reason given for error: " + snapshot_monitor.internal_error_reason
elif (application_monitor.error_has_occurred == True):
if (application_monitor.error_due_to_credentials == True):
print ("INFO - Stopping application due to error caused by credentials")
print ("Please fix your credentials and then restart this application again", flush=True)
wrapper_error_occurred = True
send_finished_email = False
else:
# Is the error something in the canary failed?
if (application_monitor.error_code != 0):
cut_ticket_using_cloudwatch(
git_repo_name=command_parser_arguments.git_repo_name,
git_hash=command_parser_arguments.git_hash,
git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace,
git_fixed_namespace_text="mqtt5_canary",
cloudwatch_region="us-east-1",
ticket_description="The Short Running Canary exited with a non-zero exit code! This likely means something in the canary failed.",
ticket_reason="The Short Running Canary exited with a non-zero exit code",
ticket_allow_duplicates=True,
ticket_category=command_parser_arguments.ticket_category,
ticket_item=command_parser_arguments.ticket_item,
ticket_group=command_parser_arguments.ticket_group,
ticket_type=command_parser_arguments.ticket_type,
ticket_severity=4)
wrapper_error_occurred = True
finished_email_body += "Failure due to MQTT5 application exiting with a non-zero exit code! This means something in the Canary application itself failed"
else:
print ("INFO - Stopping application. No error has occurred, application has stopped normally", flush=True)
application_monitor.print_stdout()
finished_email_body += "Short Running Canary finished successfully and run without errors!"
wrapper_error_occurred = False
else:
print ("ERROR - Short Running Canary stopped due to unknown reason!", flush=True)
cut_ticket_using_cloudwatch(
git_repo_name=command_parser_arguments.git_repo_name,
git_hash=command_parser_arguments.git_hash,
git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace,
git_fixed_namespace_text="mqtt5_canary",
cloudwatch_region="us-east-1",
ticket_description="The Short Running Canary stopped for an unknown reason!",
ticket_reason="The Short Running Canary stopped for unknown reason",
ticket_allow_duplicates=True,
ticket_category=command_parser_arguments.ticket_category,
ticket_item=command_parser_arguments.ticket_item,
ticket_group=command_parser_arguments.ticket_group,
ticket_type=command_parser_arguments.ticket_type,
ticket_severity=4)
wrapper_error_occurred = True
finished_email_body += "Failure due to unknown reason! This shouldn't happen and means something has gone wrong!"
except Exception as e:
print ("ERROR: Could not (possibly) cut ticket due to exception!")
print ("Exception: " + str(e), flush=True)
# Clean everything up and stop
snapshot_monitor.cleanup_monitor(error_occurred=wrapper_error_occurred)
application_monitor.cleanup_monitor(error_occurred=wrapper_error_occurred)
print ("Short Running Canary finished!", flush=True)
finished_email_body += "\n\nYou can find the log file for this run at the following S3 location: "
finished_email_body += "https://s3.console.aws.amazon.com/s3/object/"
finished_email_body += command_parser_arguments.s3_bucket_name
finished_email_body += "?region=" + command_parser_arguments.cloudwatch_region
finished_email_body += "&prefix=" + command_parser_arguments.git_repo_name + "/" + datetime_string + "/"
if (wrapper_error_occurred == True):
finished_email_body += "Failed_Logs/"
finished_email_body += command_parser_arguments.git_hash + ".log"
if (command_parser_arguments.codebuild_log_path != ""):
print ("\n Codebuild log path: " + command_parser_arguments.codebuild_log_path + "\n")
# Send the finish email
if (send_finished_email == True):
if (wrapper_error_occurred == True):
snapshot_monitor.send_email(email_body=finished_email_body, email_subject_text_append="Had an error")
else:
snapshot_monitor.send_email(email_body=finished_email_body, email_subject_text_append="Finished")
exit (application_monitor.error_code)
# Start the application!
application_thread()
|
