xref: /aosp_15_r20/external/aws-crt-java/codebuild/CanaryWrapper_24_7.py (revision 3c7ae9de214676c52d19f01067dc1a404272dc11)

# Python wrapper script for collecting Canary metrics, setting up alarms, reporting metrics to Cloudwatch,
# checking the alarms to ensure everything is correct at the end of the run, and checking for new
# builds in S3, downloading them, and launching them if they exist (24/7 operation)
#
# Will only stop running if the Canary application itself has an issue - in which case it Canary application will
# need to be fixed and then the wrapper script restarted

# Needs to be installed prior to running
# Part of standard packages in Python 3.4+
import argparse
import time
# Dependencies in project folder
from CanaryWrapper_Classes import *
from CanaryWrapper_MetricFunctions import *

# TODO - Using subprocess may not work on Windows for starting/stopping the application thread.
#        Canary will likely be running on Linux, so it's probably okay, but need to confirm/check at some point....
# ================================================================================
# Code for command line argument parsing

command_parser = argparse.ArgumentParser("CanaryWrapper_24_7")
command_parser.add_argument("--canary_executable", type=str, required=True,
    help="The path to the canary executable")
command_parser.add_argument("--canary_arguments", type=str, default="",
    help="The arguments to pass/launch the canary executable with")
command_parser.add_argument("--s3_bucket_name", type=str, default="canary-wrapper-folder",
    help="(OPTIONAL, default=canary-wrapper-folder) The name of the S3 bucket where success logs will be stored")
command_parser.add_argument("--s3_bucket_application", type=str, required=True,
    help="(OPTIONAL, default=canary-wrapper-folder) The S3 URL to monitor for changes MINUS the bucket name")
command_parser.add_argument("--s3_bucket_application_in_zip", type=str, required=False, default="",
    help="(OPTIONAL, default="") The file path in the zip folder where the application is stored. Will be ignored if set to empty string")
command_parser.add_argument("--lambda_name", type=str, default="iot-send-email-lambda",
    help="(OPTIONAL, default='CanarySendEmailLambda') The name of the Lambda used to send emails")
command_parser_arguments = command_parser.parse_args()

# ================================================================================
# Global variables that both threads use to communicate.
# NOTE - These should likely be replaced with futures or similar for better thread safety.
#        However, these variables are only either read or written to from a single thread, no
#        thread should read and write to these variables.

# The local file path (and extension) of the Canary application that the wrapper will manage
# (This will also be the filename and directory used when a new file is detected in S3)
# [THIS IS READ ONLY]
canary_local_application_path = command_parser_arguments.canary_executable
if (canary_local_application_path == ""):
    print ("ERROR - required canary_executable is empty!")
    exit (1) # cannot run without a canary executable
# This is the arguments passed to the local file path when starting
# [THIS IS READ ONLY]
canary_local_application_arguments = command_parser_arguments.canary_arguments
# The "Git Hash" to use for metrics and dimensions
# [THIS IS READ ONLY]
canary_local_git_hash_stub = "Canary"
# The "Git Repo" name to use for metrics and dimensions. Is hard-coded since this is a 24/7 canary that should only run for MQTT
# [THIS IS READ ONLY]
canary_local_git_repo_stub = "MQTT5_24_7"
# The Fixed Namespace name for the Canary
# [THIS IS READ ONLY]
canary_local_git_fixed_namespace = "MQTT5_24_7_Canary"
# The S3 bucket name to monitor for the application
# [THIS IS READ ONLY]
canary_s3_bucket_name = command_parser_arguments.s3_bucket_name
if (canary_s3_bucket_name == ""):
    canary_s3_bucket_name = "canary-wrapper-folder"
# The file in the S3 bucket to monitor (The application filepath and file. Example: "canary/canary_application.exe")
# [THIS IS READ ONLY]
canary_s3_bucket_application_path = command_parser_arguments.s3_bucket_application
if (canary_s3_bucket_application_path == ""):
    print ("ERROR - required s3_bucket_application is empty!")
    exit (1) # cannot run without a s3_bucket_application to monitor
# The location of the file in the S3 zip, if the S3 file being monitored is a zip
# (THIS IS READ ONLY)
canary_s3_bucket_application_path_zip = command_parser_arguments.s3_bucket_application_in_zip
if (canary_s3_bucket_application_path_zip == ""):
    canary_s3_bucket_application_path_zip = None
# The name of the email lambda. If an empty string is set, it defaults to 'iot-send-email-lambda'
if (command_parser_arguments.lambda_name == ""):
    command_parser_arguments.lambda_name = "iot-send-email-lambda"
# The region the canary is running in
# (THIS IS READ ONLY)
canary_region_stub = "us-east-1"

# How long (in seconds) to wait before gathering metrics and pushing them to Cloudwatch
canary_metrics_wait_time = 600 # 10 minutes
# How long (in seconds) to run the Application thread loop. Should be shorter or equal to the Canary Metrics time
canary_application_loop_wait_time = 300 # 5 minutes

# For testing - set both to 30 seconds
# canary_metrics_wait_time = 30
# canary_application_loop_wait_time = 30

# ================================================================================

# Make the snapshot class
data_snapshot = DataSnapshot(
    git_hash=canary_local_git_hash_stub,
    git_repo_name=canary_local_git_repo_stub,
    git_hash_as_namespace=False,
    datetime_string=None,
    git_fixed_namespace_text=canary_local_git_fixed_namespace,
    output_log_filepath="output.txt",
    output_to_console=True,
    cloudwatch_region=canary_region_stub,
    cloudwatch_make_dashboard=True,
    cloudwatch_teardown_alarms_on_complete=True,
    cloudwatch_teardown_dashboard_on_complete=False,
    s3_bucket_name=canary_s3_bucket_name,
    s3_bucket_upload_on_complete=True,
    lambda_name=command_parser_arguments.lambda_name,
    metric_frequency=canary_metrics_wait_time)

# Make sure nothing failed
if (data_snapshot.abort_due_to_internal_error == True):
    print ("INFO - Stopping application due to error caused by credentials")
    print ("Please fix your credentials and then restart this application again")
    exit(0)

# Register metrics
data_snapshot.register_metric(
    new_metric_name="total_cpu_usage",
    new_metric_function=get_metric_total_cpu_usage,
    new_metric_unit="Percent",
    new_metric_alarm_threshold=70,
    new_metric_reports_to_skip=1,
    new_metric_alarm_severity=5,
    is_percent=True)
data_snapshot.register_metric(
    new_metric_name="total_memory_usage_value",
    new_metric_function=get_metric_total_memory_usage_value,
    new_metric_unit="Bytes")
data_snapshot.register_metric(
    new_metric_name="total_memory_usage_percent",
    new_metric_function=get_metric_total_memory_usage_percent,
    new_metric_unit="Percent",
    new_metric_alarm_threshold=70,
    new_metric_reports_to_skip=0,
    new_metric_alarm_severity=5,
    is_percent=True)

data_snapshot.register_dashboard_widget("Process CPU Usage - Percentage", ["total_cpu_usage"], 60)
data_snapshot.register_dashboard_widget("Process Memory Usage - Percentage", ["total_memory_usage_percent"], 60)

# Print diagnosis information
data_snapshot.output_diagnosis_information("24/7 Canary cannot show dependencies!")

# Make the S3 class
s3_monitor = S3Monitor(
    s3_bucket_name=canary_s3_bucket_name,
    s3_file_name=canary_s3_bucket_application_path,
    s3_file_name_in_zip=canary_s3_bucket_application_path_zip,
    canary_local_application_path=canary_local_application_path,
    data_snapshot=data_snapshot)

if (s3_monitor.had_internal_error == True):
    print ("INFO - Stopping application due to error caused by credentials")
    print ("Please fix your credentials and then restart this application again")
    exit(0)

# Make the snapshot (metrics) monitor
snapshot_monitor = SnapshotMonitor(
    wrapper_data_snapshot=data_snapshot,
    wrapper_metrics_wait_time=canary_metrics_wait_time)

# Make sure nothing failed
if (snapshot_monitor.had_internal_error == True):
    print ("INFO - Stopping application due to error caused by credentials")
    print ("Please fix your credentials and then restart this application again")
    exit(0)

# Make the application monitor
application_monitor = ApplicationMonitor(
    wrapper_application_path=canary_local_application_path,
    wrapper_application_arguments=canary_local_application_arguments,
    wrapper_application_restart_on_finish=True,
    data_snapshot=data_snapshot)

# Make sure nothing failed
if (application_monitor.error_has_occurred == True):
    print ("INFO - Stopping application due to error caused by credentials")
    print ("Please fix your credentials and then restart this application again")
    exit(0)

# For tracking if we stopped due to a metric alarm
stopped_due_to_metric_alarm = False

def execution_loop():
    while True:
        s3_monitor.monitor_loop_function(time_passed=canary_application_loop_wait_time)

        # Is there an error?
        if (s3_monitor.had_internal_error == True):
            print ("[Debug] S3 monitor had an internal error!")
            break

        # Is there a new file?
        if (s3_monitor.s3_file_needs_replacing == True):
            # Stop the application
            print ("[Debug] Stopping application monitor...")
            application_monitor.stop_monitoring()
            print ("[Debug] Getting S3 file...")
            s3_monitor.replace_current_file_for_new_file()
            # Start the application
            print ("[Debug] Starting application monitor...")
            application_monitor.start_monitoring()
            # Allow the snapshot monitor to cut a ticket
            snapshot_monitor.can_cut_ticket = True

        snapshot_monitor.monitor_loop_function(
            time_passed=canary_application_loop_wait_time, psutil_process=application_monitor.application_process_psutil)
        application_monitor.monitor_loop_function(
            time_passed=canary_application_loop_wait_time)

        # Did a metric go into alarm?
        if (snapshot_monitor.has_cut_ticket == True):
            # Do not allow it to cut anymore tickets until it gets a new build
            snapshot_monitor.can_cut_ticket = False

        # If an error has occurred or otherwise this thread needs to stop, then break the loop
        if (application_monitor.error_has_occurred == True or snapshot_monitor.had_internal_error == True):
            if (application_monitor.error_has_occurred == True):
                print ("[Debug] Application monitor error occurred!")
            else:
                print ("[Debug] Snapshot monitor internal error ocurred!")
            break

        time.sleep(canary_application_loop_wait_time)


def application_thread():
    # Start the application going
    snapshot_monitor.start_monitoring()
    application_monitor.start_monitoring()
    # Allow the snapshot monitor to cut tickets
    snapshot_monitor.can_cut_ticket = True

    start_email_body = "MQTT5 24/7 Canary Wrapper has started. This will run and continue to test new MQTT5 application builds as"
    start_email_body += " they pass CodeBuild and are uploaded to S3."
    snapshot_monitor.send_email(email_body=start_email_body, email_subject_text_append="Started")

    # Start the execution loop
    execution_loop()

    # Make sure everything is stopped
    snapshot_monitor.stop_monitoring()
    application_monitor.stop_monitoring()

    # Track whether this counts as an error (and therefore we should cleanup accordingly) or not
    wrapper_error_occurred = False

    send_finished_email = True
    finished_email_body = "MQTT5 24/7 Canary Wrapper has stopped."
    finished_email_body += "\n\n"

    try:
        # Find out why we stopped
        # S3 Monitor
        if (s3_monitor.had_internal_error == True):
            if (s3_monitor.error_due_to_credentials == False):
                print ("ERROR - S3 monitor stopped due to internal error!")
                cut_ticket_using_cloudwatch(
                    git_repo_name=canary_local_git_repo_stub,
                    git_hash=canary_local_git_hash_stub,
                    git_hash_as_namespace=False,
                    git_fixed_namespace_text=canary_local_git_fixed_namespace,
                    cloudwatch_region=canary_region_stub,
                    ticket_description="Snapshot monitor stopped due to internal error! Reason info: " + s3_monitor.internal_error_reason,
                    ticket_reason="S3 monitor stopped due to internal error",
                    ticket_allow_duplicates=True,
                    ticket_category="AWS",
                    ticket_type="SDKs and Tools",
                    ticket_item="IoT SDK for CPP",
                    ticket_group="AWS IoT Device SDK",
                    ticket_severity=4)
                finished_email_body += "Failure due to S3 monitor stopping due to an internal error."
                finished_email_body += " Reason given for error: " + s3_monitor.internal_error_reason
                wrapper_error_occurred = True
        # Snapshot Monitor
        elif (snapshot_monitor.had_internal_error == True):
            if (snapshot_monitor.has_cut_ticket == True):
                # We do not need to cut a ticket here - it's cut by the snapshot monitor!
                print ("ERROR - Snapshot monitor stopped due to metric in alarm!")
                finished_email_body += "Failure due to required metrics being in alarm! A new ticket should have been cut!"
                finished_email_body += "\nMetrics in Alarm: " + str(snapshot_monitor.cloudwatch_current_alarms_triggered)
                finished_email_body += "\nNOTE - this shouldn't occur in the 24/7 Canary! If it does, then the wrapper needs adjusting."
                wrapper_error_occurred = True
            else:
                print ("ERROR - Snapshot monitor stopped due to internal error!")
                cut_ticket_using_cloudwatch(
                    git_repo_name=canary_local_git_repo_stub,
                    git_hash=canary_local_git_hash_stub,
                    git_hash_as_namespace=False,
                    git_fixed_namespace_text=canary_local_git_fixed_namespace,
                    cloudwatch_region=canary_region_stub,
                    ticket_description="Snapshot monitor stopped due to internal error! Reason info: " + snapshot_monitor.internal_error_reason,
                    ticket_reason="Snapshot monitor stopped due to internal error",
                    ticket_allow_duplicates=True,
                    ticket_category="AWS",
                    ticket_type="SDKs and Tools",
                    ticket_item="IoT SDK for CPP",
                    ticket_group="AWS IoT Device SDK",
                    ticket_severity=4)
                wrapper_error_occurred = True
                finished_email_body += "Failure due to Snapshot monitor stopping due to an internal error."
                finished_email_body += " Reason given for error: " + snapshot_monitor.internal_error_reason
        # Application Monitor
        elif (application_monitor.error_has_occurred == True):
            if (application_monitor.error_due_to_credentials == True):
                print ("INFO - Stopping application due to error caused by credentials")
                print ("Please fix your credentials and then restart this application again")
                wrapper_error_occurred = True
                send_finished_email = False
            else:
                # Is the error something in the canary failed?
                if (application_monitor.error_code != 0):
                    cut_ticket_using_cloudwatch(
                        git_repo_name=canary_local_git_repo_stub,
                        git_hash=canary_local_git_hash_stub,
                        git_hash_as_namespace=False,
                        git_fixed_namespace_text=canary_local_git_fixed_namespace,
                        cloudwatch_region=canary_region_stub,
                        ticket_description="The 24/7 Canary exited with a non-zero exit code! This likely means something in the canary failed.",
                        ticket_reason="The 24/7 Canary exited with a non-zero exit code",
                        ticket_allow_duplicates=True,
                        ticket_category="AWS",
                        ticket_type="SDKs and Tools",
                        ticket_item="IoT SDK for CPP",
                        ticket_group="AWS IoT Device SDK",
                        ticket_severity=3)
                    wrapper_error_occurred = True
                    finished_email_body += "Failure due to MQTT5 application exiting with a non-zero exit code!"
                    finished_email_body += " This means something in the Canary application itself failed"
                else:
                    cut_ticket_using_cloudwatch(
                        git_repo_name=canary_local_git_repo_stub,
                        git_hash=canary_local_git_hash_stub,
                        git_hash_as_namespace=False,
                        git_fixed_namespace_text=canary_local_git_fixed_namespace,
                        cloudwatch_region=canary_region_stub,
                        ticket_description="The 24/7 Canary exited with a zero exit code but did not restart!",
                        ticket_reason="The 24/7 Canary exited with a zero exit code but did not restart",
                        ticket_allow_duplicates=True,
                        ticket_category="AWS",
                        ticket_type="SDKs and Tools",
                        ticket_item="IoT SDK for CPP",
                        ticket_group="AWS IoT Device SDK",
                        ticket_severity=3)
                    wrapper_error_occurred = True
                    finished_email_body += "Failure due to MQTT5 application stopping and not automatically restarting!"
                    finished_email_body += " This shouldn't occur and means something is wrong with the Canary wrapper!"
        # Other
        else:
            print ("ERROR - 24/7 Canary stopped due to unknown reason!")
            cut_ticket_using_cloudwatch(
                git_repo_name=canary_local_git_repo_stub,
                git_hash=canary_local_git_hash_stub,
                git_hash_as_namespace=False,
                git_fixed_namespace_text=canary_local_git_fixed_namespace,
                cloudwatch_region=canary_region_stub,
                ticket_description="The 24/7 Canary stopped for an unknown reason!",
                ticket_reason="The 24/7 Canary stopped for unknown reason",
                ticket_allow_duplicates=True,
                ticket_category="AWS",
                ticket_type="SDKs and Tools",
                ticket_item="IoT SDK for CPP",
                ticket_group="AWS IoT Device SDK",
                ticket_severity=3)
            wrapper_error_occurred = True
            finished_email_body += "Failure due to unknown reason! This shouldn't happen and means something has gone wrong!"
    except Exception as e:
        print ("ERROR: Could not (possibly) cut ticket due to exception!")
        print ("Exception: " + str(e), flush=True)

    # Clean everything up and stop
    snapshot_monitor.cleanup_monitor(error_occurred=wrapper_error_occurred)
    application_monitor.cleanup_monitor(error_occurred=wrapper_error_occurred)
    print ("24/7 Canary finished!")

    finished_email_body += "\n\nYou can find the log file for this run at the following S3 location: "
    finished_email_body += "https://s3.console.aws.amazon.com/s3/object/"
    finished_email_body += command_parser_arguments.s3_bucket_name
    finished_email_body += "?region=" + canary_region_stub
    finished_email_body += "&prefix=" + canary_local_git_repo_stub + "/"
    if (wrapper_error_occurred == True):
        finished_email_body += "Failed_Logs/"
    finished_email_body += canary_local_git_hash_stub + ".log"
    # Send the finish email
    if (send_finished_email == True):
        if (wrapper_error_occurred == True):
            snapshot_monitor.send_email(email_body=finished_email_body, email_subject_text_append="Had an error")
        else:
            snapshot_monitor.send_email(email_body=finished_email_body, email_subject_text_append="Finished")

    exit (-1)


# Start the application!
application_thread()