From 6e009ad7fe572413cd63b9c08bbe14edd8136055 Mon Sep 17 00:00:00 2001 From: Frank Raassild Date: Tue, 21 Jul 2026 16:07:38 +0300 Subject: [PATCH 1/3] Manage CloudWatch status-check alarms with VM lifecycle Create per-VM status-check alarms at create, delete at terminate/gc, plus an alarm-sync command. Config-driven (cloudwatch_alarm_checks, _prefix, _topic, _desc_*); no-op when unconfigured, non-fatal on AWS errors. --- vmtool/aws.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/vmtool/aws.py b/vmtool/aws.py index a9cdcc0..258faa2 100644 --- a/vmtool/aws.py +++ b/vmtool/aws.py @@ -358,6 +358,11 @@ def get_route53(self): """ return self.get_boto3_client('route53') + def get_cloudwatch(self): + """Get cached CloudWatch connection. + """ + return self.get_boto3_client('cloudwatch') + def get_ec2_client(self, region=None): return self.get_boto3_client('ec2', region) @@ -2306,9 +2311,89 @@ def vm_create_start(self): return ids + # CloudWatch status-check alarms, attached to VMs like DNS/EIP. + # No-op unless cloudwatch_alarm_checks is set. AWS evaluates the + # metrics and pushes state changes; vmtool only manages the objects. + CW_ALARM_SPECS = { + 'system': ('StatusCheckFailed_System', 1), + 'instance': ('StatusCheckFailed_Instance', 2), + 'ebs': ('StatusCheckFailed_AttachedEBS', 1), + } + CW_ALARM_PAGES = ('system', 'ebs') + + def assign_status_alarms(self, vm_id): + """Create status-check alarms for VM. Idempotent, non-fatal. + """ + checks = self.cf.getlist('cloudwatch_alarm_checks', []) + if not checks: + return + for check in checks: + if check not in self.CW_ALARM_SPECS: + eprintf("WARNING: cloudwatch_alarm_checks: unknown check, skipping alarms: %s", check) + return + prefix = self.cf.get('cloudwatch_alarm_prefix', 'vm-status-') + topic = self.cf.get('cloudwatch_alarm_topic', '') + client = self.get_cloudwatch() + for check in checks: + metric, evals = self.CW_ALARM_SPECS[check] + desc = self.cf.get('cloudwatch_alarm_desc_%s' % check, + '%s status check failed' % check) + actions = [topic] if topic and check in self.CW_ALARM_PAGES else [] + try: + client.put_metric_alarm( + AlarmName='%s%s-%s' % (prefix, check, vm_id), + AlarmDescription='%s - %s' % (self.full_role, desc), + Namespace='AWS/EC2', + MetricName=metric, + Dimensions=[{'Name': 'InstanceId', 'Value': vm_id}], + Statistic='Maximum', + Period=60, + EvaluationPeriods=evals, + Threshold=1, + ComparisonOperator='GreaterThanOrEqualToThreshold', + AlarmActions=actions, + OKActions=actions) + except Exception as ex: + eprintf("WARNING: alarm create failed (%s/%s): %s", vm_id, check, ex) + return + time_printf("Status alarms set: %s", vm_id) + + def drop_status_alarms(self, *vm_ids): + """Delete status-check alarms of VMs. Non-fatal. + """ + if not vm_ids or not self.cf.getlist('cloudwatch_alarm_checks', []): + return + prefix = self.cf.get('cloudwatch_alarm_prefix', 'vm-status-') + names = ['%s%s-%s' % (prefix, check, vm_id) + for vm_id in vm_ids for check in self.CW_ALARM_SPECS] + client = self.get_cloudwatch() + try: + for pos in range(0, len(names), 100): + client.delete_alarms(AlarmNames=names[pos:pos + 100]) + if self.options.verbose: + printf("Status alarms dropped: %s", " ".join(vm_ids)) + except Exception as ex: + eprintf("WARNING: alarm delete failed: %s", ex) + + def cmd_alarm_sync(self, *vm_ids): + """Create/update status-check alarms for VMs (all running role VMs + by default). Does not remove alarms of terminated VMs. + + Group: vm + """ + for check in self.cf.getlist('cloudwatch_alarm_checks', []): + if check not in self.CW_ALARM_SPECS: + raise UsageError('cloudwatch_alarm_checks: unknown check: %s' % check) + if not vm_ids: + vm_ids = [vm['InstanceId'] for vm in self.get_running_vms()] + for vm_id in vm_ids: + self.assign_status_alarms(vm_id) + def vm_create_finish(self, ids): for vm_id in ids: self.new_ssh_key(vm_id) + for vm_id in ids: + self.assign_status_alarms(vm_id) time_printf("Instances are ready") time.sleep(10) self._vm_map = {} @@ -2455,6 +2540,7 @@ def cmd_terminate(self, *ids): raise UsageError("Instances not stopped: %s" % " ".join(bad)) else: client.terminate_instances(InstanceIds=ids) + self.drop_status_alarms(*ids) def cmd_gc(self): """Terminate all stopped vms. @@ -2496,6 +2582,7 @@ def cmd_gc(self): if garbage: printf("Terminating: %s", " ".join(garbage)) client.terminate_instances(InstanceIds=garbage) + self.drop_status_alarms(*garbage) elif keep_count > 0: printf("Keeping stopped instances") else: From 07539a336fdcade08c18eb0bc9d5f711b6065306 Mon Sep 17 00:00:00 2001 From: Frank Raassild Date: Wed, 22 Jul 2026 16:01:56 +0300 Subject: [PATCH 2/3] Manage CloudWatch status-check alarms with VM lifecycle Create per-VM status-check alarms at create, delete them at terminate/gc, prune orphans of externally-deleted VMs during gc. Adds alarm-sync command. Config-driven (cloudwatch_alarm_checks, _prefix, _topic, _desc_*); off by default, non-fatal on AWS errors. --- vmtool/aws.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/vmtool/aws.py b/vmtool/aws.py index 258faa2..ad2d8d5 100644 --- a/vmtool/aws.py +++ b/vmtool/aws.py @@ -2375,6 +2375,33 @@ def drop_status_alarms(self, *vm_ids): except Exception as ex: eprintf("WARNING: alarm delete failed: %s", ex) + def prune_status_alarms(self): + """Delete status-check alarms whose VM no longer exists. Non-fatal. + """ + if not self.cf.getlist('cloudwatch_alarm_checks', []): + return + prefix = self.cf.get('cloudwatch_alarm_prefix', 'vm-status-') + client = self.get_cloudwatch() + try: + alive = set() + for vm in self.ec2_iter_instances(): + alive.add(vm['InstanceId']) + orphans = [] + for page in client.get_paginator('describe_alarms').paginate(AlarmNamePrefix=prefix): + for alarm in page['MetricAlarms']: + # alarm name is - + name = alarm['AlarmName'] + pos = name.find('-i-') + if pos < 0 or name[pos + 1:] in alive: + continue + orphans.append(name) + for pos in range(0, len(orphans), 100): + client.delete_alarms(AlarmNames=orphans[pos:pos + 100]) + if orphans: + printf("Pruned %d orphaned status alarms", len(orphans)) + except Exception as ex: + eprintf("WARNING: alarm prune failed: %s", ex) + def cmd_alarm_sync(self, *vm_ids): """Create/update status-check alarms for VMs (all running role VMs by default). Does not remove alarms of terminated VMs. @@ -2587,6 +2614,7 @@ def cmd_gc(self): printf("Keeping stopped instances") else: printf("No stopped instances") + self.prune_status_alarms() def cmd_ssh(self, *args): """SSH to VM and run command (optional). From 0b55f1f0640512da101f7f58efd18d8a21b6dcc3 Mon Sep 17 00:00:00 2001 From: Frank Raassild Date: Wed, 22 Jul 2026 17:55:39 +0300 Subject: [PATCH 3/3] Partial state reporting when alert fails --- vmtool/aws.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vmtool/aws.py b/vmtool/aws.py index ad2d8d5..65d5e6f 100644 --- a/vmtool/aws.py +++ b/vmtool/aws.py @@ -2334,7 +2334,7 @@ def assign_status_alarms(self, vm_id): prefix = self.cf.get('cloudwatch_alarm_prefix', 'vm-status-') topic = self.cf.get('cloudwatch_alarm_topic', '') client = self.get_cloudwatch() - for check in checks: + for done, check in enumerate(checks): metric, evals = self.CW_ALARM_SPECS[check] desc = self.cf.get('cloudwatch_alarm_desc_%s' % check, '%s status check failed' % check) @@ -2354,7 +2354,8 @@ def assign_status_alarms(self, vm_id): AlarmActions=actions, OKActions=actions) except Exception as ex: - eprintf("WARNING: alarm create failed (%s/%s): %s", vm_id, check, ex) + eprintf("WARNING: alarm create failed (%s/%s), %d/%d alarms set, fix with alarm-sync: %s", + vm_id, check, done, len(checks), ex) return time_printf("Status alarms set: %s", vm_id)