From ed027974703356a080ac170bf45aca88a0fd0aee Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Fri, 16 Oct 2026 08:44:15 -0400 Subject: [PATCH] fix(infra): watch the disk series the agent publishes The alarm named {InstanceId, path, fstype}, which matches no series (the agent's also carries device), so it sat in INSUFFICIENT_DATA from creation. It now reads the {InstanceId} rollup, as the memory alarm does, and counts missing data as breaching so a silent agent is reported too. Co-Authored-By: Claude Opus 5.5 Change-Id: I7921b92f27bf3bbae4fdd122c9d06f368f3ba1b7 --- infra/README.md | 8 ++++++-- infra/monitoring.tf | 23 +++++++++++++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/infra/README.md b/infra/README.md index 98c7054..869be2e 100644 --- a/infra/README.md +++ b/infra/README.md @@ -257,7 +257,7 @@ something to do about it: |---|---| | `instance-unreachable` | status checks failing; nothing is being served | | `cpu-credits-low` | a burstable instance about to be throttled to baseline | -| `disk-nearly-full` | over 85%; the likeliest failure here | +| `disk-nearly-full` | over 85%, the likeliest failure here; or the agent has stopped reporting | | `memory-nearly-full` | over 90%; the index holds its view in memory | | `not-indexing` | the site is down, **or up and connected to nothing** | @@ -269,7 +269,11 @@ all, so it is a Route53 health check that matches on the response body — Disk and memory are not EC2 metrics; nothing outside the machine can see them. The CloudWatch agent publishes both, which is why the instance role carries -`CloudWatchAgentServerPolicy`. +`CloudWatchAgentServerPolicy`. Both alarms read the agent's `{InstanceId}` +series. An alarm watches only a series whose dimensions match its own +exactly, and one that matches nothing waits in `INSUFFICIENT_DATA`, sending +nothing. So the disk alarm counts missing data as breaching, and it is also +the alarm that says the agent has stopped. **AWS emails a confirmation link before it will deliver anything.** Until that is clicked the topic accepts alarms and drops them all, which is this file's diff --git a/infra/monitoring.tf b/infra/monitoring.tf index 9b57fe5..1d65c8a 100644 --- a/infra/monitoring.tf +++ b/infra/monitoring.tf @@ -56,9 +56,19 @@ resource "aws_cloudwatch_metric_alarm" "cpu_credits" { # 3. The likeliest failure here by some distance. Jetstream's segment window # grows with the event rate, and a full disk stops the whole pipeline. +# +# An alarm's dimensions have to match a series exactly. The agent's own +# series for `/` carries `path`, `fstype` and `device`, and the device name +# is the instance type's business, so this reads the `{InstanceId}` series that +# user data's `aggregation_dimensions` publishes. The agent collects `/` +# alone, so that series is the root volume. The memory alarm reads the same +# shape of series. +# +# Missing data is breaching: if the agent stops publishing, this is the +# alarm that says so, rather than both custom metrics going quiet. resource "aws_cloudwatch_metric_alarm" "disk" { alarm_name = "${local.name}-disk-nearly-full" - alarm_description = "Over 85% of the root volume is used. Jetstream's window is the thing that grows; the volume can be resized up without downtime." + alarm_description = "Over 85% of the root volume is used, or the CloudWatch agent has stopped reporting it. Jetstream's window is the thing that grows; the volume can be resized up without downtime." namespace = "CWAgent" metric_name = "disk_used_percent" statistic = "Maximum" @@ -66,13 +76,10 @@ resource "aws_cloudwatch_metric_alarm" "disk" { evaluation_periods = 2 threshold = 85 comparison_operator = "GreaterThanThreshold" - dimensions = { - InstanceId = aws_instance.main.id - path = "/" - fstype = "xfs" - } - alarm_actions = [aws_sns_topic.alarms.arn] - treat_missing_data = "missing" + dimensions = { InstanceId = aws_instance.main.id } + alarm_actions = [aws_sns_topic.alarms.arn] + ok_actions = [aws_sns_topic.alarms.arn] + treat_missing_data = "breaching" } # 4. The index holds its view in memory, and that grows with the population. -- knotmirror 0.0.1