Fix CloudWatch log group collision for multiple executor fleets

examples/multiple-executors/README.md

@@ -5,7 +5,8 @@ This example uses [networking](https://registry.terraform.io/modules/sourcegraph
 The following variables must be supplied:
 
 - `sourcegraph_external_url`, `sourcegraph_executor_proxy_password`, `queue_name`, `metrics_environment_label`, and `instance_tag`: Analogous to the `executor_*` variables in the `single-executor` example.
-- `resource_prefix`: A prefix unique to each set of compute resources. This prevents collisions between two uses of the `executors` module. We recommend this value be constructed the same way `instance_tag` is constructed.
+- `resource_prefix`: A prefix unique to each set of compute resources. This prevents collisions between two uses of the `executors` module — including the CloudWatch log group each fleet's hosts ship syslog to (the on-host agent is reconfigured at boot to match). Give each fleet a distinct `resource_prefix` (or set `randomize_resource_names = true`) so two fleets in the same account/region don't collide on a single shared log group. We recommend this value be constructed the same way `instance_tag` is constructed.
+- `randomize_resource_names`: When `true`, appends a random suffix to resource names. Either this or a distinct `resource_prefix` per fleet is required to run more than one `executors` module in the same account and region.
 - `docker_registry_mirror`: This variable is given the value `"http://${module.docker-mirror.ip_address}:5000"`, which converts the raw external IP address to an address resolvable by the executor instances.
 
 If your deployment environment already has a Docker registry that can be used, only the `executor` submodule must be used (and references to the `networking` and `docker-mirror` modules can be dropped). The Docker registry mirror address can be supplied along with its containing VPC and subnet as pre-existing identifier literals.

modules/docker-mirror/main.tf

@@ -2,7 +2,11 @@ locals {
   resource_prefix = (var.resource_prefix == "" || substr(var.resource_prefix, -1, -2) == "-") ? var.resource_prefix : "${var.resource_prefix}-"
 
   cloudwatch_log_group = {
-    name = var.randomize_resource_names ? "${local.resource_prefix}sourcegraph-executors-docker-registry-mirror-${random_id.cloudwatch_log_group[0].hex}" : null
+    # The docker-mirror AMI's CloudWatch agent ships syslog to this group. When
+    # randomizing, use a unique name so multiple deployments in the same
+    # account/region don't collide; the boot-time startup script reconfigures
+    # the agent to match. Otherwise keep the legacy fixed name the AMI defaults to.
+    name = var.randomize_resource_names ? "${local.resource_prefix}sourcegraph-executors-docker-registry-mirror-${random_id.cloudwatch_log_group[0].hex}" : "executors_docker_mirror"
   }
   instance = {
     name = var.randomize_resource_names ? "${local.resource_prefix}sourcegraph-executors-docker-registry-mirror-${random_id.instance[0].hex}" : "sourcegraph-executors-docker-registry-mirror"
@@ -28,8 +32,10 @@ resource "random_id" "cloudwatch_log_group" {
 # Create a log group in CloudWatch. This is where the docker mirror will ingest
 # its logs to.
 resource "aws_cloudwatch_log_group" "syslogs" {
-  # TODO: This is hardcoded in the executor docker mirror image.
-  name              = "executors_docker_mirror"
+  # The docker-mirror AMI's CloudWatch agent defaults to a group literally named
+  # "executors_docker_mirror". The startup script reconfigures the agent to use
+  # this name at boot, so randomized deployments get a unique log group.
+  name              = local.cloudwatch_log_group.name
   retention_in_days = 7
 
   tags = {
@@ -121,7 +127,9 @@ resource "aws_instance" "default" {
 
   iam_instance_profile = aws_iam_instance_profile.instance.name
 
-  user_data_base64 = base64encode(file("${path.module}/startup-script.sh"))
+  user_data_base64 = base64encode(templatefile("${path.module}/startup-script.sh.tpl", {
+    cloudwatch_log_group_name = local.cloudwatch_log_group.name
+  }))
 }
 
 # Reserve a fixed disk to retain docker mirror data across rollouts.

modules/docker-mirror/outputs.tf

@@ -0,0 +1,4 @@
+output "cloudwatch_log_group_name" {
+  value       = aws_cloudwatch_log_group.syslogs.name
+  description = "The name of the CloudWatch log group that the docker-mirror host ships syslog to."
+}

modules/docker-mirror/startup-script.sh.tpl

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# Pull the terraform-provided CloudWatch log group name into the environment.
+CLOUDWATCH_LOG_GROUP_NAME="${cloudwatch_log_group_name}"
+
+# Point the host's CloudWatch agent at the configured log group. The docker-mirror
+# AMI ships syslog to a fixed "executors_docker_mirror" group by default; when a
+# specific (e.g. randomized) log group name is configured we reconfigure the agent
+# at boot so multiple deployments in the same account/region don't collide on one
+# shared log group.
+if [ "$${CLOUDWATCH_LOG_GROUP_NAME}" != '' ]; then
+  CLOUDWATCH_CONFIG_FILE_PATH=/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
+  if [ -x /usr/local/bin/configure-cloudwatch-agent-log-group ]; then
+    # Newer AMIs ship a helper that owns the agent config layout.
+    /usr/local/bin/configure-cloudwatch-agent-log-group "$${CLOUDWATCH_LOG_GROUP_NAME}"
+  elif [ -x /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl ]; then
+    # Fallback for AMIs predating the helper: (re)write the agent config with the
+    # desired log group name and reload. fetch-config consumes this file into the
+    # agent's own managed config, so we always write it fresh rather than assume
+    # the build-time source file is still present.
+    cat <<CWCONFIG >"$${CLOUDWATCH_CONFIG_FILE_PATH}"
+{
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": [
+          {
+            "file_path": "/var/log/syslog",
+            "log_group_name": "$${CLOUDWATCH_LOG_GROUP_NAME}",
+            "timezone": "UTC"
+          }
+        ]
+      }
+    },
+    "log_stream_name": "{instance_id}-syslog"
+  }
+}
+CWCONFIG
+    /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:"$${CLOUDWATCH_CONFIG_FILE_PATH}"
+  fi
+fi
+
+# Wait until the block device is attached.
+while true; do
+  sleep 1
+  test -e /dev/nvme1n1 && break
+done
+
+# Alias the nvme devices to /dev/sdX.
+# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html#available-ec2-device-names
+VOLUMES_NAME="$(find /dev -maxdepth 1 | grep -i 'nvme[0-21]n1$')"
+for VOLUME in $${VOLUMES_NAME}; do
+  ALIAS=$(sudo nvme id-ctrl -H -v "$${VOLUME}" | { grep -Po '/dev/(sd[b-z]|xvd[b-z])' || test $? = 1; })
+  if [ -n "$${ALIAS}" ]; then
+    sudo ln -s "$${VOLUME}" "$${ALIAS}"
+  fi
+done
+
+REALPATH="$(realpath /dev/sdh)"
+# Ensure /dev/sdh has a file system.
+if [ "$(sudo file -s "$REALPATH")" == "$REALPATH: data" ]; then
+  sudo mkfs.ext4 /dev/sdh
+fi
+
+# Mount /dev/sdh to /mnt/registry.
+sudo mkdir -p /mnt/registry
+echo "/dev/sdh /mnt/registry ext4 defaults 0 2" | sudo tee -a /etc/fstab
+sudo mount -a
+sudo chown ubuntu:ubuntu /mnt/registry
+
+# Enable and start the registry service.
+sudo systemctl enable docker_registry
+sudo systemctl start docker_registry

modules/executors/main.tf

@@ -11,7 +11,15 @@ locals {
     name = var.randomize_resource_names ? "${local.prefix}executors-${random_id.security_group[0].hex}" : "${var.resource_prefix}SourcegraphExecutorsMetricsAccess"
   }
   cloudwatch_log_group = {
-    name = var.randomize_resource_names ? "${local.prefix}executors-${random_id.cloudwatch_log_group[0].hex}" : null
+    # The executor AMI's CloudWatch agent ships syslog to this group; the
+    # boot-time startup script reconfigures the agent to this name (see
+    # startup-script.sh.tpl). Multiple fleets in one account/region must not
+    # share one group, so the name carries the same prefix/random suffix as the
+    # other executor resources (IAM role, launch template, ASG, ...):
+    #   - randomize_resource_names: unique per fleet via the random suffix
+    #   - resource_prefix set:      unique per fleet via the prefix
+    #   - neither:                  the legacy fixed "executors" name (single fleet)
+    name = var.randomize_resource_names ? "${local.prefix}executors-${random_id.cloudwatch_log_group[0].hex}" : (var.resource_prefix != "" ? "${local.prefix}executors" : "executors")
   }
   iam_instance_profile = {
     name = var.randomize_resource_names ? "${local.prefix}executors-${random_id.iam_instance_profile[0].hex}" : "${local.prefix}_executors"
@@ -137,8 +145,10 @@ resource "random_id" "cloudwatch_log_group" {
 }
 
 resource "aws_cloudwatch_log_group" "syslogs" {
-  # TODO: This is hardcoded in the executor image.
-  name              = "executors"
+  # The executor AMI's CloudWatch agent defaults to a group literally named
+  # "executors". The startup script reconfigures the agent to use this name at
+  # boot, so randomized deployments get a unique, non-colliding log group.
+  name              = local.cloudwatch_log_group.name
   retention_in_days = 7
 
   tags = {
@@ -267,6 +277,7 @@ resource "aws_launch_template" "executor" {
       "EXECUTOR_USE_FIRECRACKER"            = var.use_firecracker
       "EXECUTOR_DOCKER_AUTH_CONFIG"         = var.docker_auth_config
       "PRIVATE_CA_CERTIFICATE"              = var.private_ca_cert_path != "" ? file(var.private_ca_cert_path) : ""
+      "CLOUDWATCH_LOG_GROUP_NAME"           = local.cloudwatch_log_group.name
     }
   }))
 

modules/executors/outputs.tf

@@ -0,0 +1,4 @@
+output "cloudwatch_log_group_name" {
+  value       = aws_cloudwatch_log_group.syslogs.name
+  description = "The name of the CloudWatch log group that the executor hosts ship syslog to."
+}

modules/executors/startup-script.sh.tpl

@@ -50,6 +50,43 @@ EXECUTOR_DOCKER_AUTH_CONFIG="$${EXECUTOR_DOCKER_AUTH_CONFIG}"
 $${DOCKER_REGISTRY_NODE_EXPORTER_URL_LINE}
 EOF
 
+# Point the host's CloudWatch agent at the configured log group. The executor
+# AMI ships syslog to a fixed "executors" group by default; when a specific
+# (e.g. randomized) log group name is configured we reconfigure the agent at
+# boot so multiple executor deployments in the same account/region don't collide
+# on one shared log group.
+if [ "$${CLOUDWATCH_LOG_GROUP_NAME}" != '' ]; then
+  CLOUDWATCH_CONFIG_FILE_PATH=/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
+  if [ -x /usr/local/bin/configure-cloudwatch-agent-log-group ]; then
+    # Newer executor AMIs ship a helper that owns the agent config layout.
+    /usr/local/bin/configure-cloudwatch-agent-log-group "$${CLOUDWATCH_LOG_GROUP_NAME}"
+  elif [ -x /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl ]; then
+    # Fallback for AMIs predating the helper: (re)write the agent config with the
+    # desired log group name and reload. fetch-config consumes this file into the
+    # agent's own managed config, so we always write it fresh rather than assume
+    # the build-time source file is still present.
+    cat <<CWCONFIG >"$${CLOUDWATCH_CONFIG_FILE_PATH}"
+{
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": [
+          {
+            "file_path": "/var/log/syslog",
+            "log_group_name": "$${CLOUDWATCH_LOG_GROUP_NAME}",
+            "timezone": "UTC"
+          }
+        ]
+      }
+    },
+    "log_stream_name": "{instance_id}-syslog"
+  }
+}
+CWCONFIG
+    /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:"$${CLOUDWATCH_CONFIG_FILE_PATH}"
+  fi
+fi
+
 # Enable and start the executor service
 systemctl enable executor
 systemctl start executor