[QT-405] Create solo worker module (#3148)

* Create solo worker module

---------

Co-authored-by: Josh Brand <jbrand@hashicorp.com>

enos/enos-modules.hcl

@@ -11,7 +11,7 @@ module "bats_deps" {
 
 module "boundary" {
   source  = "app.terraform.io/hashicorp-qti/aws-boundary/enos"
-  version = ">= 0.2.6"
+  version = ">= 0.5.0"
 
   project_name = "qti-enos-boundary"
   environment  = var.environment
@@ -26,6 +26,19 @@ module "boundary" {
   alb_listener_api_port = var.alb_listener_api_port
 }
 
+module "worker" {
+  source = "./modules/worker"
+
+  common_tags = {
+    "Project" : "Enos",
+    "Project Name" : "qti-enos-boundary",
+    "Enos User" : var.enos_user,
+    "Environment" : var.environment
+  }
+
+  ssh_aws_keypair = var.aws_ssh_keypair_name
+}
+
 module "build_crt" {
   source = "./modules/build_crt"
 }
@@ -70,6 +83,10 @@ module "random_stringifier" {
   source = "./modules/random_stringifier"
 }
 
+module "map2list" {
+  source = "./modules/map2list"
+}
+
 module "target" {
   source       = "./modules/target"
   target_count = var.target_count

enos/enos-scenario-e2e-aws.hcl

@@ -113,6 +113,7 @@ scenario "e2e_aws" {
       vpc_id               = step.create_base_infra.vpc_id
       target_count         = 2
       additional_tags      = step.create_tag1_inputs.tag_map
+      subnet_ids           = step.create_boundary_cluster.subnet_ids
     }
   }
 
@@ -142,6 +143,7 @@ scenario "e2e_aws" {
       vpc_id               = step.create_base_infra.vpc_id
       target_count         = 1
       additional_tags      = step.create_tag2_inputs.tag_map
+      subnet_ids           = step.create_boundary_cluster.subnet_ids
     }
   }
 

enos/enos-scenario-e2e-database.hcl

@@ -46,6 +46,13 @@ scenario "e2e_database" {
     module = module.random_stringifier
   }
 
+  step "get_subnets" {
+    module = module.map2list
+    variables {
+      map = step.create_base_infra.vpc_subnets
+    }
+  }
+
   step "create_tag_inputs" {
     module     = module.generate_aws_host_tag_vars
     depends_on = [step.create_tag]
@@ -68,6 +75,7 @@ scenario "e2e_database" {
       vpc_id               = step.create_base_infra.vpc_id
       target_count         = 1
       additional_tags      = step.create_tag_inputs.tag_map
+      subnet_ids           = step.get_subnets.list
     }
   }
 

enos/enos-scenario-e2e-static-with-vault.hcl

@@ -119,6 +119,7 @@ scenario "e2e_static_with_vault" {
       instance_type        = var.target_instance_type
       vpc_id               = step.create_base_infra.vpc_id
       target_count         = 1
+      subnet_ids           = step.create_boundary_cluster.subnet_ids
     }
   }
 

enos/enos-scenario-e2e-static.hcl

@@ -97,6 +97,7 @@ scenario "e2e_static" {
       instance_type        = var.target_instance_type
       vpc_id               = step.create_base_infra.vpc_id
       target_count         = 1
+      subnet_ids           = step.create_boundary_cluster.subnet_ids
     }
   }
 

enos/enos-scenario-e2e-ui.hcl

@@ -135,6 +135,7 @@ scenario "e2e_ui" {
       vpc_id               = step.create_base_infra.vpc_id
       target_count         = 2
       additional_tags      = step.create_tag1_inputs.tag_map
+      subnet_ids           = step.create_boundary_cluster.subnet_ids
     }
   }
 
@@ -164,6 +165,7 @@ scenario "e2e_ui" {
       vpc_id               = step.create_base_infra.vpc_id
       target_count         = 1
       additional_tags      = step.create_tag2_inputs.tag_map
+      subnet_ids           = step.create_boundary_cluster.subnet_ids
     }
   }
 

enos/modules/map2list/main.tf

@@ -0,0 +1,11 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This module was created as a workaround, because we can't currently use
+# the `keys()` function call in Enos scenarios
+
+variable "map" {}
+
+output "list" {
+  value = keys(var.map)
+}

enos/modules/target/main.tf

@@ -3,6 +3,7 @@
 
 variable "vpc_id" {}
 variable "ami_id" {}
+variable "subnet_ids" {}
 variable "target_count" {}
 variable "environment" {}
 variable "project_name" {}
@@ -13,13 +14,6 @@ variable "additional_tags" {
   default = {}
 }
 
-data "aws_subnets" "infra" {
-  filter {
-    name   = "vpc-id"
-    values = [var.vpc_id]
-  }
-}
-
 resource "aws_security_group" "boundary_target" {
   name_prefix = "boundary-target-sg"
   description = "SSH and boundary Traffic"
@@ -50,7 +44,7 @@ resource "aws_instance" "target" {
   ami                    = var.ami_id
   instance_type          = var.instance_type
   vpc_security_group_ids = [aws_security_group.boundary_target.id]
-  subnet_id              = tolist(data.aws_subnets.infra.ids)[count.index % length(data.aws_subnets.infra.ids)]
+  subnet_id              = var.subnet_ids[count.index % length(var.subnet_ids)]
   key_name               = var.aws_ssh_keypair_name
 
   tags = merge(var.additional_tags, {

enos/modules/worker/main.tf

@@ -0,0 +1,206 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+data "enos_environment" "current" {}
+
+locals {
+  selected_az = data.aws_availability_zones.available.names[random_integer.az.result]
+  common_tags = merge(
+    var.common_tags,
+    {
+      Type   = var.cluster_tag
+      Module = "boundary-worker"
+      Pet    = random_pet.worker.id
+    },
+  )
+}
+
+resource "random_pet" "worker" {
+  separator = "_"
+}
+
+data "aws_availability_zones" "available" {
+  state = "available"
+  filter {
+    name   = "zone-name"
+    values = var.availability_zones
+  }
+}
+
+data "aws_availability_zone" "worker_az" {
+  name = local.selected_az
+}
+
+data "aws_kms_key" "kms_key" {
+  key_id = var.kms_key_arn
+}
+
+resource "random_integer" "az" {
+  min = 0
+  max = length(data.aws_availability_zones.available.names) - 1
+  keepers = {
+    # Generate a new integer each time the list of aws_availability_zones changes
+    # keepers have to be strings, sort the list in case order changes but zones don't
+    listener_arn = join("", sort(data.aws_availability_zones.available.names))
+  }
+}
+
+# Create a subnet so that the worker doesn't share one with a controller
+resource "aws_subnet" "default" {
+  vpc_id                  = var.vpc_name
+  cidr_block              = "10.13.9.0/24"
+  map_public_ip_on_launch = true
+  availability_zone       = local.selected_az
+  tags = merge(
+    local.common_tags,
+    {
+      "Name" = "${var.vpc_name}_worker_${random_pet.worker.id}_subnet"
+    },
+  )
+}
+
+# The worker instance is a part of this security group, not to be confused with the next rule below
+resource "aws_security_group" "default" {
+  name        = "boundary-sg-worker-${random_pet.worker.id}"
+  description = "SSH to worker to KMS and controllers"
+  vpc_id      = var.vpc_name
+
+  ingress {
+    description = "SSH to the worker instance"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["${data.enos_environment.current.public_ip_address}/32"]
+  }
+
+  egress {
+    description = "Communication from Boundary worker to controller"
+    from_port   = 9201
+    to_port     = 9201
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "Communication from Boundary worker to controller"
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = merge(
+    local.common_tags,
+    {
+      "Name" = "${var.vpc_name}_worker_sg"
+    },
+  )
+}
+
+# This module only manages a rule for the controller SG that is vital to the worker's operation
+# This module does _not_ manage the security group itself
+resource "aws_vpc_security_group_ingress_rule" "worker_to_controller" {
+  description       = "This rule allows traffic from a worker to controllers over WAN"
+  security_group_id = var.controller_sg_id
+  cidr_ipv4         = "${aws_instance.worker.public_ip}/32"
+  from_port         = 9201
+  to_port           = 9201
+  ip_protocol       = "tcp"
+}
+
+data "aws_route_table" "default" {
+  vpc_id = var.vpc_name
+
+  filter {
+    name   = "tag:Name"
+    values = ["enos-vpc_route"]
+  }
+}
+
+resource "aws_route_table_association" "worker_rta" {
+  subnet_id      = aws_subnet.default.id
+  route_table_id = data.aws_route_table.default.id
+}
+
+resource "aws_instance" "worker" {
+  ami                    = var.ubuntu_ami_id
+  instance_type          = var.worker_instance_type
+  vpc_security_group_ids = [aws_security_group.default.id]
+  subnet_id              = aws_subnet.default.id
+  key_name               = var.ssh_aws_keypair
+  iam_instance_profile   = var.iam_instance_profile_name
+  monitoring             = var.worker_monitoring
+
+  root_block_device {
+    iops        = var.ebs_iops
+    volume_size = var.ebs_size
+    volume_type = var.ebs_type
+    throughput  = var.ebs_throughput
+    tags        = local.common_tags
+  }
+
+  tags = merge(
+    local.common_tags,
+    {
+      Name = "${var.name_prefix}-boundary-worker",
+    },
+  )
+}
+
+resource "enos_bundle_install" "worker" {
+  depends_on = [aws_instance.worker, aws_route_table_association.worker_rta]
+
+  destination = var.boundary_install_dir
+  artifactory = var.boundary_artifactory_release
+  path        = var.local_artifact_path
+  release     = var.boundary_release == null ? var.boundary_release : merge(var.boundary_release, { product = "boundary", edition = "oss" })
+
+  transport = {
+    ssh = {
+      host = aws_instance.worker.public_ip
+    }
+  }
+}
+
+resource "enos_file" "worker_config" {
+  depends_on = [enos_bundle_install.worker]
+
+  destination = "/etc/boundary/boundary.hcl"
+  content = templatefile("${path.module}/templates/worker.hcl", {
+    id                   = random_pet.worker.id
+    kms_key_id           = data.aws_kms_key.kms_key.id
+    public_addr          = aws_instance.worker.public_ip
+    type                 = jsonencode(var.worker_type_tags)
+    region               = data.aws_availability_zone.worker_az.region
+    controller_addresses = jsonencode(var.controller_addresses)
+  })
+
+  transport = {
+    ssh = {
+      host = aws_instance.worker.public_ip
+    }
+  }
+}
+
+resource "enos_boundary_start" "worker_start" {
+  depends_on = [
+    enos_file.worker_config,
+    aws_vpc_security_group_ingress_rule.worker_to_controller,
+  ]
+
+  bin_path    = "/opt/boundary/bin"
+  config_path = "/etc/boundary"
+  transport = {
+    ssh = {
+      host = aws_instance.worker.public_ip
+    }
+  }
+}

enos/modules/worker/outputs.tf

@@ -0,0 +1,22 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+output "worker_ip" {
+  description = "The public IP of the Boundary worker"
+  value       = aws_instance.worker.public_ip
+}
+
+output "worker_tags" {
+  description = "The tags used in the worker's configuration"
+  value       = var.worker_type_tags
+}
+
+output "subnet_ids" {
+  description = "The ID of the subnet this worker resides in"
+  value       = [aws_subnet.default.id]
+}
+
+output "pet_id" {
+  description = "The ID of the random_pet used in this module"
+  value       = random_pet.worker.id
+}

enos/modules/worker/templates/worker.hcl

@@ -0,0 +1,31 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+listener "tcp" {
+  purpose = "proxy"
+  tls_disable = true
+  address = "0.0.0.0"
+}
+
+worker {
+  # Name attr must be unique across workers
+  name = "worker-${id}"
+  description = "Enos Boundary worker ${id}"
+
+  # Workers must be able to reach controllers on :9201
+  initial_upstreams = ${controller_addresses}
+
+  public_addr = "${public_addr}"
+
+  tags {
+    region = ["${region}"]
+    type = ${type}
+  }
+}
+
+# must be same key as used on controller config
+kms "awskms" {
+  purpose    = "worker-auth"
+  region     = "${region}"
+  kms_key_id = "${kms_key_id}"
+}

enos/modules/worker/variables.tf

@@ -0,0 +1,131 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+variable "vpc_name" {
+  description = "The name of the existing VPC to be used for this module"
+  type        = string
+}
+
+variable "availability_zones" {
+  description = "List of AWS availability zones to use (or * for all available)"
+  type        = list(string)
+  default     = ["*"]
+}
+
+variable "common_tags" {
+  description = "Tags to set for all resources"
+  type        = map(string)
+  default     = { "Project" : "Enos" }
+}
+
+variable "kms_key_arn" {
+  description = "ARN of KMS key used for SSHing to this module's instance"
+  type        = string
+}
+
+variable "ubuntu_ami_id" {
+  description = "Ubuntu LTS AMI from the VPC this module will use"
+  type        = string
+}
+
+variable "worker_instance_type" {
+  description = "The EC2 Instance type to be used for the worker's node"
+  type        = string
+  default     = "t2.micro"
+}
+
+variable "ssh_aws_keypair" {
+  description = "The name of the SSH keypair used to connect to EC2 instances"
+  type        = string
+}
+
+variable "worker_monitoring" {
+  description = "Enable detailed monitoring for workers"
+  type        = bool
+  default     = false
+}
+
+variable "ebs_iops" {
+  description = "EBS IOPS for the root volume"
+  type        = number
+  default     = null
+}
+
+variable "ebs_size" {
+  description = "EBS volume size"
+  type        = number
+  default     = 8
+}
+
+variable "ebs_type" {
+  description = "EBS volume type"
+  type        = string
+  default     = "gp2"
+}
+
+variable "ebs_throughput" {
+  description = "EBS data throughput (MiB/s) (only for gp2)"
+  default     = null
+}
+
+variable "boundary_artifactory_release" {
+  description = "Boundary release, version, and edition to install from artifactory.hashicorp.engineering"
+  type = object({
+    username = string
+    token    = string
+    url      = string
+    sha256   = string
+  })
+  default = null
+}
+
+variable "local_artifact_path" {
+  description = "The path to a local boundary.zip"
+  type        = string
+  default     = null
+}
+
+variable "boundary_release" {
+  description = "Boundary release, version, and edition to install from releases.hashicorp.com"
+  type = object({
+    version = string
+  })
+  default = null
+}
+
+variable "boundary_install_dir" {
+  description = "The remote directory where the Boundary binary will be installed"
+  type        = string
+  default     = "/opt/boundary/bin"
+}
+
+variable "iam_instance_profile_name" {
+  description = "The name of the AWS IAM instance profile from the Boundary cluster module"
+  type        = string
+}
+
+variable "name_prefix" {
+  description = "The name_prefix from the Boundary cluster module"
+  type        = string
+}
+
+variable "cluster_tag" {
+  description = "The cluster_tag from the Boundary cluster module"
+  type        = string
+}
+
+variable "worker_type_tags" {
+  description = "A list of tags to add in the worker's configuration file"
+  type        = list(string)
+  default     = ["prod", "webservers"]
+}
+
+variable "controller_addresses" {
+  description = "A list of addresses that will be used as initial_upstreams in the worker's configuration"
+  type        = list(string)
+}
+
+variable "controller_sg_id" {
+  description = "The controllers' security group ID for adding rules allowing this worker to communicate with them"
+  type        = string
+}