Create IPv6 and dual stack enos tests (#5212)

Co-authored-by: Danielle Miu <29378233+DanielleMiu@users.noreply.github.com>
Co-authored-by: Damian Debkowski <damian.debkowski@hashicorp.com>

enos/modules/aws_vault/iam.tf

@@ -36,17 +36,20 @@ data "aws_iam_policy_document" "vault_profile" {
 }
 
 resource "aws_iam_role" "vault_instance_role" {
+  count              = var.deploy ? 1 : 0
   name               = "vault_instance_role-${random_string.cluster_id.result}"
   assume_role_policy = data.aws_iam_policy_document.vault_instance_role.json
 }
 
 resource "aws_iam_instance_profile" "vault_profile" {
-  name = "vault_instance_profile-${random_string.cluster_id.result}"
-  role = aws_iam_role.vault_instance_role.name
+  count = var.deploy ? 1 : 0
+  name  = "vault_instance_profile-${random_string.cluster_id.result}"
+  role  = aws_iam_role.vault_instance_role[0].name
 }
 
 resource "aws_iam_role_policy" "vault_policy" {
+  count  = var.deploy ? 1 : 0
   name   = "vault_policy-${random_string.cluster_id.result}"
-  role   = aws_iam_role.vault_instance_role.id
+  role   = aws_iam_role.vault_instance_role[0].id
   policy = data.aws_iam_policy_document.vault_profile.json
 }

enos/modules/aws_vault/locals.tf

@@ -11,9 +11,9 @@ locals {
     "awskms" = null
     "shamir" = 3
   }
-  followers   = toset(slice(local.instances, 1, length(local.instances)))
-  instances   = [for idx in range(var.instance_count) : tostring(idx)]
-  leader      = toset(slice(local.instances, 0, 1))
+  followers   = var.deploy ? toset(slice(local.instances, 1, length(local.instances))) : toset([])
+  instances   = var.deploy ? [for idx in range(var.instance_count) : tostring(idx)] : []
+  leader      = var.deploy ? toset(slice(local.instances, 0, 1)) : toset([])
   name_suffix = "${var.project_name}-${var.environment}"
   recovery_shares = {
     "awskms" = 5
@@ -53,4 +53,41 @@ locals {
   audit_device_file_path = "/var/log/vault/vault_audit.log"
   vault_service_user     = "vault"
   enable_audit_device    = var.enable_file_audit_device && var.vault_init
+  network_stack = {
+    "4" = {
+      ingress_cidr_blocks = flatten([
+        formatlist("%s/32", data.enos_environment.localhost.public_ipv4_addresses),
+        join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
+        formatlist("%s/32", var.sg_additional_ips),
+      ])
+      ingress_ipv6_cidr_blocks = [],
+      egress_cidr_blocks       = ["0.0.0.0/0"],
+      egress_ipv6_cidr_blocks  = [],
+      ipv6_address_count       = 0,
+    },
+    "6" = {
+      ingress_cidr_blocks = [],
+      ingress_ipv6_cidr_blocks = flatten([
+        [for ip in coalesce(data.enos_environment.localhost.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+        [data.aws_vpc.infra.ipv6_cidr_block],
+        [for ip in var.sg_additional_ipv6_ips : cidrsubnet("${ip}/64", 0, 0)],
+      ])
+      egress_cidr_blocks      = [],
+      egress_ipv6_cidr_blocks = ["::/0"],
+      ipv6_address_count      = 1,
+    },
+    "dual" = {
+      ingress_cidr_blocks = flatten([
+        formatlist("%s/32", data.enos_environment.localhost.public_ipv4_addresses),
+      ])
+      ingress_ipv6_cidr_blocks = flatten([
+        [for ip in coalesce(data.enos_environment.localhost.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+        [data.aws_vpc.infra.ipv6_cidr_block],
+        [for ip in var.sg_additional_ipv6_ips : cidrsubnet("${ip}/64", 0, 0)],
+      ])
+      egress_cidr_blocks      = [],
+      egress_ipv6_cidr_blocks = ["::/0"],
+      ipv6_address_count      = 1,
+    }
+  }
 }

enos/modules/aws_vault/outputs.tf

@@ -8,7 +8,7 @@ output "instance_ids" {
 
 output "instance_public_ips" {
   description = "Public IPs of Vault instances"
-  value       = [for instance in aws_instance.vault_instance : instance.public_ip]
+  value       = var.ip_version == "4" ? [for instance in aws_instance.vault_instance : instance.public_ip] : flatten([for instance in aws_instance.vault_instance : instance.ipv6_addresses])
 }
 
 output "instance_private_ips" {
@@ -40,6 +40,10 @@ output "vault_root_token" {
   value = coalesce(var.vault_root_token, try(enos_vault_init.leader[0].root_token, null), "none")
 }
 
+output "vault_transit_token" {
+  value = try([for token in enos_remote_exec.vault_kms_policy : trimspace(token.stdout)][0], "")
+}
+
 output "vault_unseal_keys_b64" {
   value = try(enos_vault_init.leader[0].unseal_keys_b64, [])
 }

enos/modules/aws_vault/security-groups.tf

@@ -2,52 +2,44 @@
 # SPDX-License-Identifier: BUSL-1.1
 
 resource "aws_security_group" "enos_vault_sg" {
+  count       = var.deploy ? 1 : 0
   name        = "vault-sg-${random_string.cluster_id.result}"
   description = "SSH and Vault Traffic"
   vpc_id      = var.vpc_id
 
   # SSH
   ingress {
-    from_port = 22
-    to_port   = 22
-    protocol  = "tcp"
-    cidr_blocks = flatten([
-      formatlist("%s/32", data.enos_environment.localhost.public_ipv4_addresses),
-      join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
-    ])
+    from_port        = 22
+    to_port          = 22
+    protocol         = "tcp"
+    cidr_blocks      = local.network_stack[var.ip_version].ingress_cidr_blocks
+    ipv6_cidr_blocks = local.network_stack[var.ip_version].ingress_ipv6_cidr_blocks
   }
 
   # Vault traffic
   ingress {
-    from_port = 8200
-    to_port   = 8201
-    protocol  = "tcp"
-    cidr_blocks = flatten([
-      formatlist("%s/32", data.enos_environment.localhost.public_ipv4_addresses),
-      join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
-      formatlist("%s/32", var.sg_additional_ips),
-    ])
+    from_port        = 8200
+    to_port          = 8201
+    protocol         = "tcp"
+    cidr_blocks      = local.network_stack[var.ip_version].ingress_cidr_blocks
+    ipv6_cidr_blocks = local.network_stack[var.ip_version].ingress_ipv6_cidr_blocks
   }
 
   # Consul Agent traffic
   ingress {
-    from_port = 8301
-    to_port   = 8301
-    protocol  = "tcp"
-    cidr_blocks = flatten([
-      formatlist("%s/32", data.enos_environment.localhost.public_ipv4_addresses),
-      join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
-    ])
+    from_port        = 8301
+    to_port          = 8301
+    protocol         = "tcp"
+    cidr_blocks      = local.network_stack[var.ip_version].ingress_cidr_blocks
+    ipv6_cidr_blocks = local.network_stack[var.ip_version].ingress_ipv6_cidr_blocks
   }
 
   ingress {
-    from_port = 8301
-    to_port   = 8301
-    protocol  = "udp"
-    cidr_blocks = flatten([
-      formatlist("%s/32", data.enos_environment.localhost.public_ipv4_addresses),
-      join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
-    ])
+    from_port        = 8301
+    to_port          = 8301
+    protocol         = "udp"
+    cidr_blocks      = local.network_stack[var.ip_version].ingress_cidr_blocks
+    ipv6_cidr_blocks = local.network_stack[var.ip_version].ingress_ipv6_cidr_blocks
   }
 
   # Internal Traffic
@@ -59,10 +51,11 @@ resource "aws_security_group" "enos_vault_sg" {
   }
 
   egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = local.network_stack[var.ip_version].egress_cidr_blocks
+    ipv6_cidr_blocks = local.network_stack[var.ip_version].egress_ipv6_cidr_blocks
   }
 
   tags = merge(

enos/modules/aws_vault/templates/create-kms-policy.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+function retry {
+  local retries=$1
+  shift
+  local count=0
+
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+
+    if [ "$count" -lt "$retries" ]; then
+      sleep "$wait"
+    else
+      return "$exit"
+    fi
+  done
+
+  return 0
+}
+
+export VAULT_ADDR=http://localhost:8200
+export VAULT_TOKEN='${vault_root_token}'
+
+cat > ./kms-transit.hcl << EOF
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+path "transit/encrypt/boundary-recovery" {
+  capabilities = ["create", "update"]
+}
+
+path "transit/decrypt/boundary-recovery" {
+  capabilities = ["create", "update"]
+}
+
+path "transit/encrypt/boundary-worker-auth" {
+  capabilities = ["create", "update"]
+}
+
+path "transit/decrypt/boundary-worker-auth" {
+  capabilities = ["create", "update"]
+}
+
+path "transit/encrypt/boundary-root" {
+  capabilities = ["create", "update"]
+}
+
+path "transit/decrypt/boundary-root" {
+  capabilities = ["create", "update"]
+}
+
+path "transit/encrypt/boundary-bsr" {
+  capabilities = ["create", "update"]
+}
+
+path "transit/decrypt/boundary-bsr" {
+  capabilities = ["create", "update"]
+}
+EOF
+
+retry 5 ${vault_bin_path} secrets enable transit > /dev/null
+
+retry 5 ${vault_bin_path} policy write boundary-kms-transit-policy ./kms-transit.hcl  > /dev/null
+
+rm ./kms-transit.hcl > /dev/null
+
+retry 5 ${vault_bin_path} token create -policy=boundary-kms-transit-policy -field=token

enos/modules/aws_vault/variables.tf

@@ -71,6 +71,12 @@ variable "environment" {
   type        = string
 }
 
+variable "deploy" {
+  description = "Flag to toggle whether or not all the resources defined in this module should be deployed."
+  type        = bool
+  default     = false
+}
+
 variable "instance_count" {
   description = "Number of EC2 instances in each subnet"
   type        = number
@@ -101,7 +107,13 @@ variable "project_name" {
 }
 
 variable "sg_additional_ips" {
-  description = "A list of additional IPs to allow by Vault security groups (use with caution)"
+  description = "A list of additional IPv4 IPs to allow by Vault security groups (use with caution)"
+  type        = list(string)
+  default     = []
+}
+
+variable "sg_additional_ipv6_ips" {
+  description = "A list of additional IPv6 IPs to allow by Vault security groups (use with caution)"
   type        = list(string)
   default     = []
 }
@@ -253,3 +265,14 @@ variable "enable_file_audit_device" {
   type        = bool
   default     = true
 }
+
+variable "ip_version" {
+  type        = string
+  description = "Optional variable that configures the vault instance to run on a ipv4, ipv6, or dualstack network"
+  default     = "4"
+
+  validation {
+    condition     = contains(["4", "6", "dual"], var.ip_version)
+    error_message = "Valid values for ip_version are (4, 6, dual)."
+  }
+}

enos/modules/aws_vault/vault-instances.tf

@@ -5,10 +5,11 @@ resource "aws_instance" "vault_instance" {
   for_each               = local.vault_instances
   ami                    = var.ami_id
   instance_type          = var.instance_type
-  vpc_security_group_ids = [aws_security_group.enos_vault_sg.id]
+  vpc_security_group_ids = [aws_security_group.enos_vault_sg[0].id]
   subnet_id              = tolist(data.aws_subnets.infra.ids)[each.key % length(data.aws_subnets.infra.ids)]
   key_name               = var.ssh_aws_keypair
-  iam_instance_profile   = aws_iam_instance_profile.vault_profile.name
+  iam_instance_profile   = aws_iam_instance_profile.vault_profile[0].name
+  ipv6_address_count     = local.network_stack[var.ip_version].ipv6_address_count
   tags = merge(
     var.common_tags,
     {
@@ -31,7 +32,7 @@ resource "enos_remote_exec" "install_dependencies" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[each.value].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[each.value].ipv6_addresses[0] : aws_instance.vault_instance[each.value].public_ip
     }
   }
 }
@@ -47,7 +48,7 @@ resource "enos_bundle_install" "consul" {
 
   transport = {
     ssh = {
-      host = each.value.public_ip
+      host = var.ip_version == "6" ? each.value.ipv6_addresses[0] : each.value.public_ip
     }
   }
 }
@@ -62,7 +63,7 @@ resource "enos_bundle_install" "vault" {
 
   transport = {
     ssh = {
-      host = each.value.public_ip
+      host = var.ip_version == "6" ? each.value.ipv6_addresses[0] : each.value.public_ip
     }
   }
 }
@@ -88,7 +89,7 @@ resource "enos_consul_start" "consul" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[each.key].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[each.key].ipv6_addresses[0] : aws_instance.vault_instance[each.key].public_ip
     }
   }
 }
@@ -104,13 +105,13 @@ resource "enos_vault_start" "leader" {
   config_dir     = var.vault_config_dir
   manage_service = var.manage_service
   config = {
-    api_addr     = "http://${aws_instance.vault_instance[each.key].private_ip}:8200"
-    cluster_addr = "http://${aws_instance.vault_instance[each.key].private_ip}:8201"
+    api_addr     = var.ip_version == "4" ? "http://${aws_instance.vault_instance[each.key].private_ip}:8200" : "http://[${aws_instance.vault_instance[each.key].ipv6_addresses[0]}]:8200"
+    cluster_addr = var.ip_version == "4" ? "http://${aws_instance.vault_instance[each.key].private_ip}:8201" : "http://[${aws_instance.vault_instance[each.key].ipv6_addresses[0]}]:8201"
     cluster_name = local.vault_cluster_tag
     listener = {
       type = "tcp"
       attributes = {
-        address     = "0.0.0.0:8200"
+        address     = var.ip_version == "4" ? "0.0.0.0:8200" : "[::]:8200"
         tls_disable = "true"
       }
     }
@@ -130,7 +131,7 @@ resource "enos_vault_start" "leader" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[each.key].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[each.key].ipv6_addresses[0] : aws_instance.vault_instance[each.key].public_ip
     }
   }
 }
@@ -145,14 +146,14 @@ resource "enos_vault_start" "followers" {
   config_dir     = var.vault_config_dir
   manage_service = var.manage_service
   config = {
-    api_addr     = "http://${aws_instance.vault_instance[each.key].private_ip}:8200"
-    cluster_addr = "http://${aws_instance.vault_instance[each.key].private_ip}:8201"
+    api_addr     = var.ip_version == "4" ? "http://${aws_instance.vault_instance[each.key].private_ip}:8200" : "http://[${aws_instance.vault_instance[each.key].ipv6_addresses[0]}]:8200"
+    cluster_addr = var.ip_version == "4" ? "http://${aws_instance.vault_instance[each.key].private_ip}:8201" : "http://[${aws_instance.vault_instance[each.key].ipv6_addresses[0]}]:8201"
     cluster_name = local.vault_cluster_tag
     log_level    = var.vault_log_level
     listener = {
       type = "tcp"
       attributes = {
-        address     = "0.0.0.0:8200"
+        address     = var.ip_version == "4" ? "0.0.0.0:8200" : "[::]:8200"
         tls_disable = "true"
       }
     }
@@ -171,7 +172,7 @@ resource "enos_vault_start" "followers" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[each.key].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[each.key].ipv6_addresses[0] : aws_instance.vault_instance[each.key].public_ip
     }
   }
 }
@@ -194,7 +195,7 @@ resource "enos_vault_init" "leader" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[0].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[0].ipv6_addresses[0] : aws_instance.vault_instance[0].public_ip
     }
   }
 }
@@ -212,7 +213,7 @@ resource "enos_vault_unseal" "leader" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[0].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[0].ipv6_addresses[0] : aws_instance.vault_instance[0].public_ip
     }
   }
 }
@@ -237,7 +238,7 @@ resource "enos_remote_exec" "create_audit_log_dir" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[each.value].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[each.value].ipv6_addresses[0] : aws_instance.vault_instance[each.value].public_ip
     }
   }
 }
@@ -254,7 +255,7 @@ resource "enos_remote_exec" "init_audit_device" {
 
   environment = {
     VAULT_TOKEN    = enos_vault_init.leader[each.key].root_token
-    VAULT_ADDR     = "http://127.0.0.1:8200"
+    VAULT_ADDR     = var.ip_version == "4" ? "http://127.0.0.1:8200" : "http://[::1]:8200"
     VAULT_BIN_PATH = local.vault_bin_path
     LOG_FILE_PATH  = local.audit_device_file_path
     SERVICE_USER   = local.vault_service_user
@@ -266,7 +267,7 @@ resource "enos_remote_exec" "init_audit_device" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[each.key].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[each.key].ipv6_addresses[0] : aws_instance.vault_instance[each.key].public_ip
     }
   }
 }
@@ -289,7 +290,7 @@ resource "enos_vault_unseal" "followers" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[0].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[0].ipv6_addresses[0] : aws_instance.vault_instance[0].public_ip
     }
   }
 }
@@ -315,13 +316,13 @@ resource "enos_vault_unseal" "when_vault_unseal_when_no_init_is_set" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[each.key].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[each.key].ipv6_addresses[0] : aws_instance.vault_instance[each.key].public_ip
     }
   }
 }
 
 resource "enos_remote_exec" "vault_write_license" {
-  count = var.vault_init ? 1 : 0
+  count = var.deploy && var.vault_init ? 1 : 0
   depends_on = [
     enos_vault_unseal.leader,
     enos_vault_unseal.when_vault_unseal_when_no_init_is_set,
@@ -335,7 +336,26 @@ resource "enos_remote_exec" "vault_write_license" {
 
   transport = {
     ssh = {
-      host = aws_instance.vault_instance[0].public_ip
+      host = var.ip_version == "6" ? aws_instance.vault_instance[0].ipv6_addresses[0] : aws_instance.vault_instance[0].public_ip
+    }
+  }
+}
+
+resource "enos_remote_exec" "vault_kms_policy" {
+  count = var.deploy && var.vault_init ? 1 : 0
+  depends_on = [
+    enos_remote_exec.vault_write_license,
+  ]
+
+  content = templatefile("${path.module}/templates/create-kms-policy.sh", {
+    vault_bin_path   = local.vault_bin_path,
+    vault_root_token = coalesce(var.vault_root_token, try(enos_vault_init.leader[0].root_token, null), "none")
+    policy_path      = "${path.module}/policies/kms-transit.hcl"
+  })
+
+  transport = {
+    ssh = {
+      host = var.ip_version == "6" ? aws_instance.vault_instance[0].ipv6_addresses[0] : aws_instance.vault_instance[0].public_ip
     }
   }
 }