test(e2e): Add support for Vault LDAP testing (#6175)

* chore(e2e): Update vault version that has vault ldap feature

* chore(e2e): Fix firewall and security group rules

* chore(e2e): Setup ldaps

* CR: spacing

enos/enos-scenario-e2e-aws-rdp-base.hcl

@@ -340,6 +340,10 @@ scenario "e2e_aws_rdp_base" {
     value = step.create_rdp_domain_controller.private_ip
   }
 
+  output "rdp_domain_controller_ipv6" {
+    value = step.create_rdp_domain_controller.ipv6
+  }
+
   output "rdp_domain_controller_admin_username" {
     value = step.create_rdp_domain_controller.admin_username
   }
@@ -403,4 +407,12 @@ scenario "e2e_aws_rdp_base" {
   output "windows_worker_private_ip" {
     value = step.create_windows_worker.private_ip
   }
+
+  output "vault_address_public" {
+    value = step.create_vault_cluster.instance_public_ips_ipv4[0]
+  }
+
+  output "vault_root_token" {
+    value = step.create_vault_cluster.vault_root_token
+  }
 }

enos/enos-variables.hcl

@@ -130,7 +130,7 @@ variable "vault_instance_type" {
 variable "vault_version" {
   description = "Version of Vault to use"
   type        = string
-  default     = "1.12.2"
+  default     = "1.17.6"
 }
 
 variable "test_email" {

enos/modules/aws_rdp_domain_controller/main.tf

@@ -79,7 +79,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -92,7 +93,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -106,7 +108,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -119,7 +122,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -133,7 +137,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -146,7 +151,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -160,7 +166,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -173,7 +180,8 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -187,7 +195,37 @@ resource "aws_security_group" "rdp_ingress" {
       join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
     ])
     ipv6_cidr_blocks = flatten([
-      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)]
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
+    ])
+  }
+
+  # Allow LDAPS (Lightweight Directory Access Protocol Secure) traffic to query Active Directory
+  ingress {
+    from_port = 636
+    to_port   = 636
+    protocol  = "tcp"
+    cidr_blocks = flatten([
+      formatlist("%s/32", data.enos_environment.current.public_ipv4_addresses),
+      join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
+    ])
+    ipv6_cidr_blocks = flatten([
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
+    ])
+  }
+
+  ingress {
+    from_port = 636
+    to_port   = 636
+    protocol  = "udp"
+    cidr_blocks = flatten([
+      formatlist("%s/32", data.enos_environment.current.public_ipv4_addresses),
+      join(",", data.aws_vpc.infra.cidr_block_associations.*.cidr_block),
+    ])
+    ipv6_cidr_blocks = flatten([
+      [for ip in coalesce(data.enos_environment.current.public_ipv6_addresses, []) : cidrsubnet("${ip}/64", 0, 0)],
+      data.aws_vpc.infra.ipv6_cidr_block
     ])
   }
 
@@ -225,10 +263,12 @@ resource "aws_security_group" "allow_all_internal" {
   vpc_id = var.vpc_id
 
   ingress {
-    from_port = 0
-    to_port   = 0
-    protocol  = "-1"
-    self      = true
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    self             = true
+    cidr_blocks      = [data.aws_vpc.infra.cidr_block]
+    ipv6_cidr_blocks = [data.aws_vpc.infra.ipv6_cidr_block]
   }
 
   egress {
@@ -280,6 +320,91 @@ resource "aws_instance" "domain_controller" {
                   # Force an immediate time synchronization
                   w32tm /resync /force
 
+                  # Set up SSH so we can remotely manage the instance
+                  # This is set up slightly different on the domain controller
+                  # due to issues when setting up SSH and creating a domain in
+                  # the same user_data script. Now, SSH is set up as a scheduled
+                  # task that will execute on next boot
+                  # Note: Windows Server 2016 does not support OpenSSH
+                  %{if var.server_version != "2016"~}
+                  $sshSetupScript = @'
+  # set variables for retry loops
+  $timeout = 300
+  $interval = 30
+  # Install OpenSSH Server and Client
+  # Loop to make sure that SSH installs correctly
+  $elapsed = 0
+  do {
+    try {
+      Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
+      Set-Service -Name sshd -StartupType 'Automatic'
+      Start-Service sshd
+      $result = Get-Process -Name "sshd" -ErrorAction SilentlyContinue
+      if ($result) {
+        Write-Host "Successfully added and started openSSH server"
+        break
+      }
+    } catch {
+        Write-Host "SSH server was not installed, retrying"
+        Start-Sleep -Seconds $interval
+        $elapsed += $interval
+    }
+    if ($elapsed -ge $timeout) {
+        Write-Host "SSH server installation failed after 5 minutes. Exiting."
+        exit 1
+    }
+  } while ($true)
+  $elapsed = 0
+  do {
+    try {
+      Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0
+      Set-Service -Name ssh-agent -StartupType Automatic
+      Start-Service ssh-agent
+      $result = Get-Process -Name "ssh-agent" -ErrorAction SilentlyContinue
+      if ($result) {
+          Write-Host "Successfully added and started openSSH agent"
+          break
+      }
+    } catch {
+        Write-Host "SSH server was not installed, retrying"
+        Start-Sleep -Seconds $interval
+        $elapsed += $interval
+    }
+    if ($elapsed -ge $timeout) {
+        Write-Host "SSH server installation failed after 5 minutes. Exiting."
+        exit 1
+    }
+  } while ($true)
+  # Set PowerShell as the default SSH shell
+  New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" -Name DefaultShell -Value (Get-Command powershell.exe).Path -PropertyType String -Force
+  # Configure SSH server to use private key authentication so that scripts don't have to use passwords
+  # Save the private key from instance metadata
+  $ImdsToken = (Invoke-WebRequest -Uri 'http://169.254.169.254/latest/api/token' -Method 'PUT' -Headers @{'X-aws-ec2-metadata-token-ttl-seconds' = 2160} -UseBasicParsing).Content
+  $ImdsHeaders = @{'X-aws-ec2-metadata-token' = $ImdsToken}
+  $AuthorizedKey = (Invoke-WebRequest -Uri 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' -Headers $ImdsHeaders -UseBasicParsing).Content
+  $AuthorizedKeysPath = 'C:\ProgramData\ssh\administrators_authorized_keys'
+  New-Item -Path $AuthorizedKeysPath -ItemType File -Value $AuthorizedKey -Force
+  # Set the correct permissions on the authorized_keys file
+  icacls "C:\ProgramData\ssh\administrators_authorized_keys" /inheritance:r
+  icacls "C:\ProgramData\ssh\administrators_authorized_keys" /grant "Administrators:F" /grant "SYSTEM:F"
+  icacls "C:\ProgramData\ssh\administrators_authorized_keys" /remove "Users"
+  icacls "C:\ProgramData\ssh\administrators_authorized_keys" /remove "Authenticated Users"
+  # Ensure the SSH agent pulls in the new key.
+  Set-Service -Name ssh-agent -StartupType "Automatic"
+  Restart-Service -Name ssh-agent
+  Restart-Service -Name sshd
+  # Open the firewall for SSH connections
+  New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22
+'@
+                  Set-Content -Path "C:\ssh-setup.ps1" -Value $sshSetupScript
+
+                  # Register a scheduled task to run the SSH setup script at next boot
+                  $Action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-NoProfile -ExecutionPolicy Bypass -File C:\ssh-setup.ps1"
+                  $Trigger = New-ScheduledTaskTrigger -AtStartup
+                  $Principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest
+                  Register-ScheduledTask -TaskName "SetupOpenSSH" -Action $Action -Trigger $Trigger -Principal $Principal;
+                  %{endif~}
+
                   # Open firewall ports for RDP functionality
                   New-NetFirewallRule -Name kerberostcp -DisplayName 'Kerberos TCP' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 88
                   New-NetFirewallRule -Name kerberosudp -DisplayName 'Kerberos UDP' -Enabled True -Direction Inbound -Protocol UDP -Action Allow -LocalPort 88
@@ -288,6 +413,8 @@ resource "aws_instance" "domain_controller" {
                   New-NetFirewallRule -Name ldaptcp -DisplayName 'LDAP TCP' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 389
                   New-NetFirewallRule -Name ldapudp -DisplayName 'LDAP UDP' -Enabled True -Direction Inbound -Protocol UDP -Action Allow -LocalPort 389
                   New-NetFirewallRule -Name smbtcp -DisplayName 'SMB TCP' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 445
+                  New-NetFirewallRule -Name ldapstcp -DisplayName 'LDAPS TCP' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 636
+                  New-NetFirewallRule -Name ldapsudp -DisplayName 'LDAPS UDP' -Enabled True -Direction Inbound -Protocol UDP -Action Allow -LocalPort 636
                   New-NetFirewallRule -Name rdptcp -DisplayName 'RDP TCP' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 3389
                   New-NetFirewallRule -Name rdpudp -DisplayName 'RDP UDP' -Enabled True -Direction Inbound -Protocol UDP -Action Allow -LocalPort 3389
 
@@ -328,3 +455,63 @@ resource "time_sleep" "wait_10_minutes" {
   depends_on      = [aws_instance.domain_controller]
   create_duration = "10m"
 }
+
+# wait for the SSH service to be available on the instance. We specifically use
+# BatchMode=Yes to prevent SSH from prompting for a password to ensure that we
+# can just SSH using the private key
+resource "enos_local_exec" "wait_for_ssh" {
+  depends_on = [time_sleep.wait_10_minutes]
+  count      = var.server_version != "2016" ? 1 : 0
+  inline     = ["timeout 600s bash -c 'until ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.domain_controller.public_ip} \"echo ready\"; do sleep 10; done'"]
+}
+
+locals {
+  test_dir        = "C:/Test"
+  vault_ldap_user = "VaultLDAP"
+}
+
+resource "enos_local_exec" "make_dir" {
+  depends_on = [
+    enos_local_exec.wait_for_ssh,
+  ]
+
+  count  = var.server_version != "2016" ? 1 : 0
+  inline = ["ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.domain_controller.public_ip} mkdir -Force ${local.test_dir}"]
+}
+
+resource "local_file" "ldaps_script" {
+  depends_on = [
+    enos_local_exec.make_dir,
+  ]
+  count = var.server_version != "2016" ? 1 : 0
+  content = templatefile("${path.module}/scripts/setup_ldaps.ps1", {
+    active_directory_domain = var.active_directory_domain
+    vault_ldap_user         = local.vault_ldap_user
+  })
+  filename = "${path.root}/.terraform/tmp/setup_ldaps.ps1"
+}
+
+resource "enos_local_exec" "add_ldaps_script" {
+  depends_on = [
+    local_file.ldaps_script,
+  ]
+
+  count  = var.server_version != "2016" ? 1 : 0
+  inline = ["scp -i ${abspath(local_sensitive_file.private_key.filename)} -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${abspath(local_file.ldaps_script[0].filename)} Administrator@${aws_instance.domain_controller.public_ip}:${local.test_dir}"]
+}
+
+resource "enos_local_exec" "run_ldaps_script" {
+  depends_on = [
+    enos_local_exec.add_ldaps_script,
+  ]
+
+  count  = var.server_version != "2016" ? 1 : 0
+  inline = ["ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.domain_controller.public_ip} ${local.test_dir}/${basename(local_file.ldaps_script[0].filename)}"]
+}
+
+resource "local_file" "ldaps_script_output" {
+  depends_on = [enos_local_exec.run_ldaps_script]
+  count      = var.server_version != "2016" ? 1 : 0
+  content    = enos_local_exec.run_ldaps_script[0].stdout
+  filename   = "${path.root}/.terraform/tmp/setup_ldaps.out"
+}

enos/modules/aws_rdp_domain_controller/outputs.tf

@@ -46,3 +46,8 @@ output "domain_name" {
   description = "The domain name the instance is joined to"
   value       = var.active_directory_domain
 }
+
+output "vault_ldap_user" {
+  description = "User created for Vault LDAP use"
+  value       = local.vault_ldap_user
+}

enos/modules/aws_rdp_domain_controller/scripts/setup_ldaps.ps1

@@ -0,0 +1,77 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# Wait for AD Web Services to be available
+$timeout = 600
+$elapsed = 0
+$interval = 15
+while ($elapsed -lt $timeout) {
+    try {
+        $domain = [System.DirectoryServices.ActiveDirectory.Domain]::GetCurrentDomain()
+        Write-Host "Domain is ready: $($domain.Name)"
+        break
+    } catch {
+        Write-Host "Domain not ready, waiting..."
+        Start-Sleep -Seconds $interval
+        $elapsed += $interval
+    }
+}
+if ($elapsed -ge $timeout) {
+    Write-Host "Timeout waiting for domain readiness"
+    exit 1
+}
+
+$dnsName = $env:computername + '.' + "${active_directory_domain}"
+$now = Get-Date
+
+# Create self-signed certificate for LDAPS. Since it's not signed by a trusted
+# CA, all LDAPS clients will have to allow insecure TLS.
+#
+# The -TextExtension is required. 2.5.29.37 is the OID for "Enhanced Key Usage"
+# and we need to set it to 1.3.6.1.5.5.7.3.1 which is the OID for server
+# authentication.
+$cert = New-SelfSignedCertificate `
+  -DnsName $dnsName `
+  -NotAfter $now.AddYears(1) `
+  -KeyUsage DigitalSignature, KeyEncipherment `
+  -Type SSLServerAuthentication `
+  -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.1") `
+  -Provider "Microsoft Strong Cryptographic Provider" `
+  -HashAlgorithm "SHA256"
+
+# Copy certificate thumbprint to NTDS service store so that AD can use it.
+$thumbprint = ($cert.Thumbprint | Out-String).Trim()
+$certDestPath = 'HKLM:/Software/Microsoft/Cryptography/Services/NTDS/SystemCertificates/My/Certificates'
+if (!(Test-Path $certDestPath)) { New-Item $certDestPath -Force }
+Copy-Item `
+  -Path HKLM:/Software/Microsoft/SystemCertificates/My/Certificates/$thumbprint `
+  -Destination $certDestPath
+
+# Signal NTDS that we have a new server certificate for use using an LDIF file.
+$sb = [System.Text.StringBuilder]::new()
+[void]$sb.AppendLine('dn:')
+[void]$sb.AppendLine('changetype: modify')
+[void]$sb.AppendLine('add: renewServerCertificate')
+[void]$sb.AppendLine('renewServerCertificate: 1')
+[void]$sb.AppendLine('-')
+
+$sb.ToString() | Out-File -FilePath "rsc.ldif" -Encoding ASCII
+ldifde -i -f rsc.ldif
+
+# Create Vault user and add it to the domain administrators group for RDP
+# access.
+New-ADUser `
+  -Enabled 1 `
+  -Name ${vault_ldap_user} `
+  -DisplayName ${vault_ldap_user} `
+  -SamAccountName ${vault_ldap_user} `
+  -PasswordNotRequired 1
+
+Add-ADGroupMember -Identity 'Domain Admins' -Members ${vault_ldap_user}
+
+# From here, a Vault LDAP engine can be configured to bind to the AD LDAPS
+# server using the domain controller's credentials and then a static role
+# for the Vault user above can be created. Additionally, a dynamic role
+# can also be created using LDIF templates.
+# The static/dynamic roles can then be used to RDP into the member server
+# via Boundary Vault LDAP credential library.

enos/modules/aws_rdp_member_server_with_worker/main.tf

@@ -215,7 +215,7 @@ ${var.domain_admin_password}
                     }
                   } while ($true)
 
-                  #logging to troubleshoot domain issues
+                  # Logging to troubleshoot domain issues
                   Resolve-DnsName -Name "${var.active_directory_domain}" -Server "${var.domain_controller_ip}" -ErrorAction SilentlyContinue
                   Get-Service -Name LanmanWorkstation, Netlogon, RpcSs | Select-Object Name, DisplayName, Status
 

enos/modules/aws_vault/security-groups.tf

@@ -44,10 +44,11 @@ resource "aws_security_group" "enos_vault_sg" {
 
   # Internal Traffic
   ingress {
-    from_port = 0
-    to_port   = 0
-    protocol  = "-1"
-    self      = true
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    self        = true
+    cidr_blocks = [data.aws_vpc.infra.cidr_block]
   }
 
   egress {

enos/modules/aws_windows_client/main.tf

@@ -106,10 +106,11 @@ resource "aws_security_group" "windows_client" {
 
   // Allow all traffic originating from the VPC
   ingress {
-    from_port = 0
-    to_port   = 0
-    protocol  = "-1"
-    self      = true
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    self        = true
+    cidr_blocks = [data.aws_vpc.infra.cidr_block]
   }
 
   egress {