test(e2e): Improve reliability of enos docker scenario (#3167)

enos/enos-scenario-e2e-docker.hcl

@@ -48,7 +48,7 @@ scenario "e2e_docker" {
       step.create_docker_network
     ]
     variables {
-      image_name   = "docker.mirror.hashicorp.services/library/postgres:latest"
+      image_name   = "${var.docker_mirror}/library/postgres:latest"
       network_name = step.create_docker_network.network_name
     }
     module = module.docker_postgres
@@ -74,7 +74,7 @@ scenario "e2e_docker" {
       step.create_docker_network
     ]
     variables {
-      image_name   = "docker.mirror.hashicorp.services/hashicorp/vault:${var.vault_version}"
+      image_name   = "${var.docker_mirror}/hashicorp/vault:${var.vault_version}"
       network_name = step.create_docker_network.network_name
     }
   }
@@ -85,7 +85,7 @@ scenario "e2e_docker" {
       step.create_docker_network
     ]
     variables {
-      image_name            = "docker.mirror.hashicorp.services/linuxserver/openssh-server"
+      image_name            = "${var.docker_mirror}/linuxserver/openssh-server:latest"
       network_name          = step.create_docker_network.network_name
       private_key_file_path = local.aws_ssh_private_key_path
     }

enos/enos-variables.hcl

@@ -148,3 +148,9 @@ variable "e2e_debug_no_run" {
   type        = bool
   default     = false
 }
+
+variable "docker_mirror" {
+  description = ""
+  type        = string
+  default     = "docker.mirror.hashicorp.services"
+}

enos/modules/docker_boundary/init.sh

@@ -8,8 +8,6 @@
 # This script must only output the JSON that comes from `boundary database init` as the output is
 # consumed by other scripts.
 
-TEST_BOUNDARY_IMAGE=$1
-TEST_DATABASE_ADDRESS=$2
 TEST_CONTAINER_NAME=boundary-init
 SOURCE=$(realpath $(dirname ${BASH_SOURCE[0]})) # get directory of this script
 
@@ -20,6 +18,6 @@ docker run \
     -e "SKIP_CHOWN=true" \
     --cap-add IPC_LOCK \
     --mount type=bind,src=$SOURCE,dst=/boundary/ \
-    --network e2e_network \
+    --network $TEST_NETWORK_NAME \
     $TEST_BOUNDARY_IMAGE \
     boundary database init -config /boundary/boundary-config.hcl -format json

enos/modules/docker_boundary/main.tf

@@ -44,7 +44,12 @@ resource "docker_image" "boundary" {
 }
 
 resource "enos_local_exec" "init_database" {
-  inline = ["bash ./${path.module}/init.sh \"${var.image_name}\" \"${var.postgres_address}\""]
+  environment = {
+    TEST_BOUNDARY_IMAGE   = var.image_name,
+    TEST_DATABASE_ADDRESS = var.postgres_address,
+    TEST_NETWORK_NAME     = var.network_name
+  }
+  inline = ["bash ./${path.module}/init.sh"]
 }
 
 locals {
@@ -102,7 +107,7 @@ resource "docker_container" "boundary" {
   }
 }
 
-resource "enos_local_exec" "wait" {
+resource "enos_local_exec" "check_address" {
   depends_on = [
     docker_container.boundary
   ]
@@ -110,6 +115,14 @@ resource "enos_local_exec" "wait" {
   inline = ["timeout 10s bash -c 'until curl http://0.0.0.0:9200; do sleep 2; done'"]
 }
 
+resource "enos_local_exec" "check_health" {
+  depends_on = [
+    enos_local_exec.check_address
+  ]
+
+  inline = ["timeout 10s bash -c 'until curl -i http://0.0.0.0:9203/health; do sleep 2; done'"]
+}
+
 output "address" {
   value = "http://0.0.0.0:9200"
 }

enos/modules/docker_openssh_server/main.tf

@@ -12,13 +12,17 @@ terraform {
       source  = "hashicorp/tls"
       version = "4.0.4"
     }
+
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
   }
 }
 
 variable "image_name" {
   description = "Name of Docker Image"
   type        = string
-  default     = "docker.mirror.hashicorp.services/linuxserver/openssh-server"
+  default     = "docker.mirror.hashicorp.services/linuxserver/openssh-server:latest"
 }
 variable "network_name" {
   description = "Name of Docker Network"
@@ -65,6 +69,18 @@ resource "docker_container" "openssh_server" {
   networks_advanced {
     name = var.network_name
   }
+  ports {
+    internal = 2222
+    external = 2222
+  }
+}
+
+resource "enos_local_exec" "wait" {
+  depends_on = [
+    docker_container.openssh_server
+  ]
+
+  inline = ["timeout 10s bash -c 'until ssh -i ${var.private_key_file_path} -p 2222 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${var.target_user}@localhost; do sleep 2; done'"]
 }
 
 output "user" {

enos/modules/docker_postgres/main.tf

@@ -7,6 +7,10 @@ terraform {
       source  = "kreuzwerker/docker"
       version = "3.0.1"
     }
+
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
   }
 }
 
@@ -75,6 +79,14 @@ resource "docker_container" "postgres" {
   }
 }
 
+resource "enos_local_exec" "wait" {
+  depends_on = [
+    docker_container.postgres
+  ]
+
+  inline = ["timeout 10s bash -c 'until docker exec ${var.container_name} pg_isready; do sleep 2; done'"]
+}
+
 output "address" {
   value = "postgres://${var.user}:${var.password}@${var.container_name}:5432/${var.database_name}?sslmode=disable"
 }

enos/modules/docker_vault/main.tf

@@ -61,11 +61,19 @@ resource "docker_container" "vault" {
   }
 }
 
-resource "enos_local_exec" "wait" {
+resource "enos_local_exec" "check_address" {
   depends_on = [
     docker_container.vault
   ]
 
+  inline = ["timeout 10s bash -c 'until curl http://0.0.0.0:8200; do sleep 2; done'"]
+}
+
+resource "enos_local_exec" "check_health" {
+  depends_on = [
+    enos_local_exec.check_address
+  ]
+
   environment = {
     VAULT_ADDR  = "http://0.0.0.0:8200"
     VAULT_TOKEN = var.vault_token