test(e2e): Add an enos module for docker-based workers (#3603)

* test(e2e): Add an enos module for a worker in docker

* fixup! test(e2e): Add an enos module for a worker in docker

enos/enos-modules.hcl

@@ -158,6 +158,10 @@ module "docker_openssh_server_ca_key" {
   source = "./modules/docker_openssh_server_ca_key"
 }
 
+module "docker_worker" {
+  source = "./modules/docker_worker"
+}
+
 module "docker_network" {
   source = "./modules/docker_network"
 }

enos/enos-scenario-e2e-docker-base-with-vault.hcl

@@ -22,6 +22,8 @@ scenario "e2e_docker_base_with_vault" {
     boundary_docker_image_file = abspath(var.boundary_docker_image_file)
     license_path               = abspath(var.boundary_license_path != null ? var.boundary_license_path : joinpath(path.root, "./support/boundary.hclic"))
 
+    network_cluster = "e2e_cluster"
+
     build_path = {
       "local" = "/tmp",
       "crt"   = var.crt_bundle_path == null ? null : abspath(var.crt_bundle_path)
@@ -44,6 +46,9 @@ scenario "e2e_docker_base_with_vault" {
 
   step "create_docker_network" {
     module = module.docker_network
+    variables {
+      network_name = local.network_cluster
+    }
   }
 
   step "create_boundary_database" {
@@ -52,7 +57,7 @@ scenario "e2e_docker_base_with_vault" {
     ]
     variables {
       image_name   = "${var.docker_mirror}/library/postgres:latest"
-      network_name = step.create_docker_network.network_name
+      network_name = [local.network_cluster]
     }
     module = module.docker_postgres
   }
@@ -75,7 +80,8 @@ scenario "e2e_docker_base_with_vault" {
     ]
     variables {
       image_name       = matrix.builder == "crt" ? var.boundary_docker_image_name : step.build_boundary_docker_image.image_name
-      network_name     = step.create_docker_network.network_name
+      network_name     = [local.network_cluster]
+      database_network = local.network_cluster
       postgres_address = step.create_boundary_database.address
       boundary_license = var.boundary_edition != "oss" ? step.read_license.license : ""
     }
@@ -88,7 +94,7 @@ scenario "e2e_docker_base_with_vault" {
     ]
     variables {
       image_name   = "${var.docker_mirror}/hashicorp/vault:${var.vault_version}"
-      network_name = step.create_docker_network.network_name
+      network_name = [local.network_cluster]
     }
   }
 
@@ -99,7 +105,7 @@ scenario "e2e_docker_base_with_vault" {
     ]
     variables {
       image_name            = "${var.docker_mirror}/linuxserver/openssh-server:latest"
-      network_name          = step.create_docker_network.network_name
+      network_name          = [local.network_cluster]
       private_key_file_path = local.aws_ssh_private_key_path
     }
   }

enos/enos-scenario-e2e-docker-base.hcl

@@ -22,6 +22,8 @@ scenario "e2e_docker_base" {
     boundary_docker_image_file = abspath(var.boundary_docker_image_file)
     license_path               = abspath(var.boundary_license_path != null ? var.boundary_license_path : joinpath(path.root, "./support/boundary.hclic"))
 
+    network_cluster = "e2e_cluster"
+
     build_path = {
       "local" = "/tmp",
       "crt"   = var.crt_bundle_path == null ? null : abspath(var.crt_bundle_path)
@@ -44,6 +46,9 @@ scenario "e2e_docker_base" {
 
   step "create_docker_network" {
     module = module.docker_network
+    variables {
+      network_name = local.network_cluster
+    }
   }
 
   step "create_boundary_database" {
@@ -52,7 +57,7 @@ scenario "e2e_docker_base" {
     ]
     variables {
       image_name   = "${var.docker_mirror}/library/postgres:latest"
-      network_name = step.create_docker_network.network_name
+      network_name = [local.network_cluster]
     }
     module = module.docker_postgres
   }
@@ -75,7 +80,8 @@ scenario "e2e_docker_base" {
     ]
     variables {
       image_name       = matrix.builder == "crt" ? var.boundary_docker_image_name : step.build_boundary_docker_image.image_name
-      network_name     = step.create_docker_network.network_name
+      network_name     = [local.network_cluster]
+      database_network = local.network_cluster
       postgres_address = step.create_boundary_database.address
       boundary_license = var.boundary_edition != "oss" ? step.read_license.license : ""
     }
@@ -88,7 +94,7 @@ scenario "e2e_docker_base" {
     ]
     variables {
       image_name            = "${var.docker_mirror}/linuxserver/openssh-server:latest"
-      network_name          = step.create_docker_network.network_name
+      network_name          = [local.network_cluster]
       private_key_file_path = local.aws_ssh_private_key_path
     }
   }

enos/modules/docker_boundary/boundary-config-bsr.hcl

@@ -0,0 +1,60 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+disable_mlock = true
+
+controller {
+  name        = "docker-controller"
+
+  database {
+    url = "env://BOUNDARY_POSTGRES_URL"
+  }
+}
+
+listener "tcp" {
+  address     = "boundary:9200"
+  purpose     = "api"
+  tls_disable = true
+}
+
+listener "tcp" {
+  address     = "boundary:9201"
+  purpose     = "cluster"
+  tls_disable = true
+}
+
+listener "tcp" {
+  address     = "boundary:9203"
+  purpose     = "ops"
+  tls_disable = true
+}
+
+kms "aead" {
+  purpose   = "root"
+  aead_type = "aes-gcm"
+  key       = "sP1fnF5Xz85RrXyELHFeZg9Ad2qt4Z4bgNHVGtD6ung="
+  key_id    = "global_root"
+}
+
+# This key_id needs to match the corresponding downstream worker's
+# "worker-auth" kms
+kms "aead" {
+  purpose   = "worker-auth"
+  aead_type = "aes-gcm"
+  key       = "OLFhJNbEb3umRjdhY15QKNEmNXokY1Iq"
+  key_id    = "global_worker-auth"
+}
+
+kms "aead" {
+  purpose   = "recovery"
+  aead_type = "aes-gcm"
+  key       = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ="
+  key_id    = "global_recovery"
+}
+
+kms "aead" {
+  purpose   = "bsr"
+  aead_type = "aes-gcm"
+  key       = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ="
+  key_id    = "bsr_key"
+}

enos/modules/docker_boundary/boundary-config.hcl

@@ -48,10 +48,12 @@ kms "aead" {
   key_id    = "global_root"
 }
 
+# This key_id needs to match the corresponding downstream worker's
+# "worker-auth" kms
 kms "aead" {
   purpose   = "worker-auth"
   aead_type = "aes-gcm"
-  key       = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ="
+  key       = "OLFhJNbEb3umRjdhY15QKNEmNXokY1Iq"
   key_id    = "global_worker-auth"
 }
 

enos/modules/docker_boundary/init.sh

@@ -18,7 +18,7 @@ docker run \
     -e "BOUNDARY_LICENSE=$TEST_BOUNDARY_LICENSE" \
     -e "SKIP_CHOWN=true" \
     --cap-add IPC_LOCK \
-    --mount type=bind,src=$SOURCE,dst=/boundary/ \
-    --network $TEST_NETWORK_NAME \
+    -v "$CONFIG:/boundary/boundary-config.hcl" \
+    --network $TEST_DATABASE_NETWORK \
     $TEST_BOUNDARY_IMAGE \
     boundary database init -config /boundary/boundary-config.hcl -format json

enos/modules/docker_boundary/main.tf

@@ -24,7 +24,11 @@ variable "image_name" {
   type        = string
 }
 variable "network_name" {
-  description = "Name of Docker Network"
+  description = "Name of Docker Networks to join"
+  type        = list(string)
+}
+variable "database_network" {
+  description = "Name of Docker Network that database lives in"
   type        = string
 }
 variable "container_name" {
@@ -40,19 +44,24 @@ variable "boundary_license" {
   description = "License string"
   type        = string
 }
-
+variable "config_file" {
+  description = "Path to config file"
+  type        = string
+  default     = "boundary-config.hcl"
+}
 
 resource "docker_image" "boundary" {
   name         = var.image_name
-  keep_locally = true
+  keep_locally = false
 }
 
 resource "enos_local_exec" "init_database" {
   environment = {
-    TEST_BOUNDARY_IMAGE   = var.image_name,
-    TEST_DATABASE_ADDRESS = var.postgres_address,
-    TEST_NETWORK_NAME     = var.network_name
+    TEST_BOUNDARY_IMAGE   = var.image_name
+    TEST_DATABASE_ADDRESS = var.postgres_address
+    TEST_DATABASE_NETWORK = var.database_network
     TEST_BOUNDARY_LICENSE = var.boundary_license
+    CONFIG                = "${abspath(path.module)}/${var.config_file}"
   }
   inline = ["bash ./${path.module}/init.sh"]
 }
@@ -62,6 +71,7 @@ locals {
   auth_method_id = local.db_init_info["auth_method"]["auth_method_id"]
   login_name     = local.db_init_info["auth_method"]["login_name"]
   password       = local.db_init_info["auth_method"]["password"]
+  address        = "http://${var.container_name}:9200"
 }
 
 resource "docker_container" "boundary" {
@@ -96,9 +106,11 @@ resource "docker_container" "boundary" {
   capabilities {
     add = ["IPC_LOCK"]
   }
-  volumes {
-    host_path      = abspath(path.module)
-    container_path = "/boundary/"
+
+  upload {
+    content = templatefile("${abspath(path.module)}/${var.config_file}", {
+    })
+    file = "/boundary/boundary-config.hcl"
   }
   healthcheck {
     test     = ["CMD", "wget", "--quiet", "-O", "/dev/null", "http://boundary:9203/health"]
@@ -108,8 +120,11 @@ resource "docker_container" "boundary" {
   }
   wait     = true
   must_run = true
-  networks_advanced {
-    name = var.network_name
+  dynamic "networks_advanced" {
+    for_each = var.network_name
+    content {
+      name = networks_advanced.value
+    }
   }
 }
 
@@ -130,7 +145,11 @@ resource "enos_local_exec" "check_health" {
 }
 
 output "address" {
-  value = "http://${var.container_name}:9200"
+  value = local.address
+}
+
+output "upstream_address" {
+  value = "${var.container_name}:9201"
 }
 
 output "auth_method_id" {

enos/modules/docker_openssh_server/main.tf

@@ -25,8 +25,8 @@ variable "image_name" {
   default     = "docker.mirror.hashicorp.services/linuxserver/openssh-server:latest"
 }
 variable "network_name" {
-  description = "Name of Docker Network"
-  type        = string
+  description = "Name of Docker Networks to join"
+  type        = list(string)
 }
 variable "container_name" {
   description = "Name of Docker Container"
@@ -66,8 +66,11 @@ resource "docker_container" "openssh_server" {
     "USER_NAME=${var.target_user}",
     "PUBLIC_KEY=${local.public_key}",
   ]
-  networks_advanced {
-    name = var.network_name
+  dynamic "networks_advanced" {
+    for_each = var.network_name
+    content {
+      name = networks_advanced.value
+    }
   }
   ports {
     internal = 2222

enos/modules/docker_openssh_server_ca_key/main.tf

@@ -25,8 +25,8 @@ variable "image_name" {
   default     = "docker.mirror.hashicorp.services/linuxserver/openssh-server:latest"
 }
 variable "network_name" {
-  description = "Name of Docker Network"
-  type        = string
+  description = "Name of Docker Networks to join"
+  type        = list(string)
 }
 variable "container_name" {
   description = "Name of Docker Container"
@@ -76,8 +76,11 @@ resource "docker_container" "openssh_server" {
     "USER_NAME=${var.target_user}",
     "PUBLIC_KEY=${local.ssh_public_key}",
   ]
-  networks_advanced {
-    name = var.network_name
+  dynamic "networks_advanced" {
+    for_each = var.network_name
+    content {
+      name = networks_advanced.value
+    }
   }
   ports {
     internal = 2222

enos/modules/docker_postgres/main.tf

@@ -20,8 +20,8 @@ variable "image_name" {
   default     = "docker.mirror.hashicorp.services/library/postgres:latest"
 }
 variable "network_name" {
-  description = "Name of Docker Network"
-  type        = string
+  description = "Name of Docker Networks to join"
+  type        = list(string)
 }
 variable "container_name" {
   description = "Name of Docker Container"
@@ -74,8 +74,11 @@ resource "docker_container" "postgres" {
   }
   wait     = true
   must_run = true
-  networks_advanced {
-    name = var.network_name
+  dynamic "networks_advanced" {
+    for_each = var.network_name
+    content {
+      name = networks_advanced.value
+    }
   }
 }
 

enos/modules/docker_vault/main.tf

@@ -24,8 +24,8 @@ variable "image_name" {
   type        = string
 }
 variable "network_name" {
-  description = "Name of Docker Network"
-  type        = string
+  description = "Name of Docker Networks to join"
+  type        = list(string)
 }
 variable "container_name" {
   description = "Name of Docker Container"
@@ -61,8 +61,11 @@ resource "docker_container" "vault" {
   capabilities {
     add = ["IPC_LOCK"]
   }
-  networks_advanced {
-    name = var.network_name
+  dynamic "networks_advanced" {
+    for_each = var.network_name
+    content {
+      name = networks_advanced.value
+    }
   }
 }
 

enos/modules/docker_worker/main.tf

@@ -0,0 +1,120 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    docker = {
+      source  = "kreuzwerker/docker"
+      version = "3.0.1"
+    }
+
+    tls = {
+      source  = "hashicorp/tls"
+      version = "4.0.4"
+    }
+
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "image_name" {
+  description = "Name of Docker Image"
+  type        = string
+}
+variable "network_name" {
+  description = "Name of Docker Networks to join"
+  type        = list(string)
+}
+variable "container_name" {
+  description = "Name of Docker Container"
+  type        = string
+  default     = "worker"
+}
+variable "boundary_license" {
+  description = "License string"
+  type        = string
+}
+variable "initial_upstream" {
+  description = "Address to upstream instance that it communicates to"
+  type        = string
+}
+variable "port" {
+  description = "Port to use"
+  type        = number
+  default     = 9402
+}
+variable "tags" {
+  description = "Tags to set on worker for use in worker filters"
+  type        = list(string)
+  default     = ["e2e"]
+}
+variable "config_file" {
+  description = "Path to config file"
+  type        = string
+  default     = "worker-config.hcl"
+}
+
+resource "docker_image" "boundary" {
+  name         = var.image_name
+  keep_locally = true
+}
+
+locals {
+  recording_storage_path = "/recordings"
+}
+
+resource "docker_container" "worker" {
+  image   = docker_image.boundary.image_id
+  name    = var.container_name
+  command = ["boundary", "server", "-config", "/boundary/worker-config.hcl"]
+  env = [
+    "BOUNDARY_LICENSE=${var.boundary_license}",
+    "HOSTNAME=boundary",
+    "SKIP_CHOWN=true",
+  ]
+  ports {
+    internal = var.port
+    external = var.port
+  }
+  capabilities {
+    add = ["IPC_LOCK"]
+  }
+  mounts {
+    type   = "tmpfs"
+    target = local.recording_storage_path
+  }
+  mounts {
+    type   = "tmpfs"
+    target = "/boundary/logs"
+  }
+  upload {
+    content = templatefile("${abspath(path.module)}/${var.config_file}", {
+      worker_name            = var.container_name
+      initial_upstream       = var.initial_upstream
+      type_tags              = jsonencode(var.tags)
+      recording_storage_path = local.recording_storage_path
+      port                   = var.port
+    })
+    file = "/boundary/worker-config.hcl"
+  }
+  healthcheck {
+    test     = ["CMD", "grep", "-i", "worker has successfully authenticated", "/boundary/logs/events.log"]
+    interval = "3s"
+    timeout  = "5s"
+    retries  = 5
+  }
+  wait     = true
+  must_run = true
+  dynamic "networks_advanced" {
+    for_each = var.network_name
+    content {
+      name = networks_advanced.value
+    }
+  }
+}
+
+output "upstream_address" {
+  value = "${var.container_name}:${var.port}"
+}

enos/modules/docker_worker/worker-config-bsr-downstream.hcl

@@ -0,0 +1,66 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+disable_mlock = true
+
+telemetry {
+  prometheus_retention_time = "24h"
+  disable_hostname          = true
+}
+
+listener "tcp" {
+  address     = "0.0.0.0:${port}"
+  purpose     = "proxy"
+  tls_disable = true
+}
+
+worker {
+  name = "${worker_name}"
+  initial_upstreams = ["${initial_upstream}"]
+
+  tags {
+    type = ${type_tags},
+  }
+
+  recording_storage_path = "${recording_storage_path}"
+}
+
+# This key_id needs to match the corresponding upstream worker's
+# "downstream-worker-auth" kms
+kms "aead" {
+  purpose   = "worker-auth"
+  aead_type = "aes-gcm"
+  key       = "X+IJMVT6OnsrIR6G/9OTcJSX+lM9FSPN"
+  key_id    = "downstream_worker-auth"
+}
+
+events {
+  audit_enabled       = true
+  sysevents_enabled   = true
+  observations_enable = true
+
+  sink "stderr" {
+    name        = "all-events"
+    description = "All events sent to stderr"
+    event_types = ["*"]
+    format      = "cloudevents-json"
+  }
+
+  sink {
+    name = "Log File"
+    event_types = ["*"]
+    format = "cloudevents-json"
+
+    file {
+      path = "/boundary/logs"
+      file_name = "events.log"
+    }
+
+    audit_config {
+      audit_filter_overrides {
+        secret    = "redact"
+        sensitive = "redact"
+      }
+    }
+  }
+}

enos/modules/docker_worker/worker-config-bsr.hcl

@@ -0,0 +1,75 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+disable_mlock = true
+
+telemetry {
+  prometheus_retention_time = "24h"
+  disable_hostname          = true
+}
+
+listener "tcp" {
+  address     = "0.0.0.0:${port}"
+  purpose     = "proxy"
+  tls_disable = true
+}
+
+worker {
+  name = "${worker_name}"
+  initial_upstreams = ["${initial_upstream}"]
+
+  tags {
+    type = ${type_tags},
+  }
+
+  recording_storage_path = "${recording_storage_path}"
+}
+
+# This key_id needs to match the corresponding upstream controller's
+# "worker-auth" kms
+kms "aead" {
+  purpose   = "worker-auth"
+  aead_type = "aes-gcm"
+  key       = "OLFhJNbEb3umRjdhY15QKNEmNXokY1Iq"
+  key_id    = "global_worker-auth"
+}
+
+# This key_id needs to match the corresponding downstream worker's
+# "worker-auth" kms
+kms "aead" {
+  purpose   = "downstream-worker-auth"
+  aead_type = "aes-gcm"
+  key       = "X+IJMVT6OnsrIR6G/9OTcJSX+lM9FSPN"
+  key_id    = "downstream_worker-auth"
+}
+
+events {
+  audit_enabled       = true
+  sysevents_enabled   = true
+  observations_enable = true
+
+  sink "stderr" {
+    name        = "all-events"
+    description = "All events sent to stderr"
+    event_types = ["*"]
+    format      = "cloudevents-json"
+  }
+
+  sink {
+    name = "Log File"
+    event_types = ["*"]
+    format = "cloudevents-json"
+
+    file {
+      path = "/boundary/logs"
+      file_name = "events.log"
+    }
+
+    audit_config {
+      audit_filter_overrides {
+        secret    = "redact"
+        sensitive = "redact"
+      }
+    }
+  }
+}

enos/modules/docker_worker/worker-config-downstream.hcl

@@ -0,0 +1,64 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+disable_mlock = true
+
+telemetry {
+  prometheus_retention_time = "24h"
+  disable_hostname          = true
+}
+
+listener "tcp" {
+  address     = "0.0.0.0:${port}"
+  purpose     = "proxy"
+  tls_disable = true
+}
+
+worker {
+  name = "${worker_name}"
+  initial_upstreams = ["${initial_upstream}"]
+
+  tags {
+    type = ${type_tags},
+  }
+}
+
+# This key_id needs to match the corresponding upstream worker's
+# "downstream-worker-auth" kms
+kms "aead" {
+  purpose   = "worker-auth"
+  aead_type = "aes-gcm"
+  key       = "X+IJMVT6OnsrIR6G/9OTcJSX+lM9FSPN"
+  key_id    = "downstream_worker-auth"
+}
+
+events {
+  audit_enabled       = true
+  sysevents_enabled   = true
+  observations_enable = true
+
+  sink "stderr" {
+    name        = "all-events"
+    description = "All events sent to stderr"
+    event_types = ["*"]
+    format      = "cloudevents-json"
+  }
+
+  sink {
+    name = "Log File"
+    event_types = ["*"]
+    format = "cloudevents-json"
+
+    file {
+      path = "/boundary/logs"
+      file_name = "events.log"
+    }
+
+    audit_config {
+      audit_filter_overrides {
+        secret    = "redact"
+        sensitive = "redact"
+      }
+    }
+  }
+}

enos/modules/docker_worker/worker-config.hcl

@@ -0,0 +1,73 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+disable_mlock = true
+
+telemetry {
+  prometheus_retention_time = "24h"
+  disable_hostname          = true
+}
+
+listener "tcp" {
+  address     = "0.0.0.0:${port}"
+  purpose     = "proxy"
+  tls_disable = true
+}
+
+worker {
+  name = "${worker_name}"
+  initial_upstreams = ["${initial_upstream}"]
+
+  tags {
+    type = ${type_tags},
+  }
+}
+
+# This key_id needs to match the corresponding upstream controller's
+# "worker-auth" kms
+kms "aead" {
+  purpose   = "worker-auth"
+  aead_type = "aes-gcm"
+  key       = "OLFhJNbEb3umRjdhY15QKNEmNXokY1Iq"
+  key_id    = "global_worker-auth"
+}
+
+# This key_id needs to match the corresponding downstream worker's
+# "worker-auth" kms
+kms "aead" {
+  purpose   = "downstream-worker-auth"
+  aead_type = "aes-gcm"
+  key       = "X+IJMVT6OnsrIR6G/9OTcJSX+lM9FSPN"
+  key_id    = "downstream_worker-auth"
+}
+
+events {
+  audit_enabled       = true
+  sysevents_enabled   = true
+  observations_enable = true
+
+  sink "stderr" {
+    name        = "all-events"
+    description = "All events sent to stderr"
+    event_types = ["*"]
+    format      = "cloudevents-json"
+  }
+
+  sink {
+    name = "Log File"
+    event_types = ["*"]
+    format = "cloudevents-json"
+
+    file {
+      path = "/boundary/logs"
+      file_name = "events.log"
+    }
+
+    audit_config {
+      audit_filter_overrides {
+        secret    = "redact"
+        sensitive = "redact"
+      }
+    }
+  }
+}