From acda0c3892822d6315650f7dab1cde3f777d6af7 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Wed, 15 Mar 2023 10:07:36 -0400 Subject: [PATCH] chore(enos): Add scenario that uses docker containers for infrastructure (#3073) Co-authored-by: Johan Brandhorst-Satzkorn --- .github/workflows/enos-run.yml | 8 +- enos/enos-modules.hcl | 20 +++ enos/enos-scenario-e2e-docker.hcl | 104 ++++++++++++++ enos/enos-variables.hcl | 6 + .../docker_boundary/boundary-config.hcl | 81 +++++++++++ enos/modules/docker_boundary/init.sh | 25 ++++ enos/modules/docker_boundary/main.tf | 127 ++++++++++++++++++ enos/modules/docker_network/main.tf | 25 ++++ enos/modules/docker_openssh_server/main.tf | 75 +++++++++++ enos/modules/docker_postgres/main.tf | 75 +++++++++++ enos/modules/docker_postgres/postgresql.conf | 14 ++ enos/modules/docker_vault/main.tf | 87 ++++++++++++ .../database/testdata/boundary-config.hcl | 2 +- 13 files changed, 647 insertions(+), 2 deletions(-) create mode 100644 enos/enos-scenario-e2e-docker.hcl create mode 100644 enos/modules/docker_boundary/boundary-config.hcl create mode 100644 enos/modules/docker_boundary/init.sh create mode 100644 enos/modules/docker_boundary/main.tf create mode 100644 enos/modules/docker_network/main.tf create mode 100644 enos/modules/docker_openssh_server/main.tf create mode 100644 enos/modules/docker_postgres/main.tf create mode 100644 enos/modules/docker_postgres/postgresql.conf create mode 100644 enos/modules/docker_vault/main.tf diff --git a/.github/workflows/enos-run.yml b/.github/workflows/enos-run.yml index 0316a2175..600ffe4c4 100644 --- a/.github/workflows/enos-run.yml +++ b/.github/workflows/enos-run.yml @@ -156,9 +156,14 @@ jobs: run: | wget https://releases.hashicorp.com/vault/1.12.2/vault_1.12.2_linux_amd64.zip -O /tmp/test-deps/vault.zip - name: Install Vault CLI - if: matrix.filter == 'e2e_static_with_vault builder:crt' || matrix.filter == 'e2e_database' || matrix.filter == 'e2e_ui builder:crt' + if: matrix.filter == 'e2e_static_with_vault builder:crt' || matrix.filter == 'e2e_database' || matrix.filter == 'e2e_ui builder:crt' || matrix.filter == 'e2e_docker builder:crt' run: | unzip /tmp/test-deps/vault.zip -d /usr/local/bin + - name: Add hosts to /etc/hosts + # This enables the use of `boundary connect` with docker containers + if: matrix.filter == 'e2e_docker builder:crt' + run: | + sudo echo "127.0.0.1 localhost boundary" | sudo tee -a /etc/hosts - name: Download Boundary Linux AMD64 bundle id: download uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 @@ -232,6 +237,7 @@ jobs: with: name: test-e2e-ui-debug path: enos/support/boundary-ui/ui/admin/tests/e2e/artifacts/test-failures + retention-days: 5 - name: Retry Enos scenario id: run_retry if: steps.run.outcome == 'failure' diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 20aeb53bc..913777ee3 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -97,3 +97,23 @@ module "test_e2e_ui" { module "test_smoke" { source = "./modules/test_smoke" } + +module "docker_postgres" { + source = "./modules/docker_postgres" +} + +module "docker_vault" { + source = "./modules/docker_vault" +} + +module "docker_boundary" { + source = "./modules/docker_boundary" +} + +module "docker_openssh_server" { + source = "./modules/docker_openssh_server" +} + +module "docker_network" { + source = "./modules/docker_network" +} diff --git a/enos/enos-scenario-e2e-docker.hcl b/enos/enos-scenario-e2e-docker.hcl new file mode 100644 index 000000000..fe121e36a --- /dev/null +++ b/enos/enos-scenario-e2e-docker.hcl @@ -0,0 +1,104 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# For this scenario to work, add the following line to /etc/hosts +# 127.0.0.1 localhost boundary + +scenario "e2e_docker" { + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.default + ] + + locals { + aws_ssh_private_key_path = abspath(var.aws_ssh_private_key_path) + local_boundary_dir = abspath(var.local_boundary_dir) + tags = merge({ + "Project Name" : var.project_name + "Project" : "Enos", + "Environment" : "ci" + }, var.tags) + } + + step "create_docker_network" { + module = module.docker_network + } + + step "create_boundary_database" { + depends_on = [ + step.create_docker_network + ] + variables { + network_name = step.create_docker_network.network_name + } + module = module.docker_postgres + } + + step "create_boundary" { + module = module.docker_boundary + depends_on = [ + step.create_docker_network, + step.create_boundary_database + ] + variables { + image_name = var.boundary_docker_image + network_name = step.create_docker_network.network_name + postgres_address = step.create_boundary_database.address + } + } + + step "create_vault" { + module = module.docker_vault + depends_on = [ + step.create_docker_network + ] + variables { + image_name = "docker.io/hashicorp/vault:${var.vault_version}" + network_name = step.create_docker_network.network_name + } + } + + step "create_host" { + module = module.docker_openssh_server + depends_on = [ + step.create_docker_network + ] + variables { + network_name = step.create_docker_network.network_name + private_key_file_path = local.aws_ssh_private_key_path + } + } + + step "run_e2e_test" { + module = module.test_e2e + depends_on = [ + step.create_boundary, + step.create_vault, + step.create_host, + step.create_boundary_database + ] + + variables { + test_package = "github.com/hashicorp/boundary/testing/internal/e2e/tests/static_with_vault" + debug_no_run = var.e2e_debug_no_run + alb_boundary_api_addr = step.create_boundary.address + auth_method_id = step.create_boundary.auth_method_id + auth_login_name = step.create_boundary.login_name + auth_password = step.create_boundary.password + local_boundary_dir = local.local_boundary_dir + aws_ssh_private_key_path = local.aws_ssh_private_key_path + target_ip = step.create_host.address + target_port = step.create_host.port + target_user = "ubuntu" + vault_addr = step.create_vault.address + vault_addr_internal = step.create_vault.address_internal + vault_root_token = step.create_vault.token + } + } + + output "test_results" { + value = step.run_e2e_test.test_results + } +} diff --git a/enos/enos-variables.hcl b/enos/enos-variables.hcl index 24f331d8c..91f7cde80 100644 --- a/enos/enos-variables.hcl +++ b/enos/enos-variables.hcl @@ -25,6 +25,12 @@ variable "enos_user" { } # Test configs +variable "boundary_docker_image" { + description = "Name of Docker Image to use" + type = string + default = "docker.io/hashicorp/boundary:latest" +} + variable "worker_instance_type" { description = "EC2 Instance type" type = string diff --git a/enos/modules/docker_boundary/boundary-config.hcl b/enos/modules/docker_boundary/boundary-config.hcl new file mode 100644 index 000000000..6f9786f99 --- /dev/null +++ b/enos/modules/docker_boundary/boundary-config.hcl @@ -0,0 +1,81 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +disable_mlock = true + +controller { + name = "docker-controller" + + database { + url = "env://BOUNDARY_POSTGRES_URL" + } +} + +worker { + name = "boundary-colocated-worker" + description = "A worker that runs alongside the controller in the same process" + address = "boundary:9202" +} + +listener "tcp" { + address = "boundary:9200" + purpose = "api" + tls_disable = true +} + +listener "tcp" { + address = "boundary:9201" + purpose = "cluster" + tls_disable = true +} + +listener "tcp" { + address = "boundary:9202" + purpose = "proxy" + tls_disable = true +} + +listener "tcp" { + address = "boundary:9203" + purpose = "ops" + tls_disable = true +} + +kms "aead" { + purpose = "root" + aead_type = "aes-gcm" + key = "sP1fnF5Xz85RrXyELHFeZg9Ad2qt4Z4bgNHVGtD6ung=" + key_id = "global_root" +} + +kms "aead" { + purpose = "worker-auth" + aead_type = "aes-gcm" + key = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ=" + key_id = "global_worker-auth" +} + +kms "aead" { + purpose = "recovery" + aead_type = "aes-gcm" + key = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ=" + key_id = "global_recovery" +} + +events { + audit_enabled = true + observations_enabled = true + sysevents_enabled = true + + sink "stderr" { + name = "all-events" + description = "All events sent to stderr" + event_types = ["*"] + format = "cloudevents-json" + + deny_filters = [ + "\"/data/request_info/method\" contains \"Status\"", + "\"/data/request_info/path\" contains \"/health\"", + ] + } +} diff --git a/enos/modules/docker_boundary/init.sh b/enos/modules/docker_boundary/init.sh new file mode 100644 index 000000000..087f2d276 --- /dev/null +++ b/enos/modules/docker_boundary/init.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This script initializes a postgres database to work with Boundary by spinning up a temporary +# Boundary docker container pointed to the specified database and invoking `boundary database init`. +# +# This script must only output the JSON that comes from `boundary database init` as the output is +# consumed by other scripts. + +TEST_BOUNDARY_IMAGE=$1 +TEST_DATABASE_ADDRESS=$2 +TEST_CONTAINER_NAME=boundary-init +SOURCE=$(realpath $(dirname ${BASH_SOURCE[0]})) # get directory of this script + +docker run \ + --rm \ + --name $TEST_CONTAINER_NAME \ + -e "BOUNDARY_POSTGRES_URL=$TEST_DATABASE_ADDRESS" \ + -e "SKIP_CHOWN=true" \ + --cap-add IPC_LOCK \ + --mount type=bind,src=$SOURCE,dst=/boundary/ \ + --network e2e_network \ + $TEST_BOUNDARY_IMAGE \ + boundary database init -config /boundary/boundary-config.hcl -format json diff --git a/enos/modules/docker_boundary/main.tf b/enos/modules/docker_boundary/main.tf new file mode 100644 index 000000000..291a607f3 --- /dev/null +++ b/enos/modules/docker_boundary/main.tf @@ -0,0 +1,127 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "3.0.1" + } + + tls = { + source = "hashicorp/tls" + version = "4.0.4" + } + + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "image_name" { + description = "Name of Docker Image" + type = string +} +variable "network_name" { + description = "Name of Docker Network" + type = string +} +variable "container_name" { + description = "Name of Docker Container" + type = string + default = "boundary" +} +variable "postgres_address" { + description = "Address to postgres database" + type = string +} + + +resource "docker_image" "boundary" { + name = var.image_name + keep_locally = true +} + +resource "enos_local_exec" "init_database" { + inline = ["bash ./${path.module}/init.sh \"${var.image_name}\" \"${var.postgres_address}\""] +} + +locals { + db_init_info = jsondecode(enos_local_exec.init_database.stdout) + auth_method_id = local.db_init_info["auth_method"]["auth_method_id"] + login_name = local.db_init_info["auth_method"]["login_name"] + password = local.db_init_info["auth_method"]["password"] +} + +resource "docker_container" "boundary" { + depends_on = [ + enos_local_exec.init_database, + ] + image = docker_image.boundary.image_id + name = var.container_name + command = ["boundary", "server", "-config", "/boundary/boundary-config.hcl"] + env = [ + "BOUNDARY_POSTGRES_URL=${var.postgres_address}", + "HOSTNAME=boundary", + "SKIP_CHOWN=true", + ] + ports { + internal = 9200 + external = 9200 + } + ports { + internal = 9201 + external = 9201 + } + ports { + internal = 9202 + external = 9202 + } + ports { + internal = 9203 + external = 9203 + } + capabilities { + add = ["IPC_LOCK"] + } + volumes { + host_path = abspath(path.module) + container_path = "/boundary/" + } + healthcheck { + test = ["CMD", "wget", "--quiet", "-O", "/dev/null", "http://boundary:9203/health"] + interval = "3s" + timeout = "5s" + retries = 5 + } + wait = true + must_run = true + networks_advanced { + name = var.network_name + } +} + +resource "enos_local_exec" "wait" { + depends_on = [ + docker_container.boundary + ] + + inline = ["timeout 10s bash -c 'until curl http://0.0.0.0:9200; do sleep 2; done'"] +} + +output "address" { + value = "http://0.0.0.0:9200" +} + +output "auth_method_id" { + value = local.auth_method_id +} + +output "login_name" { + value = local.login_name +} + +output "password" { + value = local.password +} diff --git a/enos/modules/docker_network/main.tf b/enos/modules/docker_network/main.tf new file mode 100644 index 000000000..b617e400d --- /dev/null +++ b/enos/modules/docker_network/main.tf @@ -0,0 +1,25 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "3.0.1" + } + } +} + +variable "network_name" { + description = "Name of Docker network" + type = string + default = "e2e_network" +} + +resource "docker_network" "test_network" { + name = var.network_name +} + +output "network_name" { + value = var.network_name +} diff --git a/enos/modules/docker_openssh_server/main.tf b/enos/modules/docker_openssh_server/main.tf new file mode 100644 index 000000000..3b201233e --- /dev/null +++ b/enos/modules/docker_openssh_server/main.tf @@ -0,0 +1,75 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "3.0.1" + } + + tls = { + source = "hashicorp/tls" + version = "4.0.4" + } + } +} + +variable "network_name" { + description = "Name of Docker Network" + type = string +} +variable "container_name" { + description = "Name of Docker Container" + type = string + default = "openssh-server" +} +variable "target_user" { + description = "SSH username for target" + type = string + default = "ubuntu" +} +variable "private_key_file_path" { + description = "Local Path to key used to SSH onto created hosts" + type = string +} + +data "tls_public_key" "host_key_openssh" { + private_key_openssh = file(var.private_key_file_path) +} + +locals { + public_key = data.tls_public_key.host_key_openssh.public_key_openssh +} + +resource "docker_image" "openssh_server" { + name = "lscr.io/linuxserver/openssh-server" + keep_locally = true +} + +resource "docker_container" "openssh_server" { + image = docker_image.openssh_server.image_id + name = var.container_name + env = [ + "PUID=1000", + "PGID=1000", + "TZ=US/Eastern", + "USER_NAME=${var.target_user}", + "PUBLIC_KEY=${local.public_key}", + ] + networks_advanced { + name = var.network_name + } +} + +output "user" { + value = var.target_user +} + +output "address" { + value = docker_container.openssh_server.network_data[0].ip_address +} + +output "port" { + value = "2222" +} diff --git a/enos/modules/docker_postgres/main.tf b/enos/modules/docker_postgres/main.tf new file mode 100644 index 000000000..ed2d31785 --- /dev/null +++ b/enos/modules/docker_postgres/main.tf @@ -0,0 +1,75 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "3.0.1" + } + } +} + +variable "network_name" { + description = "Name of Docker Network" + type = string +} +variable "container_name" { + description = "Name of Docker Container" + type = string + default = "database" +} +variable "user" { + description = "Postgres Database username" + type = string + default = "boundary" +} +variable "password" { + description = "Postgres Database password" + type = string + default = "boundary" +} +variable "database_name" { + description = "Postgres Database name" + type = string + default = "boundarydb" +} + +resource "docker_image" "postgres" { + name = "postgres:latest" + keep_locally = true +} + +resource "docker_container" "postgres" { + image = docker_image.postgres.image_id + name = var.container_name + command = ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"] + env = [ + "POSTGRES_DB=${var.database_name}", + "POSTGRES_USER=${var.user}", + "POSTGRES_PASSWORD=${var.password}", + ] + volumes { + host_path = abspath(path.module) + container_path = "/etc/postgresql/" + } + ports { + internal = 5432 + external = 5432 + } + healthcheck { + test = ["CMD", "pg_isready", "-U", "${var.user}", "-d", "${var.database_name}"] + interval = "3s" + timeout = "5s" + retries = 5 + } + wait = true + must_run = true + networks_advanced { + name = var.network_name + } +} + +output "address" { + value = "postgres://${var.user}:${var.password}@${var.container_name}:5432/${var.database_name}?sslmode=disable" +} diff --git a/enos/modules/docker_postgres/postgresql.conf b/enos/modules/docker_postgres/postgresql.conf new file mode 100644 index 000000000..cfdd02830 --- /dev/null +++ b/enos/modules/docker_postgres/postgresql.conf @@ -0,0 +1,14 @@ +listen_addresses = '*' +max_connections = 100 +work_mem = 50000kB +shared_buffers = 64MB +log_destination = 'stderr' +deadlock_timeout = 100 +log_min_messages = info +log_min_error_statement = error +log_error_verbosity = verbose +log_lock_waits = true +log_statement = all +log_duration = on +log_min_duration_statement = 0 +log_line_prefix = '%m [%p] (%x) %i ' diff --git a/enos/modules/docker_vault/main.tf b/enos/modules/docker_vault/main.tf new file mode 100644 index 000000000..7c607239a --- /dev/null +++ b/enos/modules/docker_vault/main.tf @@ -0,0 +1,87 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "3.0.1" + } + + tls = { + source = "hashicorp/tls" + version = "4.0.4" + } + + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + } + } +} + +variable "image_name" { + description = "Name of Docker Image" + type = string +} +variable "network_name" { + description = "Name of Docker Network" + type = string +} +variable "container_name" { + description = "Name of Docker Container" + type = string + default = "vault" +} +variable "vault_token" { + description = "Vault Root Token" + type = string + default = "boundarytok" +} + +resource "docker_image" "vault" { + name = var.image_name + keep_locally = true +} + +resource "docker_container" "vault" { + image = docker_image.vault.image_id + name = var.container_name + env = [ + "VAULT_DEV_ROOT_TOKEN_ID=${var.vault_token}" + ] + ports { + internal = 8200 + external = 8200 + } + capabilities { + add = ["IPC_LOCK"] + } + networks_advanced { + name = var.network_name + } +} + +resource "enos_local_exec" "wait" { + depends_on = [ + docker_container.vault + ] + + environment = { + VAULT_ADDR = "http://0.0.0.0:8200" + VAULT_TOKEN = var.vault_token + } + + inline = ["timeout 10s bash -c 'until vault status; do sleep 2; done'"] +} + +output "address" { + value = "0.0.0.0" +} + +output "address_internal" { + value = var.container_name +} + +output "token" { + value = var.vault_token +} diff --git a/testing/internal/e2e/tests/database/testdata/boundary-config.hcl b/testing/internal/e2e/tests/database/testdata/boundary-config.hcl index c959787cd..1dbe47e5a 100644 --- a/testing/internal/e2e/tests/database/testdata/boundary-config.hcl +++ b/testing/internal/e2e/tests/database/testdata/boundary-config.hcl @@ -15,7 +15,7 @@ controller { worker { name = "boundary-colocated-worker" description = "A worker that runs alongside the controller in the same process" - address = "boundary:9201" + address = "boundary:9202" } listener "tcp" {