test(e2e): Add test for worker-led worker registration (#4014)

* refact(e2e): Create module to run boundary commands

* test(e2e): Add worker-led registration test

.github/workflows/enos-run.yml

@@ -83,6 +83,7 @@ jobs:
           - filter: 'e2e_docker_base_with_postgres builder:crt'
           - filter: 'e2e_docker_base_with_worker builder:crt'
           - filter: 'e2e_docker_worker_registration_controller_led builder:crt'
+          - filter: 'e2e_docker_worker_registration_worker_led builder:crt'
     runs-on: ${{ fromJSON(vars.RUNNER_LARGE) }}
     env:
       GITHUB_TOKEN: ${{ secrets.SERVICE_USER_GITHUB_TOKEN }}
@@ -251,7 +252,7 @@ jobs:
           enos scenario launch --timeout 60m0s --chdir ./enos ${{ matrix.filter }}
       - name: Rename e2e tests output
         continue-on-error: true
-        run : |
+        run: |
           pushd enos
           scenario="${{ matrix.filter }}"
           for f in *.log; do mv --  "$f" "${f%.log}_${scenario%% *}.log"; done

enos/enos-modules.hcl

@@ -147,6 +147,10 @@ module "docker_boundary" {
   source = "./modules/docker_boundary"
 }
 
+module "docker_boundary_cmd" {
+  source = "./modules/docker_boundary_cmd"
+}
+
 module "docker_openssh_server" {
   source = "./modules/docker_openssh_server"
 }
@@ -162,3 +166,7 @@ module "docker_worker" {
 module "docker_network" {
   source = "./modules/docker_network"
 }
+
+module "docker_check_health" {
+  source = "./modules/docker_check_health"
+}

enos/enos-scenario-e2e-docker-worker-registration-controller-led.hcl

@@ -104,8 +104,19 @@ scenario "e2e_docker_worker_registration_controller_led" {
       postgres_address = step.create_boundary_database.address
       boundary_license = var.boundary_edition != "oss" ? step.read_license.license : ""
       config_file      = "boundary-config.hcl"
-      get_auth_token   = true
-      get_worker_token = true
+    }
+  }
+
+  step "get_worker_token" {
+    module     = module.docker_boundary_cmd
+    depends_on = [step.create_boundary]
+    variables {
+      address      = step.create_boundary.address
+      image_name   = matrix.builder == "crt" ? var.boundary_docker_image_name : step.build_boundary_docker_image.image_name
+      network_name = local.network_cluster
+      login_name   = step.create_boundary.login_name
+      password     = step.create_boundary.password
+      script       = "get_worker_token.sh"
     }
   }
 
@@ -153,7 +164,7 @@ scenario "e2e_docker_worker_registration_controller_led" {
       network_name     = [local.network_cluster, local.network_host]
       tags             = [local.egress_tag]
       port             = "9402"
-      token            = step.create_boundary.worker_token
+      token            = step.get_worker_token.output["item"]["controller_generated_activation_token"]
     }
   }
 

enos/enos-scenario-e2e-docker-worker-registration-worker-led.hcl

@@ -0,0 +1,218 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# For this scenario to work, add the following line to /etc/hosts
+# 127.0.0.1 localhost boundary
+# 127.0.0.1 localhost worker
+
+scenario "e2e_docker_worker_registration_worker_led" {
+  terraform_cli = terraform_cli.default
+  terraform     = terraform.default
+  providers = [
+    provider.enos.default
+  ]
+
+  matrix {
+    builder = ["local", "crt"]
+  }
+
+  locals {
+    aws_ssh_private_key_path   = abspath(var.aws_ssh_private_key_path)
+    local_boundary_dir         = abspath(var.local_boundary_dir)
+    local_boundary_src_dir     = abspath(var.local_boundary_src_dir)
+    boundary_docker_image_file = abspath(var.boundary_docker_image_file)
+    license_path               = abspath(var.boundary_license_path != null ? var.boundary_license_path : joinpath(path.root, "./support/boundary.hclic"))
+
+    network_cluster  = "e2e_cluster"
+    network_host     = "e2e_host"
+    network_database = "e2e_db"
+
+    build_path = {
+      "local" = "/tmp",
+      "crt"   = var.crt_bundle_path == null ? null : abspath(var.crt_bundle_path)
+    }
+    tags = merge({
+      "Project Name" : var.project_name
+      "Project" : "Enos",
+      "Environment" : "ci"
+    }, var.tags)
+  }
+
+  step "build_boundary_docker_image" {
+    module = matrix.builder == "crt" ? module.build_boundary_docker_crt : module.build_boundary_docker_local
+
+    variables {
+      path           = matrix.builder == "crt" ? local.boundary_docker_image_file : ""
+      cli_build_path = local.build_path[matrix.builder]
+      edition        = var.boundary_edition
+    }
+  }
+
+  step "create_docker_network_database" {
+    module = module.docker_network
+    variables {
+      network_name = local.network_database
+    }
+  }
+
+  step "create_docker_network_cluster" {
+    module = module.docker_network
+    variables {
+      network_name = local.network_cluster
+    }
+  }
+
+  step "create_docker_network_host" {
+    module = module.docker_network
+    variables {
+      network_name = local.network_host
+    }
+  }
+
+  step "create_boundary_database" {
+    depends_on = [
+      step.create_docker_network_cluster
+    ]
+    variables {
+      image_name   = "${var.docker_mirror}/library/postgres:latest"
+      network_name = [local.network_database]
+    }
+    module = module.docker_postgres
+  }
+
+  step "read_license" {
+    skip_step = var.boundary_edition == "oss"
+    module    = module.read_license
+
+    variables {
+      file_name = local.license_path
+    }
+  }
+
+  step "create_boundary" {
+    module = module.docker_boundary
+    depends_on = [
+      step.create_docker_network_cluster,
+      step.create_docker_network_database,
+      step.create_boundary_database,
+      step.build_boundary_docker_image
+    ]
+    variables {
+      image_name       = matrix.builder == "crt" ? var.boundary_docker_image_name : step.build_boundary_docker_image.image_name
+      network_name     = [local.network_cluster, local.network_database]
+      database_network = local.network_database
+      postgres_address = step.create_boundary_database.address
+      boundary_license = var.boundary_edition != "oss" ? step.read_license.license : ""
+      config_file      = "boundary-config.hcl"
+    }
+  }
+
+  step "create_vault" {
+    module = module.docker_vault
+    depends_on = [
+      step.create_docker_network_cluster
+    ]
+    variables {
+      image_name   = "${var.docker_mirror}/hashicorp/vault:${var.vault_version}"
+      network_name = [local.network_cluster]
+    }
+  }
+
+  step "create_host" {
+    module = module.docker_openssh_server
+    depends_on = [
+      step.create_docker_network_host
+    ]
+    variables {
+      image_name            = "${var.docker_mirror}/linuxserver/openssh-server:latest"
+      network_name          = [local.network_host]
+      private_key_file_path = local.aws_ssh_private_key_path
+    }
+  }
+
+  locals {
+    egress_tag = "egress"
+  }
+
+  step "create_worker" {
+    module = module.docker_worker
+    depends_on = [
+      step.create_docker_network_cluster,
+      step.create_docker_network_host,
+      step.build_boundary_docker_image,
+      step.create_boundary
+    ]
+    variables {
+      image_name              = matrix.builder == "crt" ? var.boundary_docker_image_name : step.build_boundary_docker_image.image_name
+      boundary_license        = var.boundary_edition != "oss" ? step.read_license.license : ""
+      config_file             = "worker-config-worker-led.hcl"
+      container_name          = "worker"
+      initial_upstream        = step.create_boundary.upstream_address
+      network_name            = [local.network_cluster, local.network_host]
+      tags                    = [local.egress_tag]
+      port                    = "9402"
+      worker_led_registration = true
+    }
+  }
+
+  step "init_worker" {
+    module = module.docker_boundary_cmd
+    depends_on = [
+      step.create_docker_network_cluster,
+      step.create_boundary,
+      step.create_worker,
+    ]
+    variables {
+      address      = step.create_boundary.address
+      image_name   = matrix.builder == "crt" ? var.boundary_docker_image_name : step.build_boundary_docker_image.image_name
+      network_name = local.network_cluster
+      login_name   = step.create_boundary.login_name
+      password     = step.create_boundary.password
+      script       = "create_worker.sh"
+      worker_token = step.create_worker.worker_led_token
+    }
+  }
+
+  step "check_worker_health" {
+    module     = module.docker_check_health
+    depends_on = [step.init_worker]
+
+    variables {
+      container_name = "worker"
+    }
+  }
+
+  step "run_e2e_test" {
+    module = module.test_e2e_docker
+    depends_on = [
+      step.create_boundary,
+      step.create_vault,
+      step.create_host,
+      step.create_worker,
+      step.check_worker_health,
+    ]
+    variables {
+      test_package             = "github.com/hashicorp/boundary/testing/internal/e2e/tests/base_with_worker"
+      docker_mirror            = var.docker_mirror
+      network_name             = step.create_docker_network_cluster.network_name
+      go_version               = var.go_version
+      debug_no_run             = var.e2e_debug_no_run
+      alb_boundary_api_addr    = step.create_boundary.address
+      auth_method_id           = step.create_boundary.auth_method_id
+      auth_login_name          = step.create_boundary.login_name
+      auth_password            = step.create_boundary.password
+      local_boundary_dir       = step.build_boundary_docker_image.cli_zip_path
+      local_boundary_src_dir   = local.local_boundary_src_dir
+      aws_ssh_private_key_path = local.aws_ssh_private_key_path
+      target_address           = step.create_host.address
+      target_port              = step.create_host.port
+      target_user              = "ubuntu"
+      vault_addr               = step.create_vault.address
+      vault_addr_internal      = step.create_vault.address_internal
+      vault_root_token         = step.create_vault.token
+      vault_port               = step.create_vault.port
+      worker_tag_egress        = local.egress_tag
+      worker_tag_collocated    = step.create_boundary.worker_tag
+    }
+  }
+}

enos/modules/docker_boundary/main.tf

@@ -54,16 +54,6 @@ variable "worker_tag" {
   type        = string
   default     = "collocated"
 }
-variable "get_auth_token" {
-  description = "Flag to retrieve a boundary auth token"
-  type        = bool
-  default     = false
-}
-variable "get_worker_token" {
-  description = "Flag to retrieve a boundary worker token"
-  type        = bool
-  default     = false
-}
 
 resource "docker_image" "boundary" {
   name         = var.image_name
@@ -160,47 +150,6 @@ resource "enos_local_exec" "check_health" {
   inline = ["timeout 10s bash -c 'until curl -i http://0.0.0.0:9203/health; do sleep 2; done'"]
 }
 
-resource "enos_local_exec" "get_auth_token" {
-  count      = var.get_auth_token ? 1 : 0
-  depends_on = [enos_local_exec.check_health]
-  environment = {
-    TEST_BOUNDARY_IMAGE           = var.image_name
-    BOUNDARY_ADDR                 = local.address
-    TEST_NETWORK_NAME             = var.network_name[0]
-    E2E_PASSWORD_ADMIN_LOGIN_NAME = local.login_name
-    E2E_PASSWORD_ADMIN_PASSWORD   = local.password
-    MODULE_DIR                    = abspath(path.module)
-    SCRIPT                        = "${abspath(path.module)}/get_auth_token.sh"
-  }
-  inline = ["bash ./${path.module}/script_runner.sh"]
-}
-
-locals {
-  auth_info  = var.get_auth_token ? jsondecode(enos_local_exec.get_auth_token[0].stdout) : null
-  auth_token = var.get_auth_token ? local.auth_info["item"]["attributes"]["token"] : ""
-}
-
-resource "enos_local_exec" "get_worker_token" {
-  count      = var.get_worker_token ? 1 : 0
-  depends_on = [enos_local_exec.check_health]
-  environment = {
-    TEST_BOUNDARY_IMAGE           = var.image_name,
-    BOUNDARY_ADDR                 = local.address
-    TEST_NETWORK_NAME             = var.network_name[0]
-    E2E_PASSWORD_ADMIN_LOGIN_NAME = local.login_name
-    E2E_PASSWORD_ADMIN_PASSWORD   = local.password
-    MODULE_DIR                    = abspath(path.module)
-    SCRIPT                        = "${abspath(path.module)}/get_worker_token.sh"
-    BOUNDARY_TOKEN                = local.auth_token
-  }
-  inline = ["bash ./${path.module}/script_runner.sh"]
-}
-
-locals {
-  worker_info  = var.get_worker_token ? jsondecode(enos_local_exec.get_worker_token[0].stdout) : null
-  worker_token = var.get_worker_token ? local.worker_info["item"]["controller_generated_activation_token"] : ""
-}
-
 output "address" {
   value = local.address
 }
@@ -224,13 +173,3 @@ output "password" {
 output "worker_tag" {
   value = var.worker_tag
 }
-
-output "auth_token" {
-  value     = local.auth_token
-  sensitive = true
-}
-
-output "worker_token" {
-  value     = local.worker_token
-  sensitive = true
-}

enos/modules/docker_boundary_cmd/create_worker.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -eu -o pipefail
+
+boundary workers create worker-led \
+  -worker-generated-auth-token $WORKER_TOKEN \
+  -token env://BOUNDARY_TOKEN \
+  -format json

enos/modules/docker_boundary_cmd/main.tf

@@ -0,0 +1,78 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "address" {
+  description = "Address/URL to the Boundary controller/server"
+  type        = string
+}
+variable "image_name" {
+  description = "Name of docker image to use"
+  type        = string
+}
+variable "network_name" {
+  description = "Name of docker network to use"
+  type        = string
+}
+variable "login_name" {
+  description = "Login Name to log in to boundary"
+  type        = string
+}
+variable "password" {
+  description = "Password to log in to boundary"
+  type        = string
+}
+variable "script" {
+  description = "Filename of a script in the module directory to run"
+  type        = string
+}
+variable "worker_token" {
+  description = "Worker generated auth token"
+  type        = string
+  default     = ""
+}
+
+resource "enos_local_exec" "get_auth_token" {
+  environment = {
+    TEST_BOUNDARY_IMAGE           = var.image_name
+    BOUNDARY_ADDR                 = var.address
+    TEST_NETWORK_NAME             = var.network_name
+    E2E_PASSWORD_ADMIN_LOGIN_NAME = var.login_name
+    E2E_PASSWORD_ADMIN_PASSWORD   = var.password
+    MODULE_DIR                    = abspath(path.module)
+    SCRIPT                        = "${abspath(path.module)}/get_auth_token.sh"
+  }
+  inline = ["bash ./${path.module}/script_runner.sh"]
+}
+
+locals {
+  auth_info  = jsondecode(enos_local_exec.get_auth_token.stdout)
+  auth_token = local.auth_info["item"]["attributes"]["token"]
+}
+
+resource "enos_local_exec" "run_script" {
+  depends_on = [enos_local_exec.get_auth_token]
+  environment = {
+    TEST_BOUNDARY_IMAGE           = var.image_name
+    BOUNDARY_ADDR                 = var.address
+    TEST_NETWORK_NAME             = var.network_name
+    E2E_PASSWORD_ADMIN_LOGIN_NAME = var.login_name
+    E2E_PASSWORD_ADMIN_PASSWORD   = var.password
+    MODULE_DIR                    = abspath(path.module)
+    SCRIPT                        = "${abspath(path.module)}/${var.script}"
+    BOUNDARY_TOKEN                = local.auth_token
+    WORKER_TOKEN                  = var.worker_token
+  }
+  inline = ["bash ./${path.module}/script_runner.sh"]
+}
+
+output "output" {
+  value = jsondecode(enos_local_exec.run_script.stdout)
+}

enos/modules/docker_boundary_cmd/script_runner.sh

@@ -18,6 +18,7 @@ docker run \
     -e "LOGIN_NAME=$E2E_PASSWORD_ADMIN_LOGIN_NAME" \
     -e "BPASS=$E2E_PASSWORD_ADMIN_PASSWORD" \
     -e "BOUNDARY_TOKEN=$BOUNDARY_TOKEN" \
+    -e "WORKER_TOKEN=$WORKER_TOKEN" \
     -e "SKIP_CHOWN=true" \
     --cap-add IPC_LOCK \
     --network $TEST_NETWORK_NAME \

enos/modules/docker_check_health/main.tf

@@ -0,0 +1,18 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "container_name" {
+  description = "Name of docker container to inspect"
+}
+
+resource "enos_local_exec" "check_health" {
+  inline = ["timeout 10s bash -c 'until docker inspect ${var.container_name} --format={{.State.Health}} worker | grep {healthy; do sleep 2; done'"]
+}

enos/modules/docker_worker/main.tf

@@ -60,6 +60,11 @@ variable "token" {
   type        = string
   default     = ""
 }
+variable "worker_led_registration" {
+  description = "Enables worker-led registration flow"
+  type        = bool
+  default     = false
+}
 
 resource "docker_image" "boundary" {
   name         = var.image_name
@@ -117,7 +122,7 @@ resource "docker_container" "worker" {
     timeout  = "5s"
     retries  = 5
   }
-  wait     = true
+  wait     = var.worker_led_registration ? false : true
   must_run = true
   dynamic "networks_advanced" {
     for_each = var.network_name
@@ -127,7 +132,17 @@ resource "docker_container" "worker" {
   }
 }
 
+resource "enos_local_exec" "get_worker_led_token" {
+  count = var.worker_led_registration ? 1 : 0
+  depends_on = [
+    docker_container.worker
+  ]
+
+  inline = ["timeout 10s bash -c 'set -eo pipefail; until docker logs ${var.container_name} 2>&1 | grep \"Worker Auth Registration Request: .*\" | cut -f2- -d':' | xargs; do sleep 2; done'"]
+}
+
 resource "enos_local_exec" "check_address" {
+  count = var.worker_led_registration ? 0 : 1
   depends_on = [
     docker_container.worker
   ]
@@ -138,3 +153,7 @@ resource "enos_local_exec" "check_address" {
 output "upstream_address" {
   value = "${var.container_name}:${var.port}"
 }
+
+output "worker_led_token" {
+  value = var.worker_led_registration ? enos_local_exec.get_worker_led_token[0].stdout : ""
+}

enos/modules/docker_worker/worker-config-worker-led.hcl

@@ -0,0 +1,63 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+disable_mlock = true
+
+telemetry {
+  prometheus_retention_time = "24h"
+  disable_hostname          = true
+}
+
+listener "tcp" {
+  address     = "0.0.0.0:${port}"
+  purpose     = "proxy"
+  tls_disable = true
+}
+
+listener "tcp" {
+  address = "0.0.0.0:${port_ops}"
+  purpose = "ops"
+  tls_disable = true
+}
+
+worker {
+  public_addr = "${worker_name}:${port}"
+  initial_upstreams = ["${initial_upstream}"]
+
+  tags {
+    type = ${type_tags},
+  }
+
+  auth_storage_path = "/tmp/boundary/worker"
+}
+
+events {
+  audit_enabled       = true
+  sysevents_enabled   = true
+  observations_enable = true
+
+  sink "stderr" {
+    name        = "all-events"
+    description = "All events sent to stderr"
+    event_types = ["*"]
+    format      = "cloudevents-json"
+  }
+
+  sink {
+    name = "Log File"
+    event_types = ["*"]
+    format = "cloudevents-json"
+
+    file {
+      path = "/boundary/logs"
+      file_name = "events.log"
+    }
+
+    audit_config {
+      audit_filter_overrides {
+        secret    = "redact"
+        sensitive = "redact"
+      }
+    }
+  }
+}