ce: Add infrastructure needed to test worker ssh known hosts (#6554)

(cherry picked from commit d022d7daf62623e5cca9c45cd234fd980d5baa9c)

enos/modules/docker_openssh_server_ca_key/custom-cont-init.d/02-add-host-keys

@@ -0,0 +1,15 @@
+#!/usr/bin/with-contenv bash
+# Copyright IBM Corp. 2020, 2025
+# SPDX-License-Identifier: BUSL-1.1
+
+chown 1000:1000 /etc/ssh/host-key
+chmod 400 /etc/ssh/host-key
+
+if ! grep -qE '^HostKey[[:space:]]+/etc/ssh/host-key$' /config/sshd/sshd_config 2>/dev/null; then
+  echo HostKey /etc/ssh/host-key >> /config/sshd/sshd_config
+fi
+
+if ! grep -qE '^HostCertificate[[:space:]]+/etc/ssh/host-key-cert.pub$' /config/sshd/sshd_config 2>/dev/null; then
+  echo HostCertificate /etc/ssh/host-key-cert.pub >> /config/sshd/sshd_config
+fi
+

enos/modules/docker_openssh_server_ca_key/main.tf

@@ -43,7 +43,7 @@ variable "private_key_file_path" {
   type        = string
 }
 
-data "tls_public_key" "host_key_openssh" {
+data "tls_public_key" "ssh_auth_key" {
   private_key_openssh = file(var.private_key_file_path)
 }
 
@@ -56,9 +56,30 @@ data "tls_public_key" "ca_key" {
   private_key_openssh = tls_private_key.ca_key.private_key_openssh
 }
 
-locals {
-  ssh_public_key = data.tls_public_key.host_key_openssh.public_key_openssh
-  ca_public_key  = data.tls_public_key.ca_key.public_key_openssh
+# host keys are used for host validation in the ssh client, but are not used by the server for authentication
+resource "tls_private_key" "host_key" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+data "tls_public_key" "host_key" {
+  private_key_openssh = tls_private_key.host_key.private_key_openssh
+}
+
+resource "local_sensitive_file" "ca_key" {
+  depends_on = [tls_private_key.ca_key]
+
+  content         = tls_private_key.ca_key.private_key_openssh
+  filename        = "${path.root}/.terraform/tmp/ca-key"
+  file_permission = "0400"
+}
+
+resource "local_sensitive_file" "host_public_key" {
+  depends_on = [tls_private_key.host_key]
+
+  content         = data.tls_public_key.host_key.public_key_openssh
+  filename        = "${path.root}/.terraform/tmp/host-key.pub"
+  file_permission = "0644"
 }
 
 data "docker_registry_image" "openssh" {
@@ -79,7 +100,7 @@ resource "docker_container" "openssh_server" {
     "PGID=1000",
     "TZ=US/Eastern",
     "USER_NAME=${var.target_user}",
-    "PUBLIC_KEY=${local.ssh_public_key}",
+    "PUBLIC_KEY=${data.tls_public_key.ssh_auth_key.public_key_openssh}",
     "SUDO_ACCESS=true",
   ]
   network_mode = "bridge"
@@ -102,9 +123,17 @@ resource "docker_container" "openssh_server" {
     file           = "/ca/ca-key"
   }
   upload {
-    content_base64 = base64encode(local.ca_public_key)
+    content_base64 = base64encode(data.tls_public_key.ca_key.public_key_openssh)
     file           = "/ca/ca-key.pub"
   }
+  upload {
+    content_base64 = base64encode(tls_private_key.host_key.private_key_openssh)
+    file           = "/etc/ssh/host-key"
+  }
+  upload {
+    content_base64 = base64encode(data.tls_public_key.host_key.public_key_openssh)
+    file           = "/etc/ssh/host-key.pub"
+  }
 }
 
 resource "enos_local_exec" "wait" {
@@ -115,6 +144,45 @@ resource "enos_local_exec" "wait" {
   inline = ["timeout 60s bash -c 'until ssh -t -t -i ${var.private_key_file_path} -p 2222 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${var.target_user}@localhost hostname; do sleep 2; done'"]
 }
 
+# this host key needs to be created after the container is created
+resource "enos_local_exec" "sign_host_key" {
+  depends_on = [
+    local_sensitive_file.ca_key,
+    local_sensitive_file.host_public_key
+  ]
+
+  inline = ["ssh-keygen -s ${local_sensitive_file.ca_key.filename} -I host-key -h -n ${docker_container.openssh_server.network_data[0].ip_address},${var.container_name} -V +52w ${local_sensitive_file.host_public_key.filename}"]
+}
+
+locals {
+  signed_host_key_path = "${trimsuffix(local_sensitive_file.host_public_key.filename, ".pub")}-cert.pub"
+}
+
+data "local_file" "signed_host_key" {
+  depends_on = [enos_local_exec.sign_host_key]
+  filename   = local.signed_host_key_path
+}
+
+resource "enos_local_exec" "copy_signed_host_key" {
+  depends_on = [data.local_file.signed_host_key]
+
+  inline = ["docker cp ${data.local_file.signed_host_key.filename} ${var.container_name}:/etc/ssh/host-key-cert.pub"]
+}
+
+resource "enos_local_exec" "restart_container_for_ssh_changes" {
+  depends_on = [enos_local_exec.copy_signed_host_key]
+
+  inline = ["docker restart ${var.container_name}"]
+}
+
+resource "enos_local_exec" "wait_after_restart" {
+  depends_on = [
+    enos_local_exec.restart_container_for_ssh_changes
+  ]
+
+  inline = ["timeout 60s bash -c 'until ssh -t -t -i ${var.private_key_file_path} -p 2222 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${var.target_user}@localhost hostname; do sleep 2; done'"]
+}
+
 output "user" {
   value = var.target_user
 }
@@ -136,5 +204,9 @@ output "ca_key_private" {
 }
 
 output "ca_key_public" {
-  value = base64encode(local.ca_public_key)
+  value = base64encode(data.tls_public_key.ca_key.public_key_openssh)
+}
+
+output "ca_key_public_string" {
+  value = data.tls_public_key.ca_key.public_key_openssh
 }

enos/modules/docker_worker/main.tf

@@ -70,6 +70,11 @@ variable "is_downstream_worker" {
   type        = bool
   default     = false
 }
+variable "ssh_ca_public_key" {
+  description = "SSH CA public key used to write worker known_hosts for host certificate validation."
+  type        = string
+  default     = ""
+}
 
 resource "docker_image" "boundary" {
   name         = var.image_name
@@ -79,12 +84,13 @@ resource "docker_image" "boundary" {
 locals {
   recording_storage_path = "/boundary/recordings"
   port_ops               = var.port + 1
+  config_file_path       = "/boundary/worker-config.hcl"
 }
 
 resource "docker_container" "worker" {
   image   = docker_image.boundary.image_id
   name    = var.container_name
-  command = ["boundary", "server", "-config", "/boundary/worker-config.hcl"]
+  command = ["boundary", "server", "-config", local.config_file_path]
   env = [
     "BOUNDARY_LICENSE=${var.boundary_license}",
     "HOSTNAME=boundary",
@@ -114,8 +120,13 @@ resource "docker_container" "worker" {
       port                   = var.port
       port_ops               = local.port_ops
       token                  = var.token
+      ssh_known_hosts_path   = var.ssh_ca_public_key != "" ? "/etc/ssh/known_hosts" : ""
     })
-    file = "/boundary/worker-config.hcl"
+    file = local.config_file_path
+  }
+  upload {
+    content = var.ssh_ca_public_key != "" ? "@cert-authority [*]:2222 ${trimspace(var.ssh_ca_public_key)}\n" : "#"
+    file    = "/etc/ssh/known_hosts"
   }
   healthcheck {
     test     = ["CMD", "grep", "-i", "worker has successfully authenticated", "/boundary/logs/events.log"]
@@ -159,3 +170,7 @@ output "upstream_address" {
 output "worker_led_token" {
   value = var.worker_led_registration ? trimspace(enos_local_exec.get_worker_led_token[0].stdout) : ""
 }
+
+output "config_location" {
+  value = local.config_file_path
+}

enos/modules/docker_worker/worker-config-bsr-downstream.hcl

@@ -12,9 +12,7 @@ listener "tcp" {
 }
 
 listener "tcp" {
-  # setting to 127.0.0.1 so that it won't be accessible by the local machine
-  # outside of the container, which is a more realistic configuration for a downstream worker
-  address = "127.0.0.1:${port_ops}"
+  address = "0.0.0.0:${port_ops}"
   purpose = "ops"
   tls_disable = true
 }
@@ -28,6 +26,10 @@ worker {
   }
 
   recording_storage_path = "${recording_storage_path}"
+
+%{ if ssh_known_hosts_path != "" ~}
+  # ssh_known_hosts_path = "${ssh_known_hosts_path}"
+%{ endif ~}
 }
 
 # This key_id needs to match the corresponding upstream worker's

enos/modules/docker_worker/worker-config-downstream.hcl

@@ -12,9 +12,7 @@ listener "tcp" {
 }
 
 listener "tcp" {
-  # setting to 127.0.0.1 so that it won't be accessible by the local machine
-  # outside of the container, which is a more realistic configuration for a downstream worker
-  address = "127.0.0.1:${port_ops}"
+  address = "0.0.0.0:${port_ops}"
   purpose = "ops"
   tls_disable = true
 }

testing/internal/e2e/boundary/version.go

@@ -0,0 +1,49 @@
+// Copyright IBM Corp. 2020, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package boundary
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	gvers "github.com/hashicorp/go-version"
+	"github.com/stretchr/testify/require"
+
+	"github.com/hashicorp/boundary/testing/internal/e2e"
+	"github.com/hashicorp/boundary/version"
+)
+
+// IsVersionAtLeast checks if the Boundary version running in the specified container is at least the given minimum version.
+func IsVersionAtLeast(t testing.TB, ctx context.Context, containerName string, minVersion string) {
+	output := e2e.RunCommand(
+		ctx,
+		"docker",
+		e2e.WithArgs(
+			"exec", containerName,
+			"boundary", "version",
+			"-format", "json",
+		),
+	)
+	require.NoError(t, output.Err, "failed to get version from container %q: %s", containerName, string(output.Stderr))
+
+	var versionResult version.Info
+	err := json.Unmarshal(output.Stdout, &versionResult)
+	require.NoError(t, err)
+
+	minSemVersion, err := gvers.NewSemver(minVersion)
+	require.NoError(t, err)
+
+	containerVersion := versionResult.Semver()
+	require.NotNil(t, containerVersion, "failed to parse version %q from container %q", versionResult.VersionNumber(), containerName)
+
+	if !containerVersion.GreaterThanOrEqual(minSemVersion) {
+		t.Skipf(
+			"Skipping test because container %q is running %q, but this test requires >= %q",
+			containerName,
+			versionResult.VersionNumber(),
+			minVersion,
+		)
+	}
+}

testing/internal/e2e/infra/docker.go

@@ -35,6 +35,14 @@ type cassandraConfig struct {
 	NetworkAlias string
 }
 
+type DockerInspectResult []struct {
+	State struct {
+		Health *struct {
+			Status string `json:"Status"`
+		} `json:"Health"`
+	} `json:"State"`
+}
+
 // StartBoundaryDatabase spins up a postgres database in a docker container.
 // Returns information about the container
 func StartBoundaryDatabase(t testing.TB, pool *dockertest.Pool, network *dockertest.Network, repository, tag string) *Container {