CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(enos): Add scenario for e2e UI tests (#2813)

Browse Source
This commit is contained in:
Michael Li authored and GitHub committed 2023-03-02 14:32:53 -05:00
1 parent 4d19d1cfc4
commit 420952f6b8
9 files changed
+435 -16
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/enos-run.yml
+41 -12
View File
@@ -29,6 +29,7 @@ jobs:
- filter: 'e2e_database'
- filter: 'e2e_static builder:crt'
- filter: 'e2e_static_with_vault builder:crt'
# - filter: 'e2e_ui builder:crt' # Don't run UI tests yet. takes too long.
runs-on: ${{ fromJSON(vars.RUNNER) }}
env:
GITHUB_TOKEN: ${{ secrets.SERVICE_USER_GITHUB_TOKEN }}
@@ -100,15 +101,15 @@ jobs:
cd /tmp/test-deps/pass/password-store-1.7.4
sudo make install
pass init ${{ secrets.ENOS_GPG_UID }}
- name: Download Vault AMD64 binary for integration testing
- name: Download Vault AMD64 binary
if: steps.dep-cache.outputs.cache-hit != 'true'
run: |
wget https://releases.hashicorp.com/vault/1.12.2/vault_1.12.2_linux_amd64.zip -O /tmp/test-deps/vault.zip
- name: Install Vault for integration testing
if: matrix.filter == 'e2e_static_with_vault builder:crt' || matrix.filter == 'e2e_database'
- name: Install Vault CLI
if: matrix.filter == 'e2e_static_with_vault builder:crt' || matrix.filter == 'e2e_database' || matrix.filter == 'e2e_ui builder:crt'
run: |
unzip /tmp/test-deps/vault.zip -d /usr/local/bin
- name: Download Linux AMD64 Boundary bundle
- name: Download Boundary Linux AMD64 bundle
id: download
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
@@ -116,15 +117,34 @@ jobs:
path: ./enos/support/downloads
- name: Unzip and rename Boundary bundle
run: |
unzip ${{steps.download.outputs.download-path}}/*.zip -d enos/support
unzip ${{steps.download.outputs.download-path}}/*.zip -d enos/support/boundary
mv ${{steps.download.outputs.download-path}}/*.zip enos/support/boundary.zip
- name: Set up Node.js
uses: actions/setup-node@v3
if: matrix.filter == 'e2e_ui builder:crt'
with:
node-version: '16.x'
- name: Checkout boundary-ui
uses: actions/checkout@v3
if: matrix.filter == 'e2e_ui builder:crt'
with:
repository: hashicorp/boundary-ui
path: enos/support/boundary-ui
- name: Install boundary-ui dependencies
if: matrix.filter == 'e2e_ui builder:crt'
run: yarn --cwd enos/support/boundary-ui install
- name: Install playwright dependencies (i.e. browsers)
if: matrix.filter == 'e2e_ui builder:crt'
run: npx playwright install --with-deps
working-directory: enos/support/boundary-ui
- name: Output Terraform version info
# Use the same env vars from the following step
env:
ENOS_VAR_aws_region: us-east-1
ENOS_VAR_aws_ssh_keypair_name: ${{ github.event.repository.name }}-ci-ssh-key
ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
ENOS_VAR_local_boundary_dir: ./support/
ENOS_VAR_local_boundary_dir: ./support/boundary
ENOS_VAR_local_boundary_ui_dir: ./support/boundary-ui
ENOS_VAR_crt_bundle_path: ./support/boundary.zip
ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
ENOS_VAR_test_email: ${{ secrets.SERVICE_USER_EMAIL }}
@@ -141,7 +161,8 @@ jobs:
ENOS_VAR_aws_region: us-east-1
ENOS_VAR_aws_ssh_keypair_name: ${{ github.event.repository.name }}-ci-ssh-key
ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
ENOS_VAR_local_boundary_dir: ./support/
ENOS_VAR_local_boundary_dir: ./support/boundary
ENOS_VAR_local_boundary_ui_dir: ./support/boundary-ui
ENOS_VAR_crt_bundle_path: ./support/boundary.zip
ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
ENOS_VAR_test_email: ${{ secrets.SERVICE_USER_EMAIL }}
@@ -152,9 +173,15 @@ jobs:
- name: Upload e2e tests output
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: test-e2e-output.zip
path: enos/test*.out
name: test-e2e-output
path: enos/test*.log
retention-days: 5
- name: Upload e2e UI tests debug info
if: matrix.filter == 'e2e_ui builder:crt' && steps.run.outcome == 'failure'
uses: actions/upload-artifact@v3
with:
name: test-e2e-ui-debug
path: enos/support/boundary-ui/ui/admin/tests/e2e/artifacts/test-failures
- name: Retry Enos scenario
id: run_retry
if: steps.run.outcome == 'failure'
@@ -162,7 +189,8 @@ jobs:
ENOS_VAR_aws_region: us-east-1
ENOS_VAR_aws_ssh_keypair_name: ${{ github.event.repository.name }}-ci-ssh-key
ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
ENOS_VAR_local_boundary_dir: ./support/
ENOS_VAR_local_boundary_dir: ./support/boundary
ENOS_VAR_local_boundary_ui_dir: ./support/boundary-ui
ENOS_VAR_crt_bundle_path: ./support/boundary.zip
ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
ENOS_VAR_test_email: ${{ secrets.SERVICE_USER_EMAIL }}
@@ -175,14 +203,15 @@ jobs:
ENOS_VAR_aws_region: us-east-1
ENOS_VAR_aws_ssh_keypair_name: ${{ github.event.repository.name }}-ci-ssh-key
ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem
ENOS_VAR_local_boundary_dir: ./support/
ENOS_VAR_local_boundary_dir: ./support/boundary
ENOS_VAR_local_boundary_ui_dir: ./support/boundary-ui
ENOS_VAR_crt_bundle_path: ./support/boundary.zip
ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
ENOS_VAR_test_email: ${{ secrets.SERVICE_USER_EMAIL }}
run: |
export ENOS_VAR_enos_user=$GITHUB_ACTOR && \
enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.filter }}
- name: Output debug information on failure
- name: Output Enos debug information on failure
if: ${{ failure() }}
run: |
env
enos/enos-modules.hcl
+4
View File
@@ -90,6 +90,10 @@ module "test_e2e" {
source = "./modules/test_e2e"
}
module "test_e2e_ui" {
source = "./modules/test_e2e_ui"
}
module "test_smoke" {
source = "./modules/test_smoke"
}
enos/enos-scenario-e2e-ui.hcl
+226
View File
@@ -0,0 +1,226 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
scenario "e2e_ui" {
terraform_cli = terraform_cli.default
terraform = terraform.default
providers = [
provider.aws.default,
provider.enos.default
]
matrix {
builder = ["local", "crt"]
}
locals {
aws_ssh_private_key_path = abspath(var.aws_ssh_private_key_path)
boundary_install_dir = abspath(var.boundary_install_dir)
local_boundary_dir = abspath(var.local_boundary_dir)
local_boundary_ui_dir = abspath(var.local_boundary_ui_dir)
build_path = {
"local" = "/tmp",
"crt" = var.crt_bundle_path == null ? null : abspath(var.crt_bundle_path)
}
tags = merge({
"Project Name" : var.project_name
"Project" : "Enos",
"Environment" : "ci"
}, var.tags)
}
step "find_azs" {
module = module.az_finder
variables {
instance_type = [
var.worker_instance_type,
var.controller_instance_type
]
}
}
step "create_db_password" {
module = module.random_stringifier
}
step "build_boundary" {
module = matrix.builder == "crt" ? module.build_crt : module.build_local
variables {
path = local.build_path[matrix.builder]
}
}
step "create_base_infra" {
module = module.infra
depends_on = [
step.find_azs,
]
variables {
availability_zones = step.find_azs.availability_zones
common_tags = local.tags
}
}
step "create_boundary_cluster" {
module = module.boundary
depends_on = [
step.create_base_infra,
step.create_db_password,
step.build_boundary
]
variables {
boundary_install_dir = local.boundary_install_dir
common_tags = local.tags
controller_instance_type = var.controller_instance_type
controller_count = var.controller_count
db_pass = step.create_db_password.string
kms_key_arn = step.create_base_infra.kms_key_arn
local_artifact_path = step.build_boundary.artifact_path
ubuntu_ami_id = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
vpc_id = step.create_base_infra.vpc_id
worker_count = var.worker_count
worker_instance_type = var.worker_instance_type
}
}
step "create_vault_cluster" {
module = module.vault
depends_on = [
step.create_base_infra,
]
variables {
ami_id = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
instance_type = var.vault_instance_type
instance_count = 1
kms_key_arn = step.create_base_infra.kms_key_arn
storage_backend = "raft"
sg_additional_ips = step.create_boundary_cluster.controller_ips
unseal_method = "awskms"
vault_release = {
version = var.vault_version
edition = "oss"
}
vpc_id = step.create_base_infra.vpc_id
}
}
step "create_tag1" {
module = module.random_stringifier
}
step "create_tag1_inputs" {
module = module.generate_aws_host_tag_vars
depends_on = [step.create_tag1]
variables {
tag_name = step.create_tag1.string
tag_value = "true"
}
}
step "create_targets_with_tag1" {
module = module.target
depends_on = [step.create_base_infra]
variables {
ami_id = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
aws_ssh_keypair_name = var.aws_ssh_keypair_name
enos_user = var.enos_user
instance_type = var.target_instance_type
vpc_id = step.create_base_infra.vpc_id
target_count = 2
additional_tags = step.create_tag1_inputs.tag_map
}
}
step "create_tag2" {
module = module.random_stringifier
}
step "create_tag2_inputs" {
module = module.generate_aws_host_tag_vars
depends_on = [step.create_tag2]
variables {
tag_name = step.create_tag2.string
tag_value = "test"
}
}
step "create_targets_with_tag2" {
module = module.target
depends_on = [step.create_base_infra]
variables {
ami_id = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
aws_ssh_keypair_name = var.aws_ssh_keypair_name
enos_user = var.enos_user
instance_type = var.target_instance_type
vpc_id = step.create_base_infra.vpc_id
target_count = 1
additional_tags = step.create_tag2_inputs.tag_map
}
}
step "create_test_id" {
module = module.random_stringifier
variables {
length = 5
}
}
step "iam_setup" {
module = module.iam_setup
depends_on = [
step.create_base_infra,
step.create_test_id
]
variables {
test_id = step.create_test_id.string
test_email = var.test_email
}
}
step "run_e2e_ui_test" {
module = module.test_e2e_ui
depends_on = [
step.create_boundary_cluster,
step.create_targets_with_tag1,
step.create_targets_with_tag2,
step.iam_setup,
step.create_vault_cluster
]
variables {
debug_no_run = var.e2e_debug_no_run
alb_boundary_api_addr = step.create_boundary_cluster.alb_boundary_api_addr
auth_method_id = step.create_boundary_cluster.auth_method_id
auth_login_name = step.create_boundary_cluster.auth_login_name
auth_password = step.create_boundary_cluster.auth_password
local_boundary_dir = local.local_boundary_dir
local_boundary_ui_dir = local.local_boundary_ui_dir
aws_ssh_private_key_path = local.aws_ssh_private_key_path
target_ip = step.create_targets_with_tag1.target_ips[0]
target_user = "ubuntu"
target_port = "22"
vault_addr = step.create_vault_cluster.instance_public_ips[0]
vault_root_token = step.create_vault_cluster.vault_root_token
aws_access_key_id = step.iam_setup.access_key_id
aws_secret_access_key = step.iam_setup.secret_access_key
aws_host_set_filter1 = step.create_tag1_inputs.tag_string
aws_host_set_ips1 = step.create_targets_with_tag1.target_ips
aws_host_set_filter2 = step.create_tag2_inputs.tag_string
aws_host_set_ips2 = step.create_targets_with_tag2.target_ips
}
}
output "test_results" {
value = step.run_e2e_ui_test.test_results
}
}
enos/enos-variables.hcl
+6
View File
@@ -84,6 +84,12 @@ variable "local_boundary_dir" {
type = string
}
variable "local_boundary_ui_dir" {
description = "Path to local boundary-ui directory"
type = string
default = null
}
variable "crt_bundle_path" {
description = "Path to CRT generated boundary bundle"
type = string
enos/enos.vars.hcl
+3
View File
@@ -29,6 +29,9 @@
// similar to what is listed below.
// local_boundary_dir = "/Users/<user>/.go/bin"
// The directory that contains the copy of boundary-ui you want to use for UI tests
// local_boundary_ui_dir = "/Users/<user>/Developer/boundary-ui"
// The path to the installation bundle for the target machines. The existing
// scenarios all use linux/amd64 architecture so bundle ought to match that
// architecture. This is only used for variants which use the `crt` builder
enos/modules/test_e2e/main.tf
+1 -1
View File
@@ -144,7 +144,7 @@ resource "enos_local_exec" "run_e2e_test" {
E2E_AWS_HOST_SET_IPS2 = local.aws_host_set_ips2
}
inline = var.debug_no_run ? [""] : ["set -o pipefail; PATH=\"${var.local_boundary_dir}:$PATH\" go test -v ${var.test_package} -count=1 -json | tparse -follow -format plain 2>&1 | tee ${path.module}/../../test-e2e-${local.package_name}.out"]
inline = var.debug_no_run ? [""] : ["set -o pipefail; PATH=\"${var.local_boundary_dir}:$PATH\" go test -v ${var.test_package} -count=1 -json | tparse -follow -format plain 2>&1 | tee ${path.module}/../../test-e2e-${local.package_name}.log"]
}
output "test_results" {
enos/modules/test_e2e_ui/main.tf
+150
View File
@@ -0,0 +1,150 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
terraform {
required_providers {
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
}
}
}
variable "debug_no_run" {
description = "If set, this module will not execute the tests so that you can still access environment variables"
type = bool
default = true
}
variable "alb_boundary_api_addr" {
description = "URL of the Boundary instance"
type = string
default = ""
}
variable "auth_method_id" {
description = "Id of Auth Method used to login to Boundary instance"
type = string
default = ""
}
variable "auth_login_name" {
description = "Name of admin user"
type = string
default = ""
}
variable "auth_password" {
description = "Password of admin user"
type = string
default = ""
}
variable "local_boundary_dir" {
description = "Local Path to boundary executable"
type = string
}
variable "local_boundary_ui_dir" {
description = "Local Path to boundary-ui directory"
type = string
}
variable "target_user" {
description = "SSH username for target"
type = string
default = ""
}
variable "aws_ssh_private_key_path" {
description = "Local Path to key used to SSH onto created hosts"
type = string
default = ""
}
variable "target_ip" {
description = "IP address of target"
type = string
default = ""
}
variable "target_port" {
description = "Port of target"
type = string
default = ""
}
variable "vault_addr" {
description = "External network address of Vault. Will be converted to a URL below"
type = string
default = ""
}
variable "vault_addr_internal" {
description = "Internal network address of Vault (i.e. within a docker network). Will be converted to a URL below"
type = string
default = ""
}
variable "vault_root_token" {
description = "Root token for vault instance"
type = string
default = ""
}
variable "aws_access_key_id" {
description = "Access Key Id for AWS IAM user used in dynamic host catalogs"
type = string
default = ""
}
variable "aws_secret_access_key" {
description = "Secret Access Key for AWS IAM user used in dynamic host catalogs"
type = string
default = ""
}
variable "aws_host_set_filter1" {
description = "Filter tag for host set used in dynamic host catalogs"
type = string
default = ""
}
variable "aws_host_set_count1" {
description = "Number of hosts in aws_host_set_filter1"
type = number
default = 0
}
variable "aws_host_set_ips1" {
description = "List of IP addresses in aws_host_set_filter1"
type = list(string)
default = [""]
}
variable "aws_host_set_filter2" {
description = "Filter tag for host set used in dynamic host catalogs"
type = string
default = ""
}
variable "aws_host_set_ips2" {
description = "List of IP addresses in aws_host_set_filter2"
type = list(string)
default = [""]
}
locals {
aws_ssh_private_key_path = abspath(var.aws_ssh_private_key_path)
vault_addr = var.vault_addr != "" ? "http://${var.vault_addr}:8200" : ""
vault_addr_internal = var.vault_addr_internal != "" ? "http://${var.vault_addr_internal}:8200" : local.vault_addr
aws_host_set_ips1 = jsonencode(var.aws_host_set_ips1)
aws_host_set_ips2 = jsonencode(var.aws_host_set_ips2)
}
resource "enos_local_exec" "run_e2e_ui_test" {
environment = {
BOUNDARY_ADDR = var.alb_boundary_api_addr,
E2E_PASSWORD_AUTH_METHOD_ID = var.auth_method_id,
E2E_PASSWORD_ADMIN_LOGIN_NAME = var.auth_login_name,
E2E_PASSWORD_ADMIN_PASSWORD = var.auth_password,
E2E_TARGET_IP = var.target_ip,
E2E_SSH_USER = var.target_user,
E2E_SSH_PORT = var.target_port,
E2E_SSH_KEY_PATH = local.aws_ssh_private_key_path,
VAULT_ADDR = local.vault_addr,
VAULT_TOKEN = var.vault_root_token,
E2E_VAULT_ADDR = local.vault_addr_internal,
E2E_AWS_ACCESS_KEY_ID = var.aws_access_key_id,
E2E_AWS_SECRET_ACCESS_KEY = var.aws_secret_access_key,
E2E_AWS_HOST_SET_FILTER = var.aws_host_set_filter1,
E2E_AWS_HOST_SET_IPS = local.aws_host_set_ips1,
E2E_AWS_HOST_SET_FILTER2 = var.aws_host_set_filter2,
E2E_AWS_HOST_SET_IPS2 = local.aws_host_set_ips2
}
inline = var.debug_no_run ? [""] : ["set -o pipefail; PATH=\"${var.local_boundary_dir}:$PATH\" yarn --cwd ${var.local_boundary_ui_dir}/ui/admin run e2e 2>&1 | tee ${path.module}/../../test-e2e-ui.log"]
}
output "test_results" {
value = enos_local_exec.run_e2e_ui_test.stdout
}
enos/scripts/test_e2e_env.sh
+1
View File
@@ -9,5 +9,6 @@ STATEDIR=$(ls -td $SCRIPTS_DIR/../.enos/*/ | head -1) # get latest directory
cd $STATEDIR
terraform show -json terraform.tfstate | jq -r '.values.root_module.child_modules[].resources[] | select(.address=="module.run_e2e_test.enos_local_exec.run_e2e_test") | .values.environment | to_entries[] | "export \(.key)=\(.value|@sh)"'
terraform show -json terraform.tfstate | jq -r '.values.root_module.child_modules[].resources[] | select(.address=="module.run_e2e_ui_test.enos_local_exec.run_e2e_ui_test") | .values.environment | to_entries[] | "export \(.key)=\(.value|@sh)"'
cd $DIR
testing/internal/e2e/README.md
+3 -3
View File
@@ -84,10 +84,10 @@ export VAULT_TOKEN=
Then, run...
```shell
go test github.com/hashicorp/boundary/testing/e2e/target // run target tests
go test github.com/hashicorp/boundary/testing/internal/e2e/tests/static
go test ./target/ // run target tests if running from this directory
go test github.com/hashicorp/boundary/testing/e2e/target -v // verbose
go test github.com/hashicorp/boundary/testing/e2e/target -v -run '^TestCreateTargetApi$' // run a specific test
go test github.com/hashicorp/boundary/testing/internal/e2e/tests/static -v // verbose
go test github.com/hashicorp/boundary/testing/internal/e2e/tests/static -v -run '^TestCreateTargetApi$' // run a specific test
```
## Adding Tests