diff --git a/.github/workflows/enos-ci-bootstrap-oss.yml b/.github/workflows/enos-ci-bootstrap-oss.yml new file mode 100644 index 000000000..8248da9e8 --- /dev/null +++ b/.github/workflows/enos-ci-bootstrap-oss.yml @@ -0,0 +1,50 @@ +name: enos-ci-bootstrap-oss + +on: + pull_request: + branches: + - main + paths: + - enos/ci/** + - .github/workflows/enos-ci-bootstrap-oss.yml + push: + branches: + - main + paths: + - enos/ci/** + - .github/workflows/enos-ci-bootstrap-oss.yml + +jobs: + bootstrap-ci-oss: + env: + TF_WORKSPACE: "boundary-ci-enos-bootstrap" + TF_VAR_repository: boundary + TF_VAR_aws_ssh_public_key: ${{ secrets.SSH_KEY_PUBLIC_CI }} + TF_TOKEN_app_terraform_io: ${{ secrets.TF_API_TOKEN }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Terraform + uses: hashicorp/setup-terraform@v2 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CI }} + aws-region: us-east-1 + role-to-assume: arn:aws:iam::271311691044:role/github_actions-boundary_ci + role-skip-session-tagging: true + role-duration-seconds: 3600 + - name: Init Terraform + id: tf_init + run: | + terraform -chdir=enos/ci/bootstrap init + - name: Plan Terraform + id: tf_plan + run: | + terraform -chdir=enos/ci/bootstrap plan + - name: Apply Terraform + if: ${{ github.ref == 'refs/heads/main' }} + id: tf_apply + run: | + terraform -chdir=enos/ci/bootstrap apply -auto-approve diff --git a/.gitignore b/.gitignore index cc72095b2..7643f12a4 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,11 @@ test*.hcl /enos/terraform-plugin-cache /enos/*.zip /enos/support +# Enos local Terraform files +enos/.terraform/* +enos/.terraform.lock.hcl +enos/*.tfstate +enos/*.tfstate.* # vim: set filetype=conf : diff --git a/enos/README.md b/enos/README.md index 6a16ef7f7..dca48a645 100644 --- a/enos/README.md +++ b/enos/README.md @@ -116,3 +116,67 @@ you could test that version against the cluster by setting `local_boundary_dir` Scenarios with `e2e_` invoke an end-to-end test suite written in Go. Different tests are invoked depending on the scenario. + +# CI Bootstrap +In order to execute any of the scenarios in this repository, it is first necessary to bootstrap the +CI AWS account with the required permissions, service quotas and supporting AWS resources. There are +two Terraform modules which are used for this purpose, [service-user-iam](./ci/service-user-iam) for +the account permissions, and service quotas and [bootstrap](./ci/bootstrap) for the supporting resources. + +## Bootstrap Process +These steps should be followed to bootstrap this repo for enos scenario execution: + +### Set up CI service user IAM role and Service Quotas +The service user that is used when executing enos scenarios from any GitHub Action workflow must have +a properly configured IAM role granting the access required to create resources in AWS. Additionally, +service quotas need to be adjusted to ensure that normal use of the ci account does not cause any +service quotas to be exceeded. The [service-user-iam](./ci/service-user-iam) module contains the IAM +Policy and Role for that grants this access as well as the service quota increase requests to adjust +the service quotas. This module should be updated whenever a new AWS resource type is required for a +scenario or a service quota limit needs to be increased. Since this is persistent and cannot be created +and destroyed each time a scenario is run, the Terraform state will be managed by Terraform Cloud. +Here are the steps to configure the GitHub Actions service user: + +#### Pre-requisites +- Access to the `hashicorp-qti` organization in Terraform Cloud. +- Full access to the CI AWS account is required. + +**Notes:** +- For help with access to Terraform Cloud and the CI Account, contact the QT team on Slack (#team-quality) + for an invite. After receiving an invite to Terraform Cloud, a personal access token can be created + by clicking `User Settings` --> `Tokens` --> `Create an API token`. +- Access to the AWS account can be done via Doormat, at: https://doormat.hashicorp.services/. + - The account name uses the following pattern: `-ci`, e.g. `boundary-ci` for the boundary repo. + - Access can be requested by clicking: `Cloud Access` --> `AWS` --> `Request Account Access`. + +1. **Create the Terraform Cloud Workspace** - The name of the workspace to be created depends on the + repository for which it is being created, but the pattern is: `-ci-service-user-iam`, + e.g. `boundary-ci-service-user-iam`. It is important that the execution mode for the workspace be + set to `local`. For help on setting up the workspace, contact the QT team on Slack (#team-quality) + + +2. **Execute the Terraform module** +```shell +> cd ./enos/ci/service-user-iam +> export TF_WORKSPACE=-ci-service-user-iam +> export TF_TOKEN_app_terraform_io= +> export TF_VAR_repository= +> terraform init +> terraform plan +> terraform apply -auto-approve +``` + +### Bootstrap the CI resources +Bootstrapping of the resources in the CI account is accomplished via the GitHub Actions workflow: +[enos-bootstrap-ci](../.github/workflows/enos-bootstrap-ci.yml). Before this workflow can be run a +workspace must be created as follows: + +1. **Create the Terraform Cloud Workspace** - The name workspace to be created depends on the repository + for which it is being created, but the pattern is: `-ci-bootstrap`, e.g. + `boundary-ci-bootstrap`. It is important that the execution mode for the workspace be set to + `local`. For help on setting up the workspace, contact the QT team on Slack (#team-quality). + +Once the workspace has been created, changes to the bootstrap module will automatically be applied via +the GitHub PR workflow. Each time a PR is created for changes to files within that module the module +will be planned via the workflow described above. If the plan is ok and the PR is merged, the module +will automatically be applied via the same workflow. diff --git a/enos/ci/bootstrap/main.tf b/enos/ci/bootstrap/main.tf new file mode 100644 index 000000000..83facaea4 --- /dev/null +++ b/enos/ci/bootstrap/main.tf @@ -0,0 +1,66 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } + + cloud { + hostname = "app.terraform.io" + organization = "hashicorp-qti" + // workspace must be exported in the environment as: TF_WORKSPACE=-ci-enos-bootstrap + } +} + +provider "aws" { + region = "us-east-1" + alias = "us_east_1" +} + +provider "aws" { + region = "us-east-2" + alias = "us_east_2" +} + +provider "aws" { + region = "us-west-1" + alias = "us_west_1" +} + +provider "aws" { + region = "us-west-2" + alias = "us_west_2" +} + + +locals { + key_name = "${var.repository}-ci-ssh-key" +} + +resource "aws_key_pair" "enos_ci_key_us_east_1" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_east_1 +} + +resource "aws_key_pair" "enos_ci_key_us_east_2" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_east_2 +} + +resource "aws_key_pair" "enos_ci_key_us_west_1" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_west_1 +} + +resource "aws_key_pair" "enos_ci_key_us_west_2" { + key_name = local.key_name + public_key = var.aws_ssh_public_key + + provider = aws.us_west_2 +} diff --git a/enos/ci/bootstrap/outputs.tf b/enos/ci/bootstrap/outputs.tf new file mode 100644 index 000000000..858318e4c --- /dev/null +++ b/enos/ci/bootstrap/outputs.tf @@ -0,0 +1,20 @@ +output "keys" { + value = { + "us-east-1" = { + name = aws_key_pair.enos_ci_key_us_east_1.key_name + arn = aws_key_pair.enos_ci_key_us_east_1.arn + } + "us-east-2" = { + name = aws_key_pair.enos_ci_key_us_east_2.key_name + arn = aws_key_pair.enos_ci_key_us_east_2.arn + } + "us-west-1" = { + name = aws_key_pair.enos_ci_key_us_west_1.key_name + arn = aws_key_pair.enos_ci_key_us_west_1.arn + } + "us-west-2" = { + name = aws_key_pair.enos_ci_key_us_west_2.key_name + arn = aws_key_pair.enos_ci_key_us_west_2.arn + } + } +} diff --git a/enos/ci/bootstrap/variables.tf b/enos/ci/bootstrap/variables.tf new file mode 100644 index 000000000..c7b76031b --- /dev/null +++ b/enos/ci/bootstrap/variables.tf @@ -0,0 +1,13 @@ +variable "aws_ssh_public_key" { + description = "The public key to use for the ssh key" + type = string +} + +variable "repository" { + description = "The repository to bootstrap the ci for, either 'boundary', 'boundary-enterprise', or 'boundary-hcp'" + type = string + validation { + condition = contains(["boundary", "boundary-enterprise", "boundary-hcp"], var.repository) + error_message = "Repository must be one of either 'boundary', 'boundary-enterprise', or 'boundary-hcp'" + } +} diff --git a/enos/ci/service-user-iam/github-actions-doormat.tf b/enos/ci/service-user-iam/github-actions-doormat.tf new file mode 100644 index 000000000..20ce0f3f7 --- /dev/null +++ b/enos/ci/service-user-iam/github-actions-doormat.tf @@ -0,0 +1,50 @@ +locals { + doormat_service_user_arn = "arn:aws:iam::397512762488:user/doormatServiceUser" + // "Github Actions Doormat repositories and qualifiers" + // see: https://docs.prod.secops.hashicorp.services/doormat/gha/ + github_actions_doormat_rwqs = { + boundary-enterprise = "github.com/hashicorp/boundary-enterprise@event_name=workflow_dispatch+push:ref=refs/heads/main+refs/heads/release/0.8.x+refs/heads/release/0.10.x+refs/heads/release/0.11.x///event_name=pull_request:base_ref=main+release/0.8.x+release/0.10.x+release/0.11.x", + boundary-hcp = "github.com/hashicorp/boundary-hcp@event_name=workflow_dispatch+push:ref=refs/heads/main+refs/heads/release/0.8.x+refs/heads/release/0.10.x+refs/heads/release/0.11.x///event_name=pull_request:base_ref=main+release/0.8.x+release/0.10.x+release/0.11.x", + } + github_actions_doormat_assume_policy_name = "AssumeServiceUserPolicy" + boundary_gha_iam_role_name = "${var.repository}-GHA-ci" +} + +// Doormat Github Actions assume policy +data "aws_iam_policy_document" "github_actions_doormat_assume" { + count = local.is_ent ? 1 : 0 + + provider = aws.us_east_1 + + statement { + actions = [ + "sts:AssumeRole", + "sts:SetSourceIdentity", + "sts:TagSession" + ] + principals { + type = "AWS" + identifiers = [local.doormat_service_user_arn] # infrasec_prod + } + } +} + +# Doormat Github Actions roles +resource "aws_iam_role" "github_actions_doormat_role" { + count = local.is_ent ? 1 : 0 + + provider = aws.us_east_1 + + name = local.boundary_gha_iam_role_name + tags = { + hc-service-uri = local.github_actions_doormat_rwqs[var.repository] + } + max_session_duration = 43200 + assume_role_policy = data.aws_iam_policy_document.github_actions_doormat_assume[0].json + + inline_policy { + name = "AssumeServiceUserPolicy" + // Use the service user policy for now + policy = data.aws_iam_policy_document.iam_policy_document.json + } +} diff --git a/enos/ci/service-user-iam/main.tf b/enos/ci/service-user-iam/main.tf new file mode 100644 index 000000000..66a95d117 --- /dev/null +++ b/enos/ci/service-user-iam/main.tf @@ -0,0 +1,217 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } + + cloud { + hostname = "app.terraform.io" + organization = "hashicorp-qti" + // workspace must be exported in the environment as: TF_WORKSPACE=-ci-enos-service-user-iam + } +} + +locals { + enterprise_repositories = ["boundary-enterprise", "boundary-hcp"] + is_ent = contains(local.enterprise_repositories, var.repository) + service_user = "github_actions-boundary_ci" + oss_aws_account_id = "271311691044" +} + +resource "aws_iam_role" "role" { + count = local.is_ent ? 0 : 1 // only create a role for the OSS repositories + + provider = aws.us_east_1 + + name = local.service_user + assume_role_policy = data.aws_iam_policy_document.assume_role_policy_document[0].json +} + +data "aws_iam_policy_document" "assume_role_policy_document" { + count = local.is_ent ? 0 : 1 // only create a policy for the OSS repositories + + provider = aws.us_east_1 + + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.oss_aws_account_id}:user/${local.service_user}"] + } + } +} + +resource "aws_iam_role_policy" "role_policy" { + count = local.is_ent ? 0 : 1 // only create a policy for the OSS repositories + + provider = aws.us_east_1 + + role = aws_iam_role.role[0].name + name = "${local.service_user}_policy" + policy = data.aws_iam_policy_document.iam_policy_document.json +} + +data "aws_iam_policy_document" "iam_policy_document" { + provider = aws.us_east_1 + + statement { + effect = "Allow" + actions = [ + "iam:ListRoles", + "iam:CreateRole", + "iam:GetRole", + "iam:DeleteRole", + "iam:ListInstanceProfiles", + "iam:ListInstanceProfilesForRole", + "iam:CreateInstanceProfile", + "iam:GetInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:ListPolicies", + "iam:CreatePolicy", + "iam:DeletePolicy", + "iam:ListRoles", + "iam:CreateRole", + "iam:AddRoleToInstanceProfile", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteRole", + "iam:ListRolePolicies", + "iam:ListAttachedRolePolicies", + "iam:AttachRolePolicy", + "iam:GetRolePolicy", + "iam:PutRolePolicy", + "iam:DetachRolePolicy", + "iam:DeleteRolePolicy", + "iam:ListUsers", + "iam:GetUser", + "iam:GetUserId", + "iam:DescribeUser", + "iam:DeleteUser", + "iam:CreateUser", + "iam:TagUser", + "iam:UntagUser", + "iam:ListUserTags", + "iam:CreateUserTag", + "iam:DeleteUserTag", + "iam:ListUserPolicies", + "iam:CreateUserPolicy", + "iam:PutUserPolicy", + "iam:DeleteUserPolicy", + "iam:ListGroupsForUser", + "iam:ListAccessKeys", + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeImages", + "ec2:DescribeTags", + "ec2:DescribeVpcClassicLink", + "ec2:DescribeVpcClassicLinkDnsSupport", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeNetworkAcls", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeSecurityGroups", + "ec2:CreateSecurityGroup", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:DeleteSecurityGroup", + "ec2:RevokeSecurityGroupIngress", + "ec2:RevokeSecurityGroupEgress", + "ec2:DescribeInstances", + "ec2:DescribeInstanceAttribute", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:ModifyInstanceAttribute", + "ec2:TerminateInstances", + "ec2:ResetInstanceAttribute", + "ec2:DeleteTags", + "ec2:DescribeVolumes", + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:DescribeVpcs", + "ec2:DescribeVpcAttribute", + "ec2:CreateVPC", + "ec2:ModifyVPCAttribute", + "ec2:DeleteVPC", + "ec2:DescribeSubnets", + "ec2:CreateSubnet", + "ec2:ModifySubnetAttribute", + "ec2:DeleteSubnet", + "ec2:DescribeInternetGateways", + "ec2:CreateInternetGateway", + "ec2:AttachInternetGateway", + "ec2:DetachInternetGateway", + "ec2:DeleteInternetGateway", + "ec2:DescribeRouteTables", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:AssociateRouteTable", + "ec2:DisassociateRouteTable", + "ec2:DeleteRouteTable", + "ec2:CreateKeyPair", + "ec2:ImportKeyPair", + "ec2:DeleteKeyPair", + "ec2:DescribeKeyPairs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteRule", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyRule", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:RemoveTags", + "elasticloadbalancing:SetRulePriorities", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "kms:ListKeys", + "kms:ListResourceTags", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", + "kms:DescribeKey", + "kms:CreateKey", + "kms:Encrypt", + "kms:Decrypt", + "kms:ScheduleKeyDeletion", + "kms:ListAliases", + "kms:CreateAlias", + "kms:DeleteAlias", + "rds:DescribeDBSubnetGroups", + "rds:CreateDBSubnetGroup", + "rds:ModifyDBSubnetGroup", + "rds:DeleteDBSubnetGroup", + "rds:DescribeDBInstances", + "rds:CreateDBInstance", + "rds:ModifyDBInstance", + "rds:DeleteDBInstance", + "rds:ListTagsForResource", + "rds:AddTagsToResource", + "rds:RemoveTagsFromResource", + ] + resources = ["*"] + } +} diff --git a/enos/ci/service-user-iam/outputs.tf b/enos/ci/service-user-iam/outputs.tf new file mode 100644 index 000000000..8a502ced6 --- /dev/null +++ b/enos/ci/service-user-iam/outputs.tf @@ -0,0 +1,19 @@ +output "ci_role" { + value = local.is_ent ? { + name = aws_iam_role.github_actions_doormat_role[0].name + arn = aws_iam_role.github_actions_doormat_role[0].arn + } : { + name = aws_iam_role.role[0].name + arn = aws_iam_role.role[0].arn + } +} + +output "ci_role_policy" { + value = local.is_ent ? { + name = local.github_actions_doormat_assume_policy_name + policy = data.aws_iam_policy_document.github_actions_doormat_assume[0].json + } : { + name = aws_iam_role_policy.role_policy[0].name + policy = aws_iam_role_policy.role_policy[0].policy + } +} diff --git a/enos/ci/service-user-iam/providers.tf b/enos/ci/service-user-iam/providers.tf new file mode 100644 index 000000000..09c86d7ba --- /dev/null +++ b/enos/ci/service-user-iam/providers.tf @@ -0,0 +1,19 @@ +provider "aws" { + region = "us-east-1" + alias = "us_east_1" +} + +provider "aws" { + region = "us-east-2" + alias = "us_east_2" +} + +provider "aws" { + region = "us-west-1" + alias = "us_west_1" +} + +provider "aws" { + region = "us-west-2" + alias = "us_west_2" +} diff --git a/enos/ci/service-user-iam/service-quotas.tf b/enos/ci/service-user-iam/service-quotas.tf new file mode 100644 index 000000000..306daab33 --- /dev/null +++ b/enos/ci/service-user-iam/service-quotas.tf @@ -0,0 +1,62 @@ +locals { + // This is the code of the service quota to request a change for. Each adjustable limit has a + // unique code. See, https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicequotas_service_quota#quota_code + subnets_per_vpc = "L-F678F1CE" + rds_max_db_subnet_groups = "L-48C6BF61" +} + +resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_1" { + provider = aws.us_east_1 + quota_code = local.subnets_per_vpc + service_code = "vpc" + value = 50 +} + +resource "aws_servicequotas_service_quota" "rds_subnet_groups_us_east_1" { + provider = aws.us_east_1 + quota_code = local.rds_max_db_subnet_groups + service_code = "rds" + value = 50 +} + +resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_2" { + provider = aws.us_east_2 + quota_code = local.subnets_per_vpc + service_code = "vpc" + value = 50 +} + +resource "aws_servicequotas_service_quota" "rds_subnet_groups_us_east_2" { + provider = aws.us_east_2 + quota_code = local.rds_max_db_subnet_groups + service_code = "rds" + value = 50 +} + +resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_1" { + provider = aws.us_west_1 + quota_code = local.subnets_per_vpc + service_code = "vpc" + value = 50 +} + +resource "aws_servicequotas_service_quota" "rds_subnet_groups_us_west_1" { + provider = aws.us_west_1 + quota_code = local.rds_max_db_subnet_groups + service_code = "rds" + value = 50 +} + +resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_2" { + provider = aws.us_west_2 + quota_code = local.subnets_per_vpc + service_code = "vpc" + value = 50 +} + +resource "aws_servicequotas_service_quota" "rds_subnet_groups_us_west_2" { + provider = aws.us_west_2 + quota_code = local.rds_max_db_subnet_groups + service_code = "rds" + value = 50 +} diff --git a/enos/ci/service-user-iam/variables.tf b/enos/ci/service-user-iam/variables.tf new file mode 100644 index 000000000..c586e8cdd --- /dev/null +++ b/enos/ci/service-user-iam/variables.tf @@ -0,0 +1,8 @@ +variable "repository" { + description = "The GitHub repository, either boundary, boundary-enterprise, or boundary-hcp" + type = string + validation { + condition = contains(["boundary", "boundary-enterprise", "boundary-hcp"], var.repository) + error_message = "Invalid repository, only boundary, boundary-enterprise, and boundary-hcp are supported" + } +}