test(e2e): Add database migration test (#2627)

* test(e2e): Add database migration test

* chore(enos): Add enos scenario for migration test

* chore(enos): Update go test command to use count option

* doc(e2e): Update readme with migration test details

* Fix: Typo

* CR: Use different library

* CR: Add additional context in comment

* CR: Move some code from test into functions

* doc(e2e): Add instructions to more easily retrieve environment variables

* refact(e2e): Create some more methods to increase test readability

.github/workflows/enos-run.yml

@@ -25,10 +25,11 @@ jobs:
       fail-fast: false # don't fail as that can skip required cleanup steps for jobs
       matrix:
         include:
-          - filter: 'integration test:cli_ui'
-          - filter: 'e2e_aws'
-          - filter: 'e2e_static'
-          - filter: 'e2e_static_with_vault'
+          - filter: 'integration test:cli_ui builder:crt'
+          - filter: 'e2e_aws builder:crt'
+          - filter: 'e2e_database'
+          - filter: 'e2e_static builder:crt'
+          - filter: 'e2e_static_with_vault builder:crt'
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{ secrets.SERVICE_USER_GITHUB_TOKEN }}
@@ -86,12 +87,12 @@ jobs:
           mkdir -p /tmp/test-deps
           ls -la /tmp/test-deps
       - name: Set up Node for Bats install
-        if: matrix.filter == 'integration test:cli_ui'
+        if: matrix.filter == 'integration test:cli_ui builder:crt'
         uses: actions/setup-node@v3
         with:
           node-version: 16
       - name: Install Bats via NPM
-        if: matrix.filter == 'integration test:cli_ui'
+        if: matrix.filter == 'integration test:cli_ui builder:crt'
         # Use npm so this workflow is portable on multiple runner distros
         run: npm install --location=global bats
       - name: Download jq for Bats CLI UI tests
@@ -101,7 +102,7 @@ jobs:
           mkdir -p /tmp/test-deps
           wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /tmp/test-deps/jq-bin
       - name: Install jq for Bats CLI UI tests
-        if: matrix.filter == 'integration test:cli_ui'
+        if: matrix.filter == 'integration test:cli_ui builder:crt'
         run: |
           chmod +x /tmp/test-deps/jq-bin
           sudo cp /tmp/test-deps/jq-bin /usr/local/bin/jq
@@ -123,7 +124,7 @@ jobs:
         run: |
           wget https://releases.hashicorp.com/vault/1.12.0/vault_1.12.0_linux_amd64.zip -O /tmp/test-deps/vault.zip
       - name: Install Vault for integration testing
-        if: matrix.filter == 'e2e_static_with_vault'
+        if: matrix.filter == 'e2e_static_with_vault builder:crt' || matrix.filter == 'e2e_database'
         run: |
           unzip /tmp/test-deps/vault.zip -d /usr/local/bin
       - name: Download Linux AMD64 Boundary bundle
@@ -150,8 +151,8 @@ jobs:
         run: |
           mkdir -p ./enos/terraform-plugin-cache
           export ENOS_VAR_enos_user=$GITHUB_ACTOR && \
-          enos scenario check --chdir ./enos ${{ matrix.filter }} builder:crt && \
-          enos scenario exec --chdir ./enos ${{ matrix.filter }} builder:crt --cmd "version"
+          enos scenario check --chdir ./enos ${{ matrix.filter }} && \
+          enos scenario exec --chdir ./enos ${{ matrix.filter }} --cmd "version"
       - name: Run Enos scenario
         id: run
         # Continue once and retry
@@ -169,7 +170,7 @@ jobs:
         run: |
           mkdir -p ./enos/terraform-plugin-cache
           export ENOS_VAR_enos_user=$GITHUB_ACTOR && \
-          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.filter }} builder:crt
+          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.filter }}
       - name: Retry Enos scenario
         id: run_retry
         if: steps.run.outcome == 'failure'
@@ -185,7 +186,7 @@ jobs:
           ENOS_VAR_skip_failing_bats_tests: "true"
         run: |
           export ENOS_VAR_enos_user=$GITHUB_ACTOR && \
-          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.filter }} builder:crt
+          enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.filter }}
       - name: Destroy Enos scenario
         env:
           ENOS_VAR_aws_region: us-east-1
@@ -199,7 +200,7 @@ jobs:
           ENOS_VAR_skip_failing_bats_tests: "true"
         run: |
           export ENOS_VAR_enos_user=$GITHUB_ACTOR && \
-          enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.filter }} builder:crt
+          enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.filter }}
       - name: Output debug information on failure
         if: ${{ failure() }}
         run: |

enos/enos-scenario-e2e-database.hcl

@@ -0,0 +1,133 @@
+scenario "e2e_database" {
+  terraform_cli = terraform_cli.default
+  terraform     = terraform.default
+  providers = [
+    provider.aws.default,
+    provider.enos.default
+  ]
+
+  locals {
+    aws_ssh_private_key_path = abspath(var.aws_ssh_private_key_path)
+    local_boundary_dir       = abspath(var.local_boundary_dir)
+    tags = merge({
+      "Project Name" : var.project_name
+      "Project" : "Enos",
+      "Environment" : "ci"
+    }, var.tags)
+  }
+
+  step "find_azs" {
+    module = module.az_finder
+
+    variables {
+      instance_type = [
+        var.worker_instance_type,
+        var.controller_instance_type
+      ]
+    }
+  }
+
+  step "create_base_infra" {
+    module = module.infra
+
+    variables {
+      availability_zones = step.find_azs.availability_zones
+      common_tags        = local.tags
+    }
+  }
+
+  step "create_tag" {
+    module = module.random_stringifier
+  }
+
+  step "create_tag_inputs" {
+    module     = module.generate_aws_host_tag_vars
+    depends_on = [step.create_tag]
+
+    variables {
+      tag_name  = step.create_tag.string
+      tag_value = "true"
+    }
+  }
+
+  step "create_targets_with_tag" {
+    module     = module.target
+    depends_on = [step.create_base_infra]
+
+    variables {
+      ami_id               = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
+      aws_ssh_keypair_name = var.aws_ssh_keypair_name
+      enos_user            = var.enos_user
+      instance_type        = var.target_instance_type
+      vpc_id               = step.create_base_infra.vpc_id
+      target_count         = 1
+      additional_tags      = step.create_tag_inputs.tag_map
+    }
+  }
+
+  step "create_test_id" {
+    module = module.random_stringifier
+    variables {
+      length = 5
+    }
+  }
+
+  step "iam_setup" {
+    module = module.iam_setup
+    depends_on = [
+      step.create_base_infra,
+      step.create_test_id
+    ]
+
+    variables {
+      test_id    = step.create_test_id.string
+      test_email = var.test_email
+    }
+  }
+
+  step "create_vault_cluster" {
+    module = module.vault
+    depends_on = [
+      step.create_base_infra,
+    ]
+
+    variables {
+      ami_id          = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
+      instance_type   = var.vault_instance_type
+      instance_count  = 1
+      kms_key_arn     = step.create_base_infra.kms_key_arn
+      storage_backend = "raft"
+      unseal_method   = "awskms"
+      vault_release = {
+        version = "1.12.0"
+        edition = "oss"
+      }
+      vpc_id = step.create_base_infra.vpc_id
+    }
+  }
+
+  step "run_e2e_test" {
+    module = module.test_e2e
+    depends_on = [
+      step.create_targets_with_tag,
+      step.iam_setup,
+      step.create_vault_cluster,
+    ]
+
+    variables {
+      test_package             = "github.com/hashicorp/boundary/testing/internal/e2e/tests/database"
+      local_boundary_dir       = local.local_boundary_dir
+      target_user              = "ubuntu"
+      aws_ssh_private_key_path = local.aws_ssh_private_key_path
+      aws_access_key_id        = step.iam_setup.access_key_id
+      aws_secret_access_key    = step.iam_setup.secret_access_key
+      aws_host_set_filter1     = step.create_tag_inputs.tag_string
+      vault_addr               = step.create_vault_cluster.instance_public_ips[0]
+      vault_root_token         = step.create_vault_cluster.vault_root_token
+    }
+  }
+
+  output "test_results" {
+    value = step.run_e2e_test.test_results
+  }
+}

enos/modules/test_e2e/main.tf

@@ -13,6 +13,7 @@ variable "test_package" {
 variable "alb_boundary_api_addr" {
   description = "URL of the Boundary instance"
   type        = string
+  default     = ""
 }
 variable "auth_method_id" {
   description = "Id of Auth Method used to login to Boundary instance"
@@ -22,10 +23,12 @@ variable "auth_method_id" {
 variable "auth_login_name" {
   description = "Name of admin user"
   type        = string
+  default     = ""
 }
 variable "auth_password" {
   description = "Password of admin user"
   type        = string
+  default     = ""
 }
 variable "local_boundary_dir" {
   description = "Local Path to boundary executable"
@@ -34,10 +37,12 @@ variable "local_boundary_dir" {
 variable "target_user" {
   description = "SSH username for target"
   type        = string
+  default     = ""
 }
 variable "aws_ssh_private_key_path" {
   description = "Local Path to key used to SSH onto created hosts"
   type        = string
+  default     = ""
 }
 variable "target_ip" {
   description = "IP address of target"
@@ -111,13 +116,13 @@ resource "enos_local_exec" "run_e2e_test" {
     VAULT_TOKEN                   = var.vault_root_token,
     E2E_AWS_ACCESS_KEY_ID         = var.aws_access_key_id,
     E2E_AWS_SECRET_ACCESS_KEY     = var.aws_secret_access_key,
-    E2E_AWS_HOST_SET_FILTER1      = var.aws_host_set_filter1,
-    E2E_AWS_HOST_SET_IPS1         = local.aws_host_set_ips1,
+    E2E_AWS_HOST_SET_FILTER       = var.aws_host_set_filter1,
+    E2E_AWS_HOST_SET_IPS          = local.aws_host_set_ips1,
     E2E_AWS_HOST_SET_FILTER2      = var.aws_host_set_filter2,
     E2E_AWS_HOST_SET_IPS2         = local.aws_host_set_ips2
   }
 
-  inline = ["PATH=\"${var.local_boundary_dir}:$PATH\" go test -v ${var.test_package}"]
+  inline = ["PATH=\"${var.local_boundary_dir}:$PATH\" go test -v ${var.test_package} -count=1"]
 }
 
 output "test_results" {

testing/internal/e2e/README.md

@@ -36,32 +36,50 @@ account.
 Set the appropriate environment variables...
 ```shell
 export E2E_TESTS=true  # This is needed for any e2e test. Otherwise, the test is skipped
+
+# For e2e/tests/static
 export BOUNDARY_ADDR=  # e.g. http://127.0.0.1:9200
 export E2E_PASSWORD_AUTH_METHOD_ID=  # e.g. ampw_1234567890
 export E2E_PASSWORD_ADMIN_LOGIN_NAME=  # e.g. "admin"
 export E2E_PASSWORD_ADMIN_PASSWORD=  # e.g. "password"
 
-# For e2e/host/static
 export E2E_TARGET_IP=  # e.g. 192.168.0.1
 export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
 export E2E_SSH_USER=  # e.g. ubuntu
 
-# For e2e/host/aws
-export E2E_AWS_ACCESS_KEY_ID=
-export E2E_AWS_SECRET_ACCESS_KEY=
-export E2E_AWS_HOST_SET_FILTER1=  # e.g. "tag:testtag=true"
-export E2E_AWS_HOST_SET_IPS1=  # e.g. "[\"1.2.3.4\", \"2.3.4.5\"]"
-export E2E_AWS_HOST_SET_FILTER2=  # e.g. "tag:testtagtwo=test"
-export E2E_AWS_HOST_SET_IPS2=  # e.g. "[\"1.2.3.4\"]
-export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
-export E2E_SSH_USER=  # e.g. ubuntu
+# For e2e/tests/static_with_vault
+export BOUNDARY_ADDR=  # e.g. http://127.0.0.1:9200
+export E2E_PASSWORD_AUTH_METHOD_ID=  # e.g. ampw_1234567890
+export E2E_PASSWORD_ADMIN_LOGIN_NAME=  # e.g. "admin"
+export E2E_PASSWORD_ADMIN_PASSWORD=  # e.g. "password"
 
-# For e2e/credential/vault
 export VAULT_ADDR=  # e.g. http://127.0.0.1:8200
 export VAULT_TOKEN=
 export E2E_TARGET_IP=  # e.g. 192.168.0.1
 export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
 export E2E_SSH_USER=  # e.g. ubuntu
+
+# For e2e/tests/aws
+export BOUNDARY_ADDR=  # e.g. http://127.0.0.1:9200
+export E2E_PASSWORD_AUTH_METHOD_ID=  # e.g. ampw_1234567890
+export E2E_PASSWORD_ADMIN_LOGIN_NAME=  # e.g. "admin"
+export E2E_PASSWORD_ADMIN_PASSWORD=  # e.g. "password"
+
+export E2E_AWS_ACCESS_KEY_ID=
+export E2E_AWS_SECRET_ACCESS_KEY=
+export E2E_AWS_HOST_SET_FILTER=  # e.g. "tag:testtag=true"
+export E2E_AWS_HOST_SET_IPS=  # e.g. "[\"1.2.3.4\", \"2.3.4.5\"]"
+export E2E_AWS_HOST_SET_FILTER2=  # e.g. "tag:testtagtwo=test"
+export E2E_AWS_HOST_SET_IPS2=  # e.g. "[\"1.2.3.4\"]
+export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
+export E2E_SSH_USER=  # e.g. ubuntu
+
+# For e2e/tests/database
+export E2E_AWS_ACCESS_KEY_ID=
+export E2E_AWS_SECRET_ACCESS_KEY=
+export E2E_AWS_HOST_SET_FILTER=  # e.g. "tag:testtag=true"
+export VAULT_ADDR=  # e.g. http://127.0.0.1:8200
+export VAULT_TOKEN=
 ```
 
 Then, run...
@@ -74,8 +92,9 @@ go test github.com/hashicorp/boundary/testing/e2e/target -v -run '^TestCreateTar
 
 ## Adding Tests
 
-Tests live under this directory. Additional tests can be added to an existing go package or a new
-one can be created. If a new package is created, a new enos scenario would also need to be created.
+Tests live in the `tests/` directory. Additional tests can be added to an existing go package or a
+new one can be created. If a new package is created, a new enos scenario would also need to be
+created.
 
 Enos is comprised of scenarios, where a scenario is the environment you want the tests to operate
 in. In one scenario, there may be a boundary cluster and a target. Another scenario might involve a
@@ -87,17 +106,17 @@ workflow must be updated to include the new scenario (see the `matrix`).
 ### Development
 To assist with iterating on tests on enos launched infrastructure, you can perform the following...
 
-Add the following snippet to print out environment variable information
-```
-# `c` is the output from `loadConfig()`
-s, _ := json.MarshalIndent(c, "", "\t")
-log.Printf("%s", s)
-```
-Launch an enos scenario
+Launch an enos scenario and print out the environment variables
 ```
+cd enos
 enos scenario launch e2e_{scenario} builder:local
 enos scenario output
+cd .enos
+ls -ltr
+cd <most_recent_directory> # bottom of list
+terraform show -json terraform.tfstate | jq -r '.values.root_module.child_modules[].resources[] | select(.address=="module.run_e2e_test.enos_local_exec.run_e2e_test") | .values.environment | to_entries[] | "export \(.key)=\(.value)"'
 ```
+
 Take the printed environment variable information and export them into another terminal session
 ```
 export BOUNDARY_ADDR=

testing/internal/e2e/boundary/boundary.go

@@ -21,11 +21,24 @@ type config struct {
 	AdminLoginPassword string `envconfig:"E2E_PASSWORD_ADMIN_PASSWORD" required:"true"`
 }
 
-type authenticateCliOutput struct {
+// AuthenticateCliOutput parses the json response from running `boundary authenticate`
+type AuthenticateCliOutput struct {
 	Item     *authmethods.AuthenticateResult
 	response *api.Response
 }
 
+// AuthMethodInfo parses auth method info in the json response from running `boundary database init`
+type AuthMethodInfo struct {
+	AuthMethodId string `json:"auth_method_id"`
+	LoginName    string `json:"login_name"`
+	Password     string `json:"password"`
+}
+
+// DbInitInfo parses the json response from running `boundary database init`
+type DbInitInfo struct {
+	AuthMethod AuthMethodInfo `json:"auth_method"`
+}
+
 func loadConfig() (*config, error) {
 	var c config
 	err := envconfig.Process("", &c)
@@ -110,7 +123,7 @@ func GetAuthenticationTokenCli(t testing.TB, ctx context.Context, loginName stri
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
-	var authenticationResult authenticateCliOutput
+	var authenticationResult AuthenticateCliOutput
 	err = json.Unmarshal(output.Stdout, &authenticationResult)
 	require.NoError(t, err)
 

testing/internal/e2e/infra/docker.go

@@ -0,0 +1,223 @@
+package infra
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/ssh"
+)
+
+// Container stores information about the docker container
+type Container struct {
+	Resource     *dockertest.Resource
+	UriLocalhost string
+	UriNetwork   string
+}
+
+// StartBoundaryDatabase spins up a postgres database in a docker container.
+// Returns information about the container
+func StartBoundaryDatabase(t testing.TB, pool *dockertest.Pool, network *dockertest.Network) *Container {
+	t.Log("Starting postgres database...")
+
+	networkAlias := "e2epostgres"
+	postgresDb := "e2eboundarydb"
+	postgresUser := "e2eboundary"
+	postgresPassword := "e2eboundary"
+	postgresConfigFilePath, err := filepath.Abs("testdata/postgresql.conf")
+	require.NoError(t, err)
+
+	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "postgres",
+		Tag:        "13-alpine",
+		Cmd:        []string{"postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"},
+		Env: []string{
+			"POSTGRES_DB=" + postgresDb,
+			"POSTGRES_USER=" + postgresUser,
+			"POSTGRES_PASSWORD=" + postgresPassword,
+		},
+		Mounts:       []string{path.Dir(postgresConfigFilePath) + ":/etc/postgresql/"},
+		ExposedPorts: []string{"5432/tcp"},
+		Name:         networkAlias,
+		Networks:     []*dockertest.Network{network},
+	})
+	require.NoError(t, err)
+
+	return &Container{
+		Resource: resource,
+		UriLocalhost: fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable",
+			postgresUser,
+			postgresPassword,
+			resource.GetHostPort("5432/tcp"),
+			postgresDb,
+		),
+		UriNetwork: fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable",
+			postgresUser,
+			postgresPassword,
+			networkAlias,
+			"5432",
+			postgresDb,
+		),
+	}
+}
+
+// InitBoundaryDatabase starts a boundary container (of the latest released version) and initializes a
+// postgres database (using `boundary database init`) at the specified postgres URI.
+// Returns information about the container
+func InitBoundaryDatabase(t testing.TB, pool *dockertest.Pool, network *dockertest.Network, postgresURI string) *Container {
+	t.Log("Initializing postgres database...")
+
+	boundaryConfigFilePath, err := filepath.Abs("testdata/boundary-config.hcl")
+	require.NoError(t, err)
+
+	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "hashicorp/boundary",
+		Tag:        "latest",
+		Cmd:        []string{"boundary", "database", "init", "-config", "/boundary/boundary-config.hcl", "-format", "json"},
+		Env: []string{
+			"BOUNDARY_POSTGRES_URL=" + postgresURI,
+			"SKIP_CHOWN=true",
+		},
+		Mounts:   []string{path.Dir(boundaryConfigFilePath) + ":/boundary/"},
+		Name:     "boundary-init",
+		Networks: []*dockertest.Network{network},
+		CapAdd:   []string{"IPC_LOCK"},
+	})
+	require.NoError(t, err)
+
+	return &Container{Resource: resource}
+}
+
+// GetDbInitInfoFromContainer extracts info from calling `boundary database init` in the specified
+// container. Returns a struct containing the generated info.
+func GetDbInitInfoFromContainer(t testing.TB, pool *dockertest.Pool, container *Container) boundary.DbInitInfo {
+	_, err := pool.Client.WaitContainer(container.Resource.Container.ID)
+	require.NoError(t, err)
+	buf := bytes.NewBuffer(nil)
+	ebuf := bytes.NewBuffer(nil)
+	err = pool.Client.Logs(docker.LogsOptions{
+		Container:    container.Resource.Container.ID,
+		OutputStream: buf,
+		ErrorStream:  ebuf,
+		Follow:       true,
+		Stdout:       true,
+		Stderr:       true,
+	})
+	require.NoError(t, err)
+	require.Empty(t, ebuf)
+
+	var dbInitInfo boundary.DbInitInfo
+	err = json.Unmarshal(buf.Bytes(), &dbInitInfo)
+	require.NoError(t, err, buf.String())
+
+	return dbInitInfo
+}
+
+// StartBoundary starts a boundary container and spins up an instance of boundary using the
+// specified database at postgresURI. Returns information about the container.
+// Returns information about the container
+func StartBoundary(t testing.TB, pool *dockertest.Pool, network *dockertest.Network, postgresURI string) *Container {
+	t.Log("Starting Boundary...")
+
+	boundaryConfigFilePath, err := filepath.Abs("testdata/boundary-config.hcl")
+	require.NoError(t, err)
+
+	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "hashicorp/boundary",
+		Tag:        "latest",
+		Cmd:        []string{"boundary", "server", "-config", "/boundary/boundary-config.hcl"},
+		Env: []string{
+			"BOUNDARY_POSTGRES_URL=" + postgresURI,
+			"HOSTNAME=boundary",
+			"SKIP_CHOWN=true",
+		},
+		Mounts:       []string{path.Dir(boundaryConfigFilePath) + ":/boundary/"},
+		Name:         "boundary",
+		Networks:     []*dockertest.Network{network},
+		ExposedPorts: []string{"9200", "9201", "9202", "9203"},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"9200/tcp": {{HostIP: "localhost", HostPort: "9200/tcp"}},
+			"9201/tcp": {{HostIP: "localhost", HostPort: "9201/tcp"}},
+			"9202/tcp": {{HostIP: "localhost", HostPort: "9202/tcp"}},
+			"9203/tcp": {{HostIP: "localhost", HostPort: "9203/tcp"}},
+		},
+		CapAdd: []string{"IPC_LOCK"},
+	})
+	require.NoError(t, err)
+
+	return &Container{
+		Resource:     resource,
+		UriLocalhost: "http://localhost:9200",
+		UriNetwork:   "http://boundary:9200",
+	}
+}
+
+// ConnectToTarget starts a boundary container and attempts to connect to the specified target. The
+// goal of this method is to create a session entry in the database.
+// Returns information about the container.
+func ConnectToTarget(t testing.TB, pool *dockertest.Pool, network *dockertest.Network, boundaryAddr string, token string, targetId string) *Container {
+	t.Log("Connecting to target...")
+
+	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "hashicorp/boundary",
+		Tag:        "latest",
+		Cmd: []string{
+			"boundary", "connect",
+			"-token", "env://E2E_AUTH_TOKEN",
+			"-target-id", targetId,
+			"-exec", "ls", // Execute something so that the command exits
+			// Note: Would have used `connect ssh` here, but ssh does not exist in the image. Also,
+			// this method only cares about creating a session entry in the database, so the ssh is unnecessary
+		},
+		Env: []string{
+			"BOUNDARY_ADDR=" + boundaryAddr,
+			"E2E_AUTH_TOKEN=" + token,
+			"SKIP_CHOWN=true",
+		},
+		Name:     "boundary-client",
+		Networks: []*dockertest.Network{network},
+		CapAdd:   []string{"IPC_LOCK"},
+	})
+	require.NoError(t, err)
+
+	return &Container{Resource: resource}
+}
+
+// StartOpenSshServer starts an openssh container to serve as a target for Boundary.
+// Returns information about the container.
+func StartOpenSshServer(t testing.TB, pool *dockertest.Pool, network *dockertest.Network, user string, privateKeyFilePath string) *Container {
+	t.Log("Starting openssh-server to serve as target...")
+
+	privateKeyRaw, err := os.ReadFile(privateKeyFilePath)
+	require.NoError(t, err)
+	signer, err := ssh.ParsePrivateKey(privateKeyRaw)
+	require.NoError(t, err)
+
+	networkAlias := "target"
+	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "lscr.io/linuxserver/openssh-server",
+		Env: []string{
+			"PUID=1000",
+			"PGID=1000",
+			"TZ=US/Eastern",
+			"USER_NAME=" + user,
+			"PUBLIC_KEY=" + string(ssh.MarshalAuthorizedKey(signer.PublicKey())),
+		},
+		Name:     networkAlias,
+		Networks: []*dockertest.Network{network},
+	})
+	require.NoError(t, err)
+
+	return &Container{
+		Resource:   resource,
+		UriNetwork: networkAlias,
+	}
+}

testing/internal/e2e/tests/aws/dynamichostcatalog_test.go

@@ -24,8 +24,8 @@ import (
 type config struct {
 	AwsAccessKeyId     string `envconfig:"E2E_AWS_ACCESS_KEY_ID" required:"true"`
 	AwsSecretAccessKey string `envconfig:"E2E_AWS_SECRET_ACCESS_KEY" required:"true"`
-	AwsHostSetFilter1  string `envconfig:"E2E_AWS_HOST_SET_FILTER1" required:"true"` // e.g. "tag:testtag=true"
-	AwsHostSetIps1     string `envconfig:"E2E_AWS_HOST_SET_IPS1" required:"true"`    // e.g. "[\"1.2.3.4\", \"2.3.4.5\"]"
+	AwsHostSetFilter1  string `envconfig:"E2E_AWS_HOST_SET_FILTER" required:"true"`  // e.g. "tag:testtag=true"
+	AwsHostSetIps1     string `envconfig:"E2E_AWS_HOST_SET_IPS" required:"true"`     // e.g. "[\"1.2.3.4\", \"2.3.4.5\"]"
 	AwsHostSetFilter2  string `envconfig:"E2E_AWS_HOST_SET_FILTER2" required:"true"` // e.g. "tag:testtagtwo=test"
 	AwsHostSetIps2     string `envconfig:"E2E_AWS_HOST_SET_IPS2" required:"true"`    // e.g. "[\"1.2.3.4\"]"
 	TargetSshKeyPath   string `envconfig:"E2E_SSH_KEY_PATH" required:"true"`         // e.g. "/Users/username/key.pem"

testing/internal/e2e/tests/database/env_test.go

@@ -0,0 +1,23 @@
+package database_test
+
+import "github.com/kelseyhightower/envconfig"
+
+type config struct {
+	TargetSshUser      string `envconfig:"E2E_SSH_USER" required:"true"`     // e.g. ubuntu
+	TargetSshKeyPath   string `envconfig:"E2E_SSH_KEY_PATH" required:"true"` // e.g. /Users/username/key.pem
+	TargetPort         string `envconfig:"E2E_SSH_PORT" default:"22"`
+	VaultSecretPath    string `envconfig:"E2E_VAULT_SECRET_PATH" default:"e2e_secrets"`
+	AwsAccessKeyId     string `envconfig:"E2E_AWS_ACCESS_KEY_ID" required:"true"`
+	AwsSecretAccessKey string `envconfig:"E2E_AWS_SECRET_ACCESS_KEY" required:"true"`
+	AwsHostSetFilter   string `envconfig:"E2E_AWS_HOST_SET_FILTER" required:"true"` // e.g. "tag:testtag=true"
+}
+
+func loadConfig() (*config, error) {
+	var c config
+	err := envconfig.Process("", &c)
+	if err != nil {
+		return nil, err
+	}
+
+	return &c, nil
+}

testing/internal/e2e/tests/database/migration_test.go

@@ -0,0 +1,297 @@
+package database_test
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/boundary/api/credentiallibraries"
+	"github.com/hashicorp/boundary/testing/internal/e2e"
+	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
+	"github.com/hashicorp/boundary/testing/internal/e2e/infra"
+	"github.com/hashicorp/boundary/testing/internal/e2e/vault"
+	_ "github.com/jackc/pgx/v4/stdlib"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/require"
+)
+
+type TestEnvironment struct {
+	Pool       *dockertest.Pool
+	Network    *dockertest.Network
+	Boundary   *infra.Container
+	Database   *infra.Container
+	Target     *infra.Container
+	DbInitInfo boundary.DbInitInfo
+}
+
+// TestDatabaseMigration tests migrating the Boundary database from the latest released version to
+// the version under test. It creates a database, populates the database with a number of resources,
+// and uses the Boundary version under test to migrate the database.
+func TestDatabaseMigration(t *testing.T) {
+	e2e.MaybeSkipTest(t)
+	c, err := loadConfig()
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	te := setupEnvironment(t, ctx, c)
+	populateBoundaryDatabase(t, ctx, c, te)
+
+	// Migrate database
+	t.Log("Stopping boundary before migrating...")
+	err = te.Pool.Client.StopContainer(te.Boundary.Resource.Container.ID, 10)
+	require.NoError(t, err)
+
+	bonfigFilePath, err := filepath.Abs("testdata/boundary-config.hcl")
+	require.NoError(t, err)
+
+	t.Log("Starting database migration...")
+	output := e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs("database", "migrate", "-config", bonfigFilePath),
+		e2e.WithEnv("BOUNDARY_POSTGRES_URL", te.Database.UriLocalhost),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	t.Logf("%s", output.Stdout)
+	t.Logf("Migration Output: %s", output.Stderr)
+}
+
+func setupEnvironment(t testing.TB, ctx context.Context, c *config) TestEnvironment {
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err)
+	err = pool.Client.Ping()
+	require.NoError(t, err)
+
+	// This ensures that the latest Boundary image is used
+	err = pool.Client.PullImage(docker.PullImageOptions{
+		Repository: "hashicorp/boundary",
+		Tag:        "latest",
+	}, docker.AuthConfiguration{})
+	require.NoError(t, err)
+
+	// Set up docker network
+	network, err := pool.CreateNetwork(t.Name())
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, pool.RemoveNetwork(network))
+	})
+
+	// Start a Boundary database and wait until it's ready
+	db := infra.StartBoundaryDatabase(t, pool, network)
+	t.Cleanup(func() {
+		pool.Purge(db.Resource)
+	})
+
+	pool.MaxWait = 10 * time.Second
+	t.Log("Waiting for database to load...")
+	err = pool.Retry(func() error {
+		db, err := sql.Open("pgx", db.UriLocalhost)
+		if err != nil {
+			return err
+		}
+		return db.Ping()
+	})
+	require.NoError(t, err)
+
+	// Create a target
+	target := infra.StartOpenSshServer(t, pool, network, c.TargetSshUser, c.TargetSshKeyPath)
+	t.Cleanup(func() {
+		pool.Purge(target.Resource)
+	})
+
+	// Initialize the database and extract resulting information
+	dbInit := infra.InitBoundaryDatabase(t, pool, network, db.UriNetwork)
+	t.Cleanup(func() {
+		pool.Purge(dbInit.Resource)
+	})
+	dbInitInfo := infra.GetDbInitInfoFromContainer(t, pool, dbInit)
+
+	// Start a Boundary server and wait until Boundary has finished loading
+	b := infra.StartBoundary(t, pool, network, db.UriNetwork)
+	t.Cleanup(func() {
+		pool.Purge(b.Resource)
+	})
+
+	buf := bytes.NewBuffer(nil)
+	ebuf := bytes.NewBuffer(nil)
+	_, err = b.Resource.Exec([]string{"boundary", "version"}, dockertest.ExecOptions{
+		StdOut: buf,
+		StdErr: ebuf,
+	})
+	require.NoError(t, err)
+	require.Empty(t, ebuf)
+	t.Logf("Using Boundary Version: %s", buf.String())
+
+	t.Log("Waiting for Boundary to finish loading...")
+	err = pool.Retry(func() error {
+		response, err := http.Get(b.UriLocalhost)
+		if err != nil {
+			t.Logf("Could not access Boundary URL: %s. Retrying...", err.Error())
+			return err
+		}
+
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("Could not connect to %s. Status Code: %d", b.UriLocalhost, response.StatusCode)
+		}
+
+		return nil
+	})
+	require.NoError(t, err)
+
+	return TestEnvironment{
+		Pool:       pool,
+		Network:    network,
+		Boundary:   b,
+		Database:   db,
+		Target:     target,
+		DbInitInfo: dbInitInfo,
+	}
+}
+
+func populateBoundaryDatabase(t testing.TB, ctx context.Context, c *config, te TestEnvironment) {
+	os.Setenv("BOUNDARY_ADDR", te.Boundary.UriLocalhost)
+	os.Setenv("E2E_PASSWORD_AUTH_METHOD_ID", te.DbInitInfo.AuthMethod.AuthMethodId)
+	os.Setenv("E2E_PASSWORD_ADMIN_LOGIN_NAME", te.DbInitInfo.AuthMethod.LoginName)
+	os.Setenv("E2E_PASSWORD_ADMIN_PASSWORD", te.DbInitInfo.AuthMethod.Password)
+
+	// Create resources for target. Uses the local CLI so that these methods can be reused.
+	// While the CLI version used won't necessarily match the controller version, it should be (and is
+	// supposed to be) backwards ompatible
+	boundary.AuthenticateAdminCli(t, ctx)
+	newOrgId := boundary.CreateNewOrgCli(t, ctx)
+	newProjectId := boundary.CreateNewProjectCli(t, ctx, newOrgId)
+	newHostCatalogId := boundary.CreateNewHostCatalogCli(t, ctx, newProjectId)
+	newHostSetId := boundary.CreateNewHostSetCli(t, ctx, newHostCatalogId)
+	newHostId := boundary.CreateNewHostCli(t, ctx, newHostCatalogId, te.Target.UriNetwork)
+	boundary.AddHostToHostSetCli(t, ctx, newHostSetId, newHostId)
+	newTargetId := boundary.CreateNewTargetCli(t, ctx, newProjectId, "2222") // openssh-server uses port 2222
+	boundary.AddHostSourceToTargetCli(t, ctx, newTargetId, newHostSetId)
+
+	// Create AWS dynamic host catalog
+	newAwsHostCatalogId := boundary.CreateNewAwsHostCatalogCli(t, ctx, newProjectId, c.AwsAccessKeyId, c.AwsSecretAccessKey)
+	newAwsHostSetId := boundary.CreateNewAwsHostSetCli(t, ctx, newAwsHostCatalogId, c.AwsHostSetFilter)
+	boundary.WaitForHostsInHostSetCli(t, ctx, newAwsHostSetId)
+
+	// Create static credentials
+	newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, ctx, newProjectId)
+	boundary.CreateNewStaticCredentialPasswordCli(t, ctx, newCredentialStoreId, c.TargetSshUser, "password")
+	boundary.CreateNewStaticCredentialJsonCli(t, ctx, newCredentialStoreId, "testdata/credential.json")
+	newCredentialsId := boundary.CreateNewStaticCredentialPrivateKeyCli(t, ctx, newCredentialStoreId, c.TargetSshUser, c.TargetSshKeyPath)
+	boundary.AddCredentialSourceToTargetCli(t, ctx, newTargetId, newCredentialsId)
+
+	// Create vault credentials
+	vaultAddr, boundaryPolicyName, kvPolicyFilePath := vault.Setup(t)
+	t.Cleanup(func() {
+		output := e2e.RunCommand(ctx, "vault",
+			e2e.WithArgs("policy", "delete", boundaryPolicyName),
+		)
+		require.NoError(t, output.Err, string(output.Stderr))
+	})
+	output := e2e.RunCommand(ctx, "vault",
+		e2e.WithArgs("secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2"),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	t.Cleanup(func() {
+		output := e2e.RunCommand(ctx, "vault",
+			e2e.WithArgs("secrets", "disable", c.VaultSecretPath),
+		)
+		require.NoError(t, output.Err, string(output.Stderr))
+	})
+	privateKeySecretName := vault.CreateKvPrivateKeyCredential(t, c.VaultSecretPath, c.TargetSshUser, c.TargetSshKeyPath, kvPolicyFilePath)
+	kvPolicyName := vault.WritePolicy(t, ctx, kvPolicyFilePath)
+	t.Cleanup(func() {
+		output := e2e.RunCommand(ctx, "vault",
+			e2e.WithArgs("policy", "delete", kvPolicyName),
+		)
+		require.NoError(t, output.Err, string(output.Stderr))
+	})
+	t.Log("Created Vault Credential")
+	output = e2e.RunCommand(ctx, "vault",
+		e2e.WithArgs(
+			"token", "create",
+			"-no-default-policy=true",
+			"-policy="+boundaryPolicyName,
+			"-policy="+kvPolicyName,
+			"-orphan=true",
+			"-period=20m",
+			"-renewable=true",
+			"-format=json",
+		),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var tokenCreateResult vault.CreateTokenResponse
+	err := json.Unmarshal(output.Stdout, &tokenCreateResult)
+	require.NoError(t, err)
+	credStoreToken := tokenCreateResult.Auth.Client_Token
+	t.Log("Created Vault Cred Store Token")
+	newVaultCredentialStoreId := boundary.CreateNewCredentialStoreVaultCli(t, ctx, newProjectId, vaultAddr, credStoreToken)
+	output = e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs(
+			"credential-libraries", "create", "vault",
+			"-credential-store-id", newVaultCredentialStoreId,
+			"-vault-path", c.VaultSecretPath+"/data/"+privateKeySecretName,
+			"-name", "e2e Automated Test Vault Credential Library",
+			"-credential-type", "ssh_private_key",
+			"-format", "json",
+		),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newCredentialLibraryResult credentiallibraries.CredentialLibraryCreateResult
+	err = json.Unmarshal(output.Stdout, &newCredentialLibraryResult)
+	require.NoError(t, err)
+	newCredentialLibraryId := newCredentialLibraryResult.Item.Id
+	t.Logf("Created Credential Library: %s", newCredentialLibraryId)
+
+	// Create a session. Uses Boundary in a docker container to do the connect in order to avoid
+	// modifying the runner's /etc/hosts file. Otherwise, you would need to add a `127.0.0.1
+	// localhost boundary` entry into /etc/hosts.
+	buf := bytes.NewBuffer(nil)
+	ebuf := bytes.NewBuffer(nil)
+	_, err = te.Boundary.Resource.Exec(
+		[]string{
+			"boundary", "authenticate", "password",
+			"-addr", te.Boundary.UriNetwork,
+			"-auth-method-id", te.DbInitInfo.AuthMethod.AuthMethodId,
+			"-login-name", te.DbInitInfo.AuthMethod.LoginName,
+			"-password", "env://E2E_TEST_BOUNDARY_PASSWORD",
+			"-format", "json",
+		},
+		dockertest.ExecOptions{
+			StdOut: buf,
+			StdErr: ebuf,
+			Env:    []string{"E2E_TEST_BOUNDARY_PASSWORD=" + te.DbInitInfo.AuthMethod.Password},
+		},
+	)
+	require.NoError(t, err)
+	require.Empty(t, ebuf)
+	var authenticationResult boundary.AuthenticateCliOutput
+	err = json.Unmarshal(buf.Bytes(), &authenticationResult)
+	require.NoError(t, err)
+	auth_token := authenticationResult.Item.Attributes["token"].(string)
+
+	connectTarget := infra.ConnectToTarget(t, te.Pool, te.Network, te.Boundary.UriNetwork, auth_token, newTargetId)
+	t.Cleanup(func() {
+		te.Pool.Purge(connectTarget.Resource)
+	})
+	_, err = te.Pool.Client.WaitContainer(connectTarget.Resource.Container.ID)
+	require.NoError(t, err)
+	buf = bytes.NewBuffer(nil)
+	ebuf = bytes.NewBuffer(nil)
+	err = te.Pool.Client.Logs(docker.LogsOptions{
+		Container:    connectTarget.Resource.Container.ID,
+		OutputStream: buf,
+		ErrorStream:  ebuf,
+		Follow:       true,
+		Stdout:       true,
+		Stderr:       true,
+	})
+	require.NoError(t, err)
+	require.Empty(t, ebuf)
+	t.Log("Created session")
+}

testing/internal/e2e/tests/database/testdata/boundary-config.hcl

@@ -0,0 +1,102 @@
+disable_mlock = true
+
+controller {
+  name        = "docker-controller"
+  description = "A controller for a docker demo!"
+
+  database {
+    url = "env://BOUNDARY_POSTGRES_URL"
+  }
+}
+
+worker {
+  name        = "boundary-colocated-worker"
+  description = "A worker that runs alongside the controller in the same process"
+  address     = "boundary:9201"
+}
+
+listener "tcp" {
+  address     = "boundary:9200"
+  purpose     = "api"
+  tls_disable = true
+}
+
+listener "tcp" {
+  address     = "boundary:9201"
+  purpose     = "cluster"
+  tls_disable = true
+}
+
+listener "tcp" {
+  address     = "boundary:9202"
+  purpose     = "proxy"
+  tls_disable = true
+}
+
+listener "tcp" {
+  address     = "boundary:9203"
+  purpose     = "ops"
+  tls_disable = true
+}
+
+kms "aead" {
+  purpose   = "root"
+  aead_type = "aes-gcm"
+  key       = "sP1fnF5Xz85RrXyELHFeZg9Ad2qt4Z4bgNHVGtD6ung="
+  key_id    = "global_root"
+}
+
+kms "aead" {
+  purpose   = "worker-auth"
+  aead_type = "aes-gcm"
+  key       = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ="
+  key_id    = "global_worker-auth"
+}
+
+kms "aead" {
+  purpose   = "recovery"
+  aead_type = "aes-gcm"
+  key       = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ="
+  key_id    = "global_recovery"
+}
+
+events {
+  audit_enabled        = true
+  observations_enabled = true
+  sysevents_enabled    = true
+
+  sink "stderr" {
+    name        = "all-events"
+    description = "All events sent to stderr"
+    event_types = ["*"]
+    format      = "cloudevents-json"
+
+    deny_filters = [
+      "\"/data/request_info/method\" contains \"Status\"",
+      "\"/data/request_info/path\" contains \"/health\"",
+    ]
+  }
+
+  sink {
+    name        = "audit-sink"
+    description = "Audit sent to a file"
+    event_types = ["audit"]
+    format      = "cloudevents-json"
+
+    deny_filters = [
+      "\"/data/request_info/method\" contains \"Status\"",
+    ]
+
+    file {
+      path      = "/logs"
+      file_name = "audit.log"
+    }
+
+    audit_config {
+      audit_filter_overrides {
+        secret    = "encrypt"
+        sensitive = "hmac-sha256"
+      }
+    }
+  }
+}

testing/internal/e2e/tests/database/testdata/credential.json

@@ -0,0 +1,4 @@
+{
+    "a": "test",
+    "b": "foo"
+}

testing/internal/e2e/tests/database/testdata/postgresql.conf

@@ -0,0 +1,14 @@
+listen_addresses           = '*'
+max_connections            = 100
+work_mem                   = 50000kB
+shared_buffers             = 64MB
+log_destination            = 'stderr'
+deadlock_timeout           = 100
+log_min_messages           = info
+log_min_error_statement    = error
+log_error_verbosity        = verbose
+log_lock_waits             = true
+log_statement              = all
+log_duration               = on
+log_min_duration_statement = 0
+log_line_prefix            = '%m [%p] (%x) %i '

testing/internal/e2e/tests/static_with_vault/testdata/boundary-controller-policy.hcl

@@ -0,0 +1,23 @@
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/renew-self" {
+  capabilities = ["update"]
+}
+
+path "auth/token/revoke-self" {
+  capabilities = ["update"]
+}
+
+path "sys/leases/renew" {
+  capabilities = ["update"]
+}
+
+path "sys/leases/revoke" {
+  capabilities = ["update"]
+}
+
+path "sys/capabilities-self" {
+  capabilities = ["update"]
+}

testing/internal/e2e/vault/vault.go

@@ -3,9 +3,9 @@ package vault
 
 import (
 	"context"
-	"embed"
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/hashicorp/boundary/testing/internal/e2e"
@@ -14,14 +14,12 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-//go:embed testdata/boundary-controller-policy.hcl
-var boundaryPolicyFile embed.FS
-
 type config struct {
 	VaultAddr  string `envconfig:"VAULT_ADDR" required:"true"` // e.g. "http://127.0.0.1:8200"
 	VaultToken string `envconfig:"VAULT_TOKEN" required:"true"`
 }
 
+// CreateTokenResponse parses the json response from running `vault token create`
 type CreateTokenResponse struct {
 	Auth struct {
 		Client_Token string
@@ -46,14 +44,8 @@ func Setup(t testing.TB) (vaultAddr string, boundaryPolicyName string, kvPolicyF
 	vaultAddr = c.VaultAddr
 
 	// Set up boundary policy
-	data, err := boundaryPolicyFile.ReadFile("testdata/boundary-controller-policy.hcl")
+	boundaryPolicyFilePath, err := filepath.Abs("testdata/boundary-controller-policy.hcl")
 	require.NoError(t, err)
-	boundaryPolicyFilePath := fmt.Sprintf("%s/%s", t.TempDir(), "boundary-controller-policy.hcl")
-	f, err := os.Create(boundaryPolicyFilePath)
-	require.NoError(t, err)
-	_, err = f.WriteString(string(data))
-	require.NoError(t, err)
-
 	boundaryPolicyName = WritePolicy(t, context.Background(), boundaryPolicyFilePath)
 
 	// Create kv policy