From 7816772ef2e1e57125fa905c12450b0f17a39b2e Mon Sep 17 00:00:00 2001
From: Michael Li <michael.li@hashicorp.com>
Date: Thu, 23 Feb 2023 18:30:51 -0500
Subject: [PATCH] test(admin/e2e): Add first playwright test (#1637)

Adds an end-to-end test for the Admin UI using Playwright. This test uses the Admin UI to create the necessary resources to connect to a target. The test then uses the CLI to start a session and checks in the Admin UI that a session is created
---
 ui/admin/.gitignore                        |   2 +
 ui/admin/package.json                      |   7 +-
 ui/admin/tests/e2e/README.md               |  49 ++++++
 ui/admin/tests/e2e/global-setup.js         |  28 +++
 ui/admin/tests/e2e/helpers/boundary-cli.js |  33 ++++
 ui/admin/tests/e2e/helpers/boundary-ui.js  | 187 +++++++++++++++++++++
 ui/admin/tests/e2e/helpers/general.js      |  19 +++
 ui/admin/tests/e2e/playwright.config.js    |  31 ++++
 ui/admin/tests/e2e/tests/target.spec.js    | 106 ++++++++++++
 9 files changed, 461 insertions(+), 1 deletion(-)
 create mode 100644 ui/admin/.gitignore
 create mode 100644 ui/admin/tests/e2e/README.md
 create mode 100644 ui/admin/tests/e2e/global-setup.js
 create mode 100644 ui/admin/tests/e2e/helpers/boundary-cli.js
 create mode 100644 ui/admin/tests/e2e/helpers/boundary-ui.js
 create mode 100644 ui/admin/tests/e2e/helpers/general.js
 create mode 100644 ui/admin/tests/e2e/playwright.config.js
 create mode 100644 ui/admin/tests/e2e/tests/target.spec.js

diff --git a/ui/admin/.gitignore b/ui/admin/.gitignore
new file mode 100644
index 000000000..a79d7f41e
--- /dev/null
+++ b/ui/admin/.gitignore
@@ -0,0 +1,2 @@
+# e2e test output
+tests/e2e/artifacts
diff --git a/ui/admin/package.json b/ui/admin/package.json
index f33919013..06d8f63ba 100644
--- a/ui/admin/package.json
+++ b/ui/admin/package.json
@@ -33,6 +33,10 @@
     "pretest": "yarn clean:coverage",
     "test": "npm-run-all test:*",
     "test:ember": "COVERAGE=true ember exam --test-port 0 --split=4 --parallel=1 --random",
+    "e2e": "yarn playwright test --config ./tests/e2e/playwright.config.js",
+    "e2e:dev:chrome": "yarn run e2e --project=chromium --headed",
+    "e2e:dev:firefox": "yarn run e2e --project=firefox --headed",
+    "e2e:dev:safari": "yarn run e2e --project=webkit --headed",
     "posttest": "ember coverage-merge && yarn clean:coverage"
   },
   "dependencies": {
@@ -48,6 +52,7 @@
     "@faker-js/faker": "^7.3.0",
     "@glimmer/component": "^1.1.2",
     "@glimmer/tracking": "^1.1.2",
+    "@playwright/test": "^1.30.0",
     "babel-loader": "^8.2.5",
     "broccoli-asset-rev": "^3.0.0",
     "clipboard": "^2.0.8",
@@ -91,10 +96,10 @@
     "qunit": "^2.19.1",
     "qunit-dom": "^2.0.0",
     "rose": "*",
+    "sinon": "^14.0.0",
     "stylelint": "^14.14.0",
     "stylelint-config-prettier-scss": "^0.0.1",
     "stylelint-config-standard-scss": "^6.0.0",
-    "sinon": "^14.0.0",
     "webpack": "^5.72.1"
   },
   "engines": {
diff --git a/ui/admin/tests/e2e/README.md b/ui/admin/tests/e2e/README.md
new file mode 100644
index 000000000..9560a2109
--- /dev/null
+++ b/ui/admin/tests/e2e/README.md
@@ -0,0 +1,49 @@
+# Admin UI e2e tests
+
+This test suite tests the Boundary Admin UI in an end-to-end setting. It was designed to be run in a
+variety of environments as long as the appropriate environment variables are set. You can choose to
+spin up your own infrastructure or use [Enos](https://github.com/hashicorp/boundary/tree/main/enos)
+to generate it for you.
+
+The test suite uses the [Playwright](https://playwright.dev/) framework.
+
+## Getting Started
+Setup...
+```bash
+cd boundary-ui/ui/admin
+yarn install
+npx playwright install # this installs the browsers used by Playwright
+```
+
+Ensure the `boundary` CLI is available on the path. You can validate this by making sure this
+command returns something.
+```bash
+which boundary
+```
+
+Run tests...
+```bash
+export BOUNDARY_ADDR=
+export E2E_PASSWORD_ADMIN_LOGIN_NAME=
+export E2E_PASSWORD_ADMIN_PASSWORD=
+export E2E_PASSWORD_AUTH_METHOD_ID=
+... # Tests may have additional variables that it needs
+cd boundary-ui/ui/admin
+yarn run e2e
+```
+
+Here are some additional commands to assist with debugging.
+```bash
+PWDEBUG=console yarn playwright test --headed --config ./tests/e2e/playwright.config.js login.spec.js
+PWDEBUG=console yarn playwright test --headed --config ./tests/e2e/playwright.config.js login.spec.js:13 --debug
+PWDEBUG=console yarn playwright test --headed --config ./tests/e2e/playwright.config.js login.spec.js --debug
+```
+
+## Developing Tests
+
+It is recommended to use locators that resemble how users interact with the application. `getByRole`
+should be the most prioritized locator as this is the closest way to how users and
+accessibility features perceive the page. `getByLabel` should be used for form fields.
+
+Additional information can be found [here](https://playwright.dev/docs/locators#locating-elements)
+and [here](https://testing-library.com/docs/queries/about/#priority).
diff --git a/ui/admin/tests/e2e/global-setup.js b/ui/admin/tests/e2e/global-setup.js
new file mode 100644
index 000000000..6467c3984
--- /dev/null
+++ b/ui/admin/tests/e2e/global-setup.js
@@ -0,0 +1,28 @@
+/* eslint-disable no-undef */
+const { chromium } = require('@playwright/test');
+const { checkEnv, authenticatedState } = require('./helpers/general');
+
+module.exports = async () => {
+  await checkEnv([
+    'BOUNDARY_ADDR',
+    'E2E_PASSWORD_ADMIN_LOGIN_NAME',
+    'E2E_PASSWORD_ADMIN_PASSWORD',
+    'E2E_PASSWORD_AUTH_METHOD_ID',
+  ]);
+
+  // Log in and save the authenticated state to reuse in tests
+  const browser = await chromium.launch();
+  const page = await browser.newPage();
+  await page.goto(process.env.BOUNDARY_ADDR);
+  await page
+    .getByLabel('Login Name')
+    .fill(process.env.E2E_PASSWORD_ADMIN_LOGIN_NAME);
+  await page
+    .getByLabel('Password')
+    .fill(process.env.E2E_PASSWORD_ADMIN_PASSWORD);
+  await page.getByRole('button', { name: 'Sign In' }).click();
+  await page.getByRole('navigation', { name: 'General' }).waitFor();
+  await page.getByText(process.env.E2E_PASSWORD_ADMIN_LOGIN_NAME).waitFor();
+  await page.context().storageState({ path: authenticatedState });
+  await browser.close();
+};
diff --git a/ui/admin/tests/e2e/helpers/boundary-cli.js b/ui/admin/tests/e2e/helpers/boundary-cli.js
new file mode 100644
index 000000000..b7ec96ce7
--- /dev/null
+++ b/ui/admin/tests/e2e/helpers/boundary-cli.js
@@ -0,0 +1,33 @@
+/* eslint-disable no-undef */
+const { execSync } = require('child_process');
+
+/**
+ * Checks that the boundary cli is available
+ */
+exports.checkBoundaryCli = async () => {
+  try {
+    execSync('which boundary');
+  } catch (e) {
+    throw new Error('boundary does not exist on the path');
+  }
+};
+
+/**
+ * Uses the boundary CLI to authenticate to the specified Boundary instance
+ */
+exports.authenticateBoundaryCli = async () => {
+  try {
+    execSync(
+      'boundary authenticate password' +
+        ' -addr=' +
+        process.env.BOUNDARY_ADDR +
+        ' -auth-method-id=' +
+        process.env.E2E_PASSWORD_AUTH_METHOD_ID +
+        ' -login-name=' +
+        process.env.E2E_PASSWORD_ADMIN_LOGIN_NAME +
+        ' -password=env://E2E_PASSWORD_ADMIN_PASSWORD'
+    );
+  } catch (e) {
+    console.log(`${e.stderr}`);
+  }
+};
diff --git a/ui/admin/tests/e2e/helpers/boundary-ui.js b/ui/admin/tests/e2e/helpers/boundary-ui.js
new file mode 100644
index 000000000..efdd7a1fa
--- /dev/null
+++ b/ui/admin/tests/e2e/helpers/boundary-ui.js
@@ -0,0 +1,187 @@
+/* eslint-disable no-undef */
+const { expect } = require('@playwright/test');
+const { nanoid } = require('nanoid');
+
+/**
+ * Uses the UI to create a new organization
+ * @param {Page} page Playwright page object
+ * @returns Name of the organization
+ */
+exports.createNewOrg = async (page) => {
+  const orgName = 'Org ' + nanoid();
+  await page
+    .getByRole('navigation', { name: 'General' })
+    .getByRole('link', { name: 'Orgs' })
+    .click();
+  await page.getByRole('link', { name: 'New Org' }).click();
+  await page.getByLabel('Name').fill(orgName);
+  await page.getByLabel('Description').fill('This is an automated test');
+  await page.getByRole('button', { name: 'Save' }).click();
+  await expect(
+    page.getByRole('alert').getByText('Success', { exact: true })
+  ).toBeVisible();
+  await page.getByRole('button', { name: 'Dismiss' }).click();
+  await expect(
+    page
+      .getByRole('navigation', { name: 'breadcrumbs' })
+      .getByRole('link', { name: orgName })
+  ).toBeVisible();
+
+  return orgName;
+};
+
+/**
+ * Uses the UI to create a new project. Assumes you have selected the desired org.
+ * @param {Page} page Playwright page object
+ * @returns Name of the project
+ */
+exports.createNewProject = async (page) => {
+  const projectName = 'Project ' + nanoid();
+  await page
+    .getByRole('navigation', { name: 'General' })
+    .getByRole('link', { name: 'Projects' })
+    .click();
+  await page.getByRole('link', { name: 'New' }).click();
+  await page.getByLabel('Name').fill(projectName);
+  await page.getByLabel('Description').fill('This is an automated test');
+  await page.getByRole('button', { name: 'Save' }).click();
+  await expect(
+    page.getByRole('alert').getByText('Success', { exact: true })
+  ).toBeVisible();
+  await page.getByRole('button', { name: 'Dismiss' }).click();
+  await expect(
+    page
+      .getByRole('navigation', { name: 'breadcrumbs' })
+      .getByRole('link', { name: projectName })
+  ).toBeVisible();
+
+  return projectName;
+};
+
+/**
+ * Uses the UI to create a new host catalog. Assumes you have selected the desired project.
+ * @param {Page} page Playwright page object
+ * @returns Name of the host catalog
+ */
+exports.createNewHostCatalog = async (page) => {
+  const hostCatalogName = 'Host Catalog ' + nanoid();
+  await page
+    .getByRole('navigation', { name: 'Resources' })
+    .getByRole('link', { name: 'Host Catalogs' })
+    .click();
+  await page.getByRole('link', { name: 'New' }).click();
+  await page.getByLabel('Name').fill(hostCatalogName);
+  await page.getByLabel('Description').fill('This is an automated test');
+  await page.getByRole('button', { name: 'Save' }).click();
+  await expect(
+    page.getByRole('alert').getByText('Success', { exact: true })
+  ).toBeVisible();
+  await page.getByRole('button', { name: 'Dismiss' }).click();
+  await expect(
+    page
+      .getByRole('navigation', { name: 'breadcrumbs' })
+      .getByRole('link', { name: hostCatalogName })
+  ).toBeVisible();
+
+  return hostCatalogName;
+};
+
+/**
+ * Uses the UI to create a new host set. Assumes you have just created a new host catalog.
+ * @param {Page} page Playwright page object
+ * @returns Name of the host set
+ */
+exports.createNewHostSet = async (page) => {
+  const hostSetName = 'Host Set ' + nanoid();
+  await page.getByRole('link', { name: 'Host Sets' }).click();
+  await page.getByRole('link', { name: 'New', exact: true }).click();
+  await page.getByLabel('Name').fill(hostSetName);
+  await page.getByLabel('Description').fill('This is an automated test');
+  await page.getByRole('button', { name: 'Save' }).click();
+  await expect(
+    page.getByRole('alert').getByText('Success', { exact: true })
+  ).toBeVisible();
+  await page.getByRole('button', { name: 'Dismiss' }).click();
+  await expect(
+    page
+      .getByRole('navigation', { name: 'breadcrumbs' })
+      .getByRole('link', { name: hostSetName })
+  ).toBeVisible();
+
+  return hostSetName;
+};
+
+/**
+ * Uses the UI to create a new host in a host set. Assumes you have just created a new host set.
+ * @param {Page} page Playwright page object
+ * @returns Name of the host
+ */
+exports.createNewHostInHostSet = async (page) => {
+  const hostName = 'Host ' + nanoid();
+  await page.getByText('Manage').click();
+  await page.getByRole('link', { name: 'Create and Add Host' }).click();
+  await page.getByLabel('Name').fill(hostName);
+  await page.getByLabel('Description').fill('This is an automated test');
+  await page.getByLabel('Address').fill(process.env.E2E_TARGET_IP);
+  await page.getByRole('button', { name: 'Save' }).click();
+  await expect(
+    page.getByRole('alert').getByText('Success', { exact: true })
+  ).toBeVisible();
+  await page.getByRole('button', { name: 'Dismiss' }).click();
+  await expect(page.getByRole('link', { name: hostName })).toBeVisible();
+
+  return hostName;
+};
+
+/**
+ * Uses the UI to create a new target. Assumes you have selected the desired project.
+ * @param {Page} page Playwright page object
+ * @returns Name of the target
+ */
+exports.createNewTarget = async (page) => {
+  const targetName = 'Target ' + nanoid();
+  await page
+    .getByRole('navigation', { name: 'Resources' })
+    .getByRole('link', { name: 'Targets' })
+    .click();
+  await page.getByRole('link', { name: 'New' }).click();
+  await page.getByLabel('Name').fill(targetName);
+  await page.getByLabel('Description').fill('This is an automated test');
+  await page.getByLabel('Default Port').fill(process.env.E2E_SSH_PORT);
+  await page.getByRole('button', { name: 'Save' }).click();
+  await expect(
+    page.getByRole('alert').getByText('Success', { exact: true })
+  ).toBeVisible();
+  await page.getByRole('button', { name: 'Dismiss' }).click();
+  await expect(
+    page
+      .getByRole('navigation', { name: 'breadcrumbs' })
+      .getByRole('link', { name: targetName })
+  ).toBeVisible();
+
+  return targetName;
+};
+
+/**
+ * Uses the UI to add a host source to a target. Assume you have selected the desired target.
+ * @param {Page} page Playwright page object
+ * @param {string} hostSourceName Name of host source that will be attached to the target
+ */
+exports.addHostSourceToTarget = async (page, hostSourceName) => {
+  await page.getByRole('link', { name: 'Host Sources', exact: true }).click();
+  await page
+    .getByRole('article')
+    .getByRole('link', { name: 'Add Host Sources', exact: true })
+    .click();
+  await page
+    .getByRole('cell', { name: hostSourceName })
+    .locator('..')
+    .getByRole('checkbox')
+    .click({ force: true });
+  await page.getByRole('button', { name: 'Add Host Sources' }).click();
+  await expect(
+    page.getByRole('alert').getByText('Success', { exact: true })
+  ).toBeVisible();
+  await page.getByRole('button', { name: 'Dismiss' }).click();
+  await expect(page.getByRole('link', { name: hostSourceName })).toBeVisible();
+};
diff --git a/ui/admin/tests/e2e/helpers/general.js b/ui/admin/tests/e2e/helpers/general.js
new file mode 100644
index 000000000..65bfaf9fd
--- /dev/null
+++ b/ui/admin/tests/e2e/helpers/general.js
@@ -0,0 +1,19 @@
+/* eslint-disable no-undef */
+exports.authenticatedState = './tests/e2e/artifacts/authenticated-state.json';
+
+/**
+ * Checks that the provided environment variables are set
+ * @param {string[]} envs List of environment variables
+ */
+exports.checkEnv = async (envs) => {
+  let missing = [];
+  for (let env in envs) {
+    if (!process.env[envs[env]]) {
+      missing.push(envs[env]);
+    }
+  }
+
+  if (missing.length > 0) {
+    throw new Error('Missing Environment Variables -- ' + missing);
+  }
+};
diff --git a/ui/admin/tests/e2e/playwright.config.js b/ui/admin/tests/e2e/playwright.config.js
new file mode 100644
index 000000000..90c7aa296
--- /dev/null
+++ b/ui/admin/tests/e2e/playwright.config.js
@@ -0,0 +1,31 @@
+/* eslint-disable no-undef */
+const { devices } = require('@playwright/test');
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  globalSetup: require.resolve('./global-setup'),
+  outputDir: './artifacts/test-failures',
+  timeout: 45000, // Timeout is shared between all tests
+  workers: 1, // Tests need to be run in serial, otherwise there may be conflicts when using the CLI
+  use: {
+    baseURL: process.env.BOUNDARY_ADDR,
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+  ],
+};
+
+module.exports = config;
diff --git a/ui/admin/tests/e2e/tests/target.spec.js b/ui/admin/tests/e2e/tests/target.spec.js
new file mode 100644
index 000000000..a7dcabed3
--- /dev/null
+++ b/ui/admin/tests/e2e/tests/target.spec.js
@@ -0,0 +1,106 @@
+/* eslint-disable no-undef */
+const { test, expect } = require('@playwright/test');
+const { exec, execSync } = require('child_process');
+const { checkEnv, authenticatedState } = require('../helpers/general');
+const {
+  authenticateBoundaryCli,
+  checkBoundaryCli,
+} = require('../helpers/boundary-cli');
+const {
+  createNewOrg,
+  createNewProject,
+  createNewHostCatalog,
+  createNewHostSet,
+  createNewHostInHostSet,
+  createNewTarget,
+  addHostSourceToTarget,
+} = require('../helpers/boundary-ui');
+
+test.use({ storageState: authenticatedState });
+
+test.beforeAll(async () => {
+  await checkEnv([
+    'E2E_TARGET_IP',
+    'E2E_SSH_USER',
+    'E2E_SSH_KEY_PATH',
+    'E2E_SSH_PORT',
+  ]);
+
+  await checkBoundaryCli();
+});
+
+test('Verify session created then cancel the session', async ({ page }) => {
+  await page.goto('/');
+
+  var connect;
+  try {
+    const orgName = await createNewOrg(page);
+    const projectName = await createNewProject(page);
+    await createNewHostCatalog(page);
+    const hostSetName = await createNewHostSet(page);
+    await createNewHostInHostSet(page);
+    const targetName = await createNewTarget(page);
+    await addHostSourceToTarget(page, hostSetName);
+
+    await authenticateBoundaryCli();
+    const orgs = JSON.parse(execSync('boundary scopes list -format json'));
+    const org = orgs.items.filter((obj) => obj.name == orgName)[0];
+    const projects = JSON.parse(
+      execSync('boundary scopes list -format json -scope-id ' + org.id)
+    );
+    const project = projects.items.filter((obj) => obj.name == projectName)[0];
+    const targets = JSON.parse(
+      execSync('boundary targets list -format json -scope-id ' + project.id)
+    );
+    const target = targets.items.filter((obj) => obj.name == targetName)[0];
+
+    connect = exec(
+      'boundary connect' +
+        ' -target-id=' +
+        target.id +
+        ' -exec /usr/bin/ssh --' +
+        ' -l ' +
+        process.env.E2E_SSH_USER +
+        ' -i ' +
+        process.env.E2E_SSH_KEY_PATH +
+        ' -o UserKnownHostsFile=/dev/null' +
+        ' -o StrictHostKeyChecking=no' +
+        ' -o IdentitiesOnly=yes' + // forces the use of the provided key
+        ' -p {{boundary.port}}' +
+        ' {{boundary.ip}}'
+    );
+
+    await page
+      .getByRole('navigation', { name: 'Resources' })
+      .getByRole('link', { name: 'Sessions' })
+      .click();
+    let i = 0;
+    let sessionIsVisible = false;
+    do {
+      i = i + 1;
+      sessionIsVisible = await page
+        .getByRole('cell', { name: targetName })
+        .isVisible();
+      if (sessionIsVisible) {
+        break;
+      }
+      await page.getByRole('button', { name: 'Refresh' }).click();
+      await expect(page.getByRole('button', { name: 'Refresh' })).toBeEnabled();
+    } while (i < 5);
+
+    if (!sessionIsVisible) {
+      throw new Error('Session is not visible');
+    }
+
+    await page
+      .getByRole('cell', { name: targetName })
+      .locator('..')
+      .getByRole('button', { name: 'Cancel' })
+      .click();
+  } finally {
+    // End `boundary connect` process
+    if (connect) {
+      connect.kill('SIGTERM');
+    }
+  }
+});