feat: 🎸 Setup initial desktop E2E tests (#2416)

* feat: 🎸 Setup initial desktop E2E tests

✅ Closes: https://hashicorp.atlassian.net/browse/ICU-14061
This commit is contained in:
Zhihe Li authored and GitHub committed 2024-08-02 13:28:50 -04:00
1 parent 4dc5e13f24
commit a68d7d103a
14 files changed
+374 -402

No files matched your search

+15
View File
@@ -44,6 +44,7 @@ module.exports = {
'./ember-cli-build.js',
'./testem.js',
'./testem-electron.js',
'./playwright.config.js',
'./blueprints/*/index.js',
'./config/**/*.js',
'./lib/*/index.js',
@@ -66,5 +67,19 @@ module.exports = {
'qunit/require-expect': [2, 'except-simple'],
},
},
{
// e2e files
files: ['tests/e2e/**'],
env: {
node: true,
},
extends: ['eslint:recommended', 'prettier'],
rules: {
'no-empty-pattern': [
'error',
{ allowObjectPatternsAsParameters: true },
],
},
},
],
};
+3 -2
View File
@@ -23,8 +23,9 @@
# Vagrant
.vagrant/
# End 2 end testing screenshots
/tests/end2end/screenshots/
# End 2 end test artifacts
/playwright-report
/tests/e2e/artifacts
# ember-try
/.node_modules.ember-try/
+1 -1
View File
@@ -39,7 +39,7 @@
"test": "concurrently \"npm:test:*\" --names \"test:\"",
"test:ember": "cross-env COVERAGE=true ember exam --test-port 0 --split=4 --parallel=1 --random",
"e2e": "yarn run build && yarn run e2e:dev",
"e2e:dev": "npx playwright test --headed",
"e2e:dev": "npx playwright test --reporter=html",
"postinstall": "cd electron-app && yarn install",
"posttest": "ember coverage-merge && yarn clean:coverage"
},
+12
View File
@@ -0,0 +1,12 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
outputDir: './tests/e2e/artifacts',
workers: 1, // Tests need to be run in serial, otherwise there may be conflicts when using the CLI
};
module.exports = config;
@@ -0,0 +1,84 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
const { test } = require('./fixtures/baseTest');
const LoginPage = require('./pages/loginPage');
const { expect } = require('@playwright/test');
test.describe('user/password authentication tests', async () => {
test('Authenticates and signs out', async ({
electronPage,
clusterUrl,
username,
password,
}) => {
const loginPage = new LoginPage(electronPage);
await loginPage.setClusterUrl(clusterUrl);
await loginPage.logInWithPassword(username, password);
await expect(electronPage).toHaveURL(
/.*\/scopes\/global\/projects\/targets$/,
);
await electronPage
.locator('details')
.filter({ hasText: 'Sign Out' })
.click();
await electronPage.getByRole('button', { name: 'Sign Out' }).click();
await expect(electronPage).toHaveURL(/.*\/scopes\/global\/authenticate\//);
});
test('Fails to authenticate with wrong password', async ({
electronPage,
clusterUrl,
username,
}) => {
const loginPage = new LoginPage(electronPage);
await loginPage.setClusterUrl(clusterUrl);
await loginPage.logInWithPassword(username, 'wrong password!!');
await expect(electronPage).toHaveURL(/.*\/scopes\/global\/authenticate\//);
await expect(electronPage.getByText('Authentication Failed')).toBeVisible();
});
});
test.describe('LDAP authentication tests', async () => {
test('Authenticates and signs out', async ({
electronPage,
clusterUrl,
username,
password,
}) => {
const loginPage = new LoginPage(electronPage);
await loginPage.setClusterUrl(clusterUrl);
await loginPage.logInWithLDAP(username, password);
await expect(electronPage).toHaveURL(
/.*\/scopes\/global\/projects\/targets$/,
);
await electronPage
.locator('details')
.filter({ hasText: 'Sign Out' })
.click();
await electronPage.getByRole('button', { name: 'Sign Out' }).click();
await expect(electronPage).toHaveURL(/.*\/scopes\/global\/authenticate\//);
});
test('Fails to authenticate with wrong password', async ({
electronPage,
clusterUrl,
username,
}) => {
const loginPage = new LoginPage(electronPage);
await loginPage.setClusterUrl(clusterUrl);
await loginPage.logInWithLDAP(username, 'wrong password!!');
await expect(electronPage).toHaveURL(/.*\/scopes\/global\/authenticate\//);
await expect(electronPage.getByText('Authentication Failed')).toBeVisible();
});
});
@@ -0,0 +1,26 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
const { electronTest } = require('../fixtures/electronTest');
const LoginPage = require('../pages/loginPage');
exports.authenticateTest = electronTest.extend({
// TODO: Should we put these in a config file?
clusterUrl: ({}, use) => {
use(process.env.BOUNDARY_ADDR ?? 'http://localhost:9200');
},
username: ({}, use) => {
use(process.env.E2E_PASSWORD_ADMIN_LOGIN_NAME ?? 'admin');
},
password: ({}, use) => {
use(process.env.E2E_PASSWORD_ADMIN_PASSWORD ?? 'password');
},
authedPage: async ({ electronPage, clusterUrl, username, password }, use) => {
const loginPage = new LoginPage(electronPage);
await loginPage.setClusterUrl(clusterUrl);
await loginPage.logInWithPassword(username, password);
use(electronPage);
},
});
+12
View File
@@ -0,0 +1,12 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
const { mergeTests } = require('@playwright/test');
const { electronTest } = require('../fixtures/electronTest');
const { authenticateTest } = require('./authenticateTest');
// Redundant since authenticateTest already extends electronTest but this is for demonstration purposes
// The last item in the list will override any previous ones in a fixture name collision
exports.test = mergeTests(authenticateTest, electronTest);
@@ -0,0 +1,134 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
/* eslint-disable ember/avoid-leaking-state-in-ember-objects */
const { test } = require('@playwright/test');
const path = require('path');
const fs = require('fs');
/**
* Get the path to the Boundary Desktop executable
* @param buildDirectory
* @return {string}
*/
const getExecutablePath = (buildDirectory = 'out') => {
const rootDir = path.resolve('./electron-app');
// directory where the builds are stored
const outDir = path.resolve(rootDir, buildDirectory);
// list of files in the out directory
const builds = fs.readdirSync(outDir);
// list of platforms we support
const platforms = ['win32', 'darwin', 'linux'];
const latestBuild = builds
.map((fileName) => {
// make sure it's a directory with "-" delimited platform in its name
const stats = fs.statSync(path.join(outDir, fileName));
const platform = fileName
.toLocaleLowerCase()
.split('-')
.find((part) => platforms.includes(part));
if (stats.isDirectory() && platform) {
return {
name: fileName,
time: fs.statSync(path.join(outDir, fileName)).mtimeMs,
platform,
};
}
})
// Usually it's just one build but if for some reason you made
// multiple builds we'll just pick the most recent one
.sort((a, b) => {
const aTime = a?.time ?? 0;
const bTime = b?.time ?? 0;
return bTime - aTime;
})
.filter((file) => file)[0];
if (!latestBuild.name) {
throw new Error('No build found in out directory');
}
switch (latestBuild.platform) {
case 'darwin':
return path.join(
outDir,
latestBuild.name,
'Boundary.app',
'Contents',
'MacOS',
'Boundary',
);
case 'win32':
return path.join(outDir, latestBuild.name, 'Boundary.exe');
case 'linux':
return path.join(outDir, latestBuild.name, 'boundary-desktop');
default:
throw new Error('Unsupported platform');
}
};
exports.electronTest = test.extend({
// The screenshot/trace config option from playwright doesn't quite work since we use electron
// so we'll manually take a screenshot/trace and attach it to the test report if the test fails.
saveTestFailureInfo: [
async ({ electronPage }, use, testInfo) => {
await use();
const path = testInfo.outputPath('trace.zip');
await electronPage.context().tracing.stop({ path });
// After the test we can check whether the test passed or failed.
if (testInfo.status !== testInfo.expectedStatus) {
const screenshot = await electronPage.screenshot();
await testInfo.attach('screenshot', {
body: screenshot,
contentType: 'image/png',
});
await testInfo.attach('trace', { path });
}
// Clean up the trace files
fs.rmSync(path);
try {
fs.rmdirSync(testInfo.outputPath());
} catch {
// Directory isn't empty, ignore error
}
},
{ auto: true },
],
launchElectronApp: async ({ playwright }, use) => {
const apps = [];
await use(async (options) => {
const app = await playwright._electron.launch({
executablePath: getExecutablePath(),
env: {
NODE_ENV: 'test',
BYPASS_APP_UPDATER: true,
},
...options,
});
apps.push(app);
return app;
});
for (const app of apps) {
await app.close();
}
},
electronApp: async ({ launchElectronApp }, use) => {
await use(await launchElectronApp());
},
electronPage: async ({ electronApp }, use) => {
const page = await electronApp.firstWindow();
await page.evaluate(() => localStorage.clear());
await page.context().tracing.start({ screenshots: true, snapshots: true });
await use(page);
},
});
+34
View File
@@ -0,0 +1,34 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
class LoginPage {
constructor(page) {
this.page = page;
}
async setClusterUrl(clusterUrl) {
await this.page.getByLabel('Cluster URL').fill(clusterUrl);
await this.page.getByRole('button', { name: 'Submit' }).click();
}
async logInWithPassword(username, password) {
await this.page.getByLabel('Login Name').fill(username);
await this.page.getByLabel('Password', { exact: true }).fill(password);
await this.page.getByRole('button', { name: 'Sign In' }).click();
}
async logInWithLDAP(username, password) {
await this.page
.getByRole('link', {
name: 'Generated global scope initial ldap auth method',
})
.click();
await this.logInWithPassword(username, password);
}
// Can't login with OIDC as we can't control the external browser that is opened
}
module.exports = LoginPage;
+16
View File
@@ -0,0 +1,16 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
const { test } = require('./fixtures/baseTest');
const { expect } = require('@playwright/test');
test.describe.fixme('Settings Page tests', async () => {
test('Navigates to settings page', async ({ authedPage }) => {
await authedPage.getByRole('link', { name: 'Settings' }).click();
await expect(authedPage).toHaveURL(
/.*\/scopes\/global\/projects\/settings$/,
);
});
});
+37
View File
@@ -0,0 +1,37 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
const { test } = require('./fixtures/baseTest');
const { expect } = require('@playwright/test');
test.describe('Targets test', async () => {
test('Connects to a target and ends session', async ({ authedPage }) => {
await authedPage
.getByRole('link', { name: 'Generated target with a direct address' })
.click();
await authedPage.getByRole('button', { name: 'Connect' }).click();
await expect(authedPage).toHaveURL(
/.*\/scopes\/global\/projects\/sessions/,
);
await authedPage.getByRole('button', { name: 'End Session' }).click();
await expect(authedPage.getByText('Canceled successfully.')).toBeVisible();
await expect(authedPage).toHaveURL(
/.*\/scopes\/global\/projects\/targets$/,
);
});
test('Searches targets correctly', async ({ authedPage }) => {
await authedPage.getByLabel('Search').fill('hashicorp');
// One row is the header
await expect(authedPage.getByRole('row')).toHaveCount(2);
await expect(
authedPage.getByRole('link', { name: 'www.hashicorp.com' }),
).toBeVisible();
});
});
@@ -1,189 +0,0 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
/* eslint-disable no-undef */
const { test, expect } = require('@playwright/test');
const { _electron: electron } = require('@playwright/test');
const helpers = require('./test-helpers');
let electronApp = null;
// Subdirectory within tests/end2end/screenshots where the screenshots of this test suite will be stored.
const screenshotsDirectory = 'authentication/';
// Path where the Boundary Desktop client binary is generated
const executablePath = helpers.returnExecutablePath(
process.platform,
process.arch,
);
// Set login variables
const clusterUrlValue = 'http://localhost:9200';
const loginUsername = 'admin';
const loginPassword = 'password';
test.beforeEach(async () => {
electronApp = await electron.launch({
executablePath: executablePath,
env: {
NODE_ENV: 'test',
BYPASS_APP_UPDATER: true,
},
});
});
test.afterEach(async () => {
electronApp.close();
electronApp = null; // Not sure we should do this.
});
test.describe('Authentication end to end test suite', async () => {
test.describe('user/password authentication test suite', async () => {
test('Authenticates using user and password method and deauthenticates ', async () => {
const boundaryWindow = await electronApp.firstWindow(); // The window that contains the app.
// Override local storage cluster URL
await boundaryWindow.evaluate(() =>
window.localStorage.setItem('desktop:clusterUrl', null),
);
// Perform login
await helpers.login(
boundaryWindow,
clusterUrlValue,
loginUsername,
loginPassword,
);
// Take screenshot
await boundaryWindow.screenshot({
path: helpers.generateScreenshotPath(
screenshotsDirectory,
'fillUserPassword',
),
fullPage: true,
});
// Click submit
await boundaryWindow.waitForSelector('button[type="submit"]');
await helpers.click(boundaryWindow, 'button[type="submit"]');
// Check we are in Targets
await boundaryWindow.waitForURL('**/#/scopes/global/projects/targets');
const windowUrl = new URL(await boundaryWindow.url());
expect(windowUrl.hash).toEqual('#/scopes/global/projects/targets');
// Take screenshot
await boundaryWindow.screenshot({
path: helpers.generateScreenshotPath(
screenshotsDirectory,
'afterLogin',
),
fullPage: true,
});
// Opens user dropdown
await boundaryWindow.waitForSelector('.rose-header-utilities');
await helpers.click(boundaryWindow, '.rose-header-utilities summary');
// User dropdown is visible
await boundaryWindow.waitForSelector('details[open]');
expect(await boundaryWindow.isVisible('details'));
// Take screenshot
await boundaryWindow.screenshot({
path: helpers.generateScreenshotPath(
screenshotsDirectory,
'userDropdown',
),
fullPage: true,
});
// Clicks Deauthenticate
await boundaryWindow.waitForSelector('text="Deauthenticate"');
await helpers.click(boundaryWindow, 'text="Deauthenticate"');
// Makes sure we are log out
await boundaryWindow.waitForSelector('main >> div.branded-card');
expect(await boundaryWindow.isVisible('h2 >> text=Authenticate'));
// TODO: create an assertion checking localstorage ember_simple_auth-session key
// TODO: try to check we are log out by url
// Take screenshot
await boundaryWindow.screenshot({
path: helpers.generateScreenshotPath(
screenshotsDirectory,
'afterLogout',
),
fullPage: true,
});
});
test('Authenticates using user and wrong password, it can not authenticate', async () => {
const boundaryWindow = await electronApp.firstWindow(); // The window that contains the app.
// Override local storage cluster URL
await boundaryWindow.evaluate(() =>
window.localStorage.setItem('desktop:clusterUrl', null),
);
// Perform login
await helpers.login(
boundaryWindow,
clusterUrlValue,
loginUsername,
'123456789',
);
// Wait for the notification
await boundaryWindow.waitForSelector('.rose-notification-body');
// Expect the notification to alert authentication has failed.
expect(
await boundaryWindow.innerText('div.rose-notification-body'),
).toEqual('Authentication Failed');
// Take screenshot
await boundaryWindow.screenshot({
path: helpers.generateScreenshotPath(
screenshotsDirectory,
'notificationFailed',
),
fullPage: true,
});
});
});
test.describe('OIDC authentication test suite', async () => {
test.skip('OIDC', async () => {
const boundaryWindow = await electronApp.firstWindow(); // The window that contains the app.
// For some reason it catches a new window opens, but it closes the window right away.
await electronApp.on('window', async (page) => {
console.log('A new window is opening');
// Catch if it closes
await page.on('close', () => {
console.log('is closing');
});
});
// Override local storage cluster URL
await boundaryWindow.evaluate(() =>
window.localStorage.setItem('desktop:clusterUrl', null),
);
const clusterUrlValue = 'http://localhost:9200';
// Fill the cluster URL input
await boundaryWindow.waitForSelector('[name=host]');
await boundaryWindow.fill('[name=host]', clusterUrlValue);
// Click the submit button
// Due to an error with await boundaryWindow.click('button[type="submit"]'); we are using a workaround.
// More info about it here: https://github.com/microsoft/playwright/issues/1808
await boundaryWindow.waitForSelector('button[type="submit"]');
await helpers.click(boundaryWindow, 'button[type="submit"]');
// Click tab OIDC
await boundaryWindow.waitForSelector('main >> div.branded-card >> nav');
await helpers.click(boundaryWindow, 'a:text("OIDC")');
await boundaryWindow.pause();
});
});
});
-126
View File
@@ -1,126 +0,0 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
/* eslint-disable no-undef */
const { test, expect } = require('@playwright/test');
const { _electron: electron } = require('@playwright/test');
const helpers = require('./test-helpers');
let electronApp = null;
// Subdirectory within tests/end2end/screenshots where the screenshots of this test suite will be stored.
const screenshotsDirectory = 'targets/';
// Path where the Boundary Desktop client binary is generated
const executablePath = helpers.returnExecutablePath(
process.platform,
process.arch,
);
// Set login variables
const clusterUrlValue = 'http://localhost:9200';
const loginUsername = 'admin';
const loginPassword = 'password';
test.beforeEach(async () => {
electronApp = await electron.launch({
executablePath: executablePath,
env: {
NODE_ENV: 'test',
BYPASS_APP_UPDATER: true,
},
});
});
test.afterEach(async () => {
electronApp.close();
electronApp = null; // Not sure we should do this.
});
test.describe('Targets end to end test suite', async () => {
test('Connects to a target', async () => {
const boundaryWindow = await electronApp.firstWindow(); // The window that contains the app.
// Override local storage cluster URL
await boundaryWindow.evaluate(() =>
window.localStorage.setItem('desktop:clusterURL', null),
);
// Perform the login
await helpers.login(
boundaryWindow,
clusterUrlValue,
loginUsername,
loginPassword,
);
// Check we are in Targets
await boundaryWindow.waitForURL('**/#/scopes/global/projects/targets');
const windowUrl = new URL(await boundaryWindow.url());
expect(windowUrl.hash).toEqual('#/scopes/global/projects/targets');
// Click connect to a target
await boundaryWindow.waitForSelector('table.rose-table');
await helpers.click(
boundaryWindow,
'table.rose-table >> tbody >> tr >> nth=0 >> button >> text=Connect',
);
// The target popup opens
await boundaryWindow.waitForSelector('section.dialog-detail');
// Take screenshot
await boundaryWindow.screenshot({
path: helpers.generateScreenshotPath(
screenshotsDirectory,
'targetConnectionDetails',
),
fullPage: true,
});
// Click copyable in popup
// TODO: read clipboard value. Running into issues reading clipboard, so will take a shortcut for now
await helpers.click(
boundaryWindow,
'section.dialog-detail >> div.rose-dialog-body >> button',
);
// Temporary: persist proxy from target
const persistedProxy = await boundaryWindow.innerText(
'section.dialog-detail >> div.rose-dialog-body >> span.copyable-content',
);
// Click close popup
await helpers.click(
boundaryWindow,
'section.dialog-detail >> footer >> button',
);
// On left nav menu, click Sessions
await boundaryWindow.waitForSelector('section.rose-layout-global >> aside');
await helpers.click(
boundaryWindow,
'section.rose-layout-global >> aside >> nav >> a >> text=Sessions',
);
// Take screenshot
await boundaryWindow.screenshot({
path: helpers.generateScreenshotPath(screenshotsDirectory, 'sessions'),
fullPage: true,
});
// Check the right session has status pending.
await boundaryWindow.waitForSelector(
'main >> div.rose-layout-page-body >> table.rose-table',
);
// Select Proxy
const currentProxy = await boundaryWindow.innerText(
'div.rose-layout-page-body >> table >> tbody >> tr >> nth=0 >> td >> nth=1 >> .copyable-content',
);
const currentStatus = await boundaryWindow.innerText(
'div.rose-layout-page-body >> table >> tbody >> tr >> nth=0 >> td >> nth=3 >> .hds-badge',
);
expect(await persistedProxy).toEqual(await currentProxy);
expect(await currentStatus).toEqual('pending');
// TODO: before closing the app we should terminate sessions.
// TODO: add test to check if sessions are not terminated we promp the confirmation dialog
});
});
-84
View File
@@ -1,84 +0,0 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
/* eslint-disable no-undef */
const path = require('path');
/**
*
* @param {string} screenshotTestDirectory screenshot directory for the specific test suite
* @param {string} fileName The file name.
* @returns A full path where to save the screenshot.
*/
exports.generateScreenshotPath = (screenshotTestDirectory, fileName) => {
const screenshotFormat = '.png';
const screenshotsRootPath = path.join(__dirname, 'screenshots');
const screenshotPath = path.join(
screenshotsRootPath,
screenshotTestDirectory,
);
return path.join(screenshotPath, fileName).concat(screenshotFormat);
};
/**
*
* @param {string} platform The operating system.
* @param {string} arch The CPU architecture.
* @returns A full path where the Desktop Client executable is located.
*/
exports.returnExecutablePath = (platform, arch) => {
try {
switch (platform) {
case 'darwin':
switch (arch) {
case 'x64': // Intel chips
return 'electron-app/out/Boundary-darwin-x64/Boundary.app/Contents/MacOS/Boundary';
case 'arm64': // M1 chips
return 'electron-app/out/Boundary-darwin-arm64/Boundary.app/Contents/MacOS/Boundary';
default:
throw new Error('The test suite is not compatible with your arch.');
}
default:
throw new Error('The test suite is not compatible with your platform.');
}
} catch (error) {
console.error(error);
}
};
/**
* Performs a click with a workarouind due to an error with plain playwright click.
* More info about the issue here: https://github.com/microsoft/playwright/issues/1808
* @param {object} appWindow The electron window to perform the action.
* @param {string} selector The DOM element selector.
*/
exports.click = async (appWindow, selector) => {
await appWindow.$eval(selector, (element) => element.click());
};
/**
* Performs login on the app window provided with the credentials provided.
* @param {object} appWindow The electron window to perform the action.
* @param {string} clusterUrl Cluster URL to set.
* @param {string} username Username to login.
* @param {string} password Password to login.
*/
exports.login = async (appWindow, clusterUrl, username, password) => {
// Fill the clusterUrl input
await appWindow.waitForSelector('[name=host]');
await appWindow.fill('[name=host]', clusterUrl);
// Click the set clusterUrl submit button
await appWindow.waitForSelector('button[type="submit"]');
await this.click(appWindow, 'button[type="submit"]');
// Fill user & password
await appWindow.fill('[name="identification"]', username);
await appWindow.fill('[name="password"]', password);
// Click submit
await appWindow.waitForSelector('button[type="submit"]');
await this.click(appWindow, 'button[type="submit"]');
};