#!/usr/bin/env node /** * Real-browser (Firefox AND Chromium, headless) end-to-end test of the Sitesmith modal. * * Builds the editor to a scratch dir, serves it from a mock server that implements the Sitesmith * backend contract (scripts/e2e-sitesmith-mock-server.mjs), and drives the REAL modal: job * start/poll/cancel/resume, mockup + reference upload, "Sites I like", asset awareness, contrast * repair, vision-unavailable mode and the one-round self-review (real html-to-image capture). * No real host or AI provider is contacted: every non-127.0.0.1 request is answered locally. * * npm run test:browser:sitesmith * * Env: * CHROMIUM_PATH / FIREFOX_PATH override the Playwright-pinned binaries * SITESMITH_E2E_DIST= where to build (default: a fresh temp dir) * SITESMITH_E2E_SKIP_BUILD=1 reuse an existing build in SITESMITH_E2E_DIST * SITESMITH_SHOTS= Firefox screenshots go here (default: /sitesmith-shots) * SITESMITH_ONLY=chromium|firefox run one browser * SITESMITH_SCENARIO=3,5b run only these scenario ids */ import { chromium, firefox } from '@playwright/test'; import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { createMockServer, makePng } from './e2e-sitesmith-mock-server.mjs'; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const distDir = path.resolve(process.env.SITESMITH_E2E_DIST || fs.mkdtempSync(path.join(os.tmpdir(), 'sitesmith-e2e-'))); const shotsDir = path.resolve(process.env.SITESMITH_SHOTS || path.join(os.tmpdir(), 'sitesmith-shots')); fs.mkdirSync(shotsDir, { recursive: true }); if (!(process.env.SITESMITH_E2E_SKIP_BUILD && fs.existsSync(path.join(distDir, 'index.html')))) { console.log(`building editor -> ${distDir}`); const b = spawnSync('npx', ['vite', 'build', '--outDir', distDir, '--emptyOutDir'], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf8' }); if (b.status !== 0) { console.error(b.stdout, b.stderr); process.exit(2); } } const srv = await createMockServer({ distDir }); // ------------------------------------------------------------------ result builders const el = (name, props, nodes = []) => ({ type: { resolvedName: name }, props, nodes }); const pageResult = (headingText, { id = 'new-h', color = '#ffffff', bg = '#0f172a', size = '36px', weight = '700' } = {}) => ({ type: 'replace', scope: 'page', message: 'Built it.', pages: [{ name: 'Home', tree: el('Container', { node_id: 'new-root', style: { minHeight: '300px', backgroundColor: '#ffffff' } }, [ el('Container', { node_id: id + '-sec', tag: 'section', style: { backgroundColor: bg, padding: '40px' } }, [ el('Heading', { node_id: id, text: headingText, level: 'h1', style: { color, fontSize: size, fontWeight: weight } }), ]), ]) }], }); // ------------------------------------------------------------------ harness let failures = 0; const log = (b, s, ok, label, detail) => { if (!ok) failures++; console.log(`[${b}] ${ok ? 'PASS' : 'FAIL'} ${s}: ${label}${!ok && detail !== undefined ? ` -- ${detail}` : ''}`); }; const INIT = `(() => { window.__phases = []; window.__bad = []; const BAD = /NetworkError|Failed to fetch/; const scan = () => { const st = document.querySelector('[data-testid=ss-job-status]'); if (st) { const ph = st.getAttribute('data-phase') + '|' + (st.querySelector('[data-testid=ss-phase-label]')?.textContent || ''); if (window.__phases[window.__phases.length - 1] !== ph) window.__phases.push(ph); } for (const d of document.querySelectorAll('[role=dialog]')) { const t = d.textContent || ''; const m = BAD.exec(t); if (m && window.__bad.length < 5) window.__bad.push(t.slice(Math.max(0, m.index - 40), m.index + 60)); } }; new MutationObserver(scan).observe(document, { subtree: true, childList: true, characterData: true, attributes: true }); try { if (!window.__keepSelfReview) localStorage.setItem('sitesmith.selfReview', 'off'); } catch (e) {} })();`; async function newPage(browser, name, { selfReview = false } = {}) { const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); const external = []; await ctx.route(/^https?:\/\/(?!127\.0\.0\.1)/, (route) => { const u = route.request().url(); external.push(u); if (/\.css(\?|$)|fonts\.googleapis/.test(u)) return route.fulfill({ status: 200, contentType: 'text/css', headers: { 'access-control-allow-origin': '*' }, body: '/* offline */' }); return route.abort(); }); await ctx.addInitScript(selfReview ? `window.__keepSelfReview = true;${INIT}` : INIT); const page = await ctx.newPage(); const errors = []; const seenUrls = []; page.on('pageerror', (e) => errors.push(String(e))); page.on('request', (r) => seenUrls.push(r.url())); page.__errors = errors; page.__urls = seenUrls; page.__external = external; page.__ctx = ctx; await page.goto(srv.url + '/'); await page.waitForSelector('.canvas-device-frame [data-craft-container]', { timeout: 20000 }); await page.waitForTimeout(500); return page; } async function openModal(page, { waitInput = true } = {}) { await page.click('button.sitesmith-btn'); await page.waitForSelector('[role=dialog]', { timeout: 10000 }); if (!waitInput) return; await page.waitForSelector('textarea.sitesmith-textarea:not([disabled])', { timeout: 10000 }); } const sendMsg = async (page, text) => { await page.fill('textarea.sitesmith-textarea', text); await page.click('[role=dialog] button:has-text("→")'); }; const canvasText = (page) => page.evaluate(() => document.querySelector('.canvas-device-frame')?.innerText ?? ''); const waitCanvas = (page, text, timeout = 20000) => page.waitForFunction((t) => (document.querySelector('.canvas-device-frame')?.innerText ?? '').includes(t), text, { timeout }); const expandSection = async (page, id) => { const btn = page.locator(`[data-testid=ss-section-${id}] > button`); if ((await btn.getAttribute('aria-expanded')) !== 'true') await btn.click(); }; const modalShot = async (page, name, file) => { if (name === 'firefox') await page.screenshot({ path: path.join(shotsDir, file) }); }; const contrastIn = (page, text) => page.evaluate((t) => { const lum = ([r, g, b]) => { const f = (c) => { c /= 255; return c <= 0.03928 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4; }; return 0.2126 * f(r) + 0.7152 * f(g) + 0.0722 * f(b); }; const rgba = (s) => { const m = s.match(/[\d.]+/g).map(Number); return { c: m.slice(0, 3), a: m.length > 3 ? m[3] : 1 }; }; const h = [...document.querySelectorAll('.canvas-device-frame h1,.canvas-device-frame h2')].find((e) => e.textContent.includes(t)); if (!h) return null; const fg = rgba(getComputedStyle(h).color); let bg = null; for (let n = h; n; n = n.parentElement) { const b = rgba(getComputedStyle(n).backgroundColor); if (b.a > 0.99) { bg = b.c; break; } } bg = bg || [255, 255, 255]; const [a, b] = [lum(fg.c), lum(bg)]; return { color: getComputedStyle(h).color, bg: `rgb(${bg.join(', ')})`, ratio: (Math.max(a, b) + 0.05) / (Math.min(a, b) + 0.05) }; }, text); // ------------------------------------------------------------------ scenarios const scenarios = []; const scenario = (id, title, fn, opts) => scenarios.push({ id, title, fn, opts }); scenario('1', 'start -> poll progress -> result applied', async ({ page, name, ok }) => { srv.setNextResult(pageResult('Brand new heading')); await openModal(page); await modalShot(page, name, '01-idle.png'); await sendMsg(page, 'Build me a home page'); await page.waitForSelector('[data-testid=ss-job-status][data-phase=generating]', { timeout: 15000 }); await modalShot(page, name, '02-progress.png'); await waitCanvas(page, 'Brand new heading'); ok('result applied to the canvas', true); const phases = await page.evaluate(() => window.__phases); const names = phases.map((p) => p.split('|')[0]); ok('successive phases seen', ['queued', 'preparing', 'captioning', 'generating', 'finalizing'].filter((p) => names.includes(p)).length >= 4, JSON.stringify(phases)); ok('phase labels are shown as text', phases.some((p) => /Designing your page/.test(p)) && phases.some((p) => /Preparing/.test(p)), JSON.stringify(phases)); const body = srv.starts[0] ?? {}; ok('start body has site_id', body.site_id === 1, JSON.stringify(body.site_id)); ok('start body has design.primaryColor', typeof body.design?.primaryColor === 'string', JSON.stringify(body.design)); ok('exactly one start (self-review off)', srv.starts.length === 1, String(srv.starts.length)); }); scenario('2', 'cancel a slow job', async ({ page, name, ok }) => { srv.setSlow(true); srv.setNextResult(pageResult('Should never appear')); await openModal(page); const before = await canvasText(page); await sendMsg(page, 'Something slow'); await page.waitForSelector('[data-testid=ss-cancel]', { timeout: 10000 }); await page.waitForSelector('[data-testid=ss-job-status][data-phase=generating]', { timeout: 10000 }); ok('input replaced by status (no textarea while running)', (await page.locator('textarea.sitesmith-textarea').count()) === 0); await modalShot(page, name, '09-running-disabled.png'); await page.click('[data-testid=ss-cancel]'); await page.waitForSelector('[data-testid=ss-note]', { timeout: 15000 }); ok('mock recorded the cancel', srv.cancels.length === 1 && typeof srv.cancels[0].job_id === 'string', JSON.stringify(srv.cancels)); const note = await page.textContent('[data-testid=ss-note]'); ok('UI shows a Cancelled note', /Cancelled/.test(note ?? ''), note); ok('canvas unchanged', (await canvasText(page)) === before && !(await canvasText(page)).includes('Should never appear')); }); scenario('3', 'reload with an active job resumes it', async ({ page, ok }) => { srv.setActiveJob({ phase: 'generating', elapsed_s: 4, result: { ok: true, response: pageResult('Resumed heading') } }); await page.reload(); await page.waitForSelector('.canvas-device-frame [data-craft-container]', { timeout: 20000 }); await openModal(page, { waitInput: false }); // a resumed job replaces the input box await page.waitForSelector('[data-testid=ss-job-status]', { timeout: 15000 }); const txt = await page.textContent('[data-testid=ss-job-status]'); ok('status shows the picked-up text', /Picked up a request that was already running/.test(txt ?? ''), txt); await waitCanvas(page, 'Resumed heading'); ok('result applied after completion', true); ok('no new start was made', srv.starts.length === 0, String(srv.starts.length)); }); scenario('4', 'mockup upload + client-side rejections', async ({ page, name, ok }) => { srv.setNextResult(pageResult('Mockup based heading')); await openModal(page); await expandSection(page, 'mockups'); const input = page.locator('input[data-testid=ss-mockup-input]'); await input.setInputFiles({ name: 'mock.png', mimeType: 'image/png', buffer: makePng(64, 48) }); await page.waitForSelector('[data-testid=ss-mockup-item] img', { timeout: 10000 }); ok('one upload reached the server', srv.uploads.length === 1 && !!srv.uploads[0].id, JSON.stringify(srv.uploads)); await page.waitForFunction(() => { const i = document.querySelector('[data-testid=ss-mockup-item] img'); return i && i.complete && i.naturalWidth > 0; }, null, { timeout: 10000 }); ok('thumbnail rendered from the server', true); await modalShot(page, name, '03-mockup-attached.png'); const alertText = () => page.locator('[data-testid=ss-mockups] [role=alert]').first().textContent(); await input.setInputFiles({ name: 'x.png', mimeType: 'image/png', buffer: Buffer.from('this is plain text, not a picture at all') }); await page.waitForSelector('[data-testid=ss-mockups] [role=alert]', { timeout: 5000 }); const a1 = await alertText(); ok('non-image rejected client-side with a message', !!a1 && a1.length > 5, a1); await page.waitForTimeout(400); ok('non-image never reached the server', srv.uploads.length === 1, String(srv.uploads.length)); await input.setInputFiles({ name: 'big.png', mimeType: 'image/png', buffer: Buffer.concat([makePng(8, 8), Buffer.alloc(9 * 1024 * 1024)]) }); await page.waitForFunction((prev) => { const a = document.querySelector('[data-testid=ss-mockups] [role=alert]'); return a && a.textContent !== prev; }, a1, { timeout: 8000 }).catch(() => {}); const a2 = await alertText(); ok('oversize file rejected client-side with a message', !!a2 && /limit|8\.0 MB|too (large|big)/i.test(a2), a2); await page.waitForTimeout(400); ok('oversize never reached the server', srv.uploads.length === 1, String(srv.uploads.length)); await sendMsg(page, 'Build toward my mockup'); await waitCanvas(page, 'Mockup based heading'); ok('start body mockup_ids has the uploaded id', Array.isArray(srv.starts[0]?.mockup_ids) && srv.starts[0].mockup_ids.includes(srv.uploads[0].id), JSON.stringify(srv.starts[0]?.mockup_ids)); }); scenario('5', 'Sites I like CRUD, selection, never fetched', async ({ page, name, ok }) => { const label = `${srv.url}/should-never-be-fetched`; srv.setNextResult(pageResult('Reference based heading')); await openModal(page); await expandSection(page, 'sites'); await page.click('[data-testid=ss-sites-i-like] button:has-text("Add a site I like")'); await page.fill('#ss-ref-url', label); await page.fill('#ss-ref-likes', 'Big photo header'); await page.click('[data-testid=ss-ref-form] button:has-text("Add site")'); await page.waitForSelector('[data-testid=ss-ref-item]', { timeout: 8000 }); ok('added entry listed', (await page.locator('[data-testid=ss-ref-item]').count()) === 1); // edit await page.click('[data-testid=ss-ref-item] button:has-text("Edit")'); await page.fill('#ss-ref-likes', 'Simple menu and white space'); await page.click('[data-testid=ss-ref-form] button:has-text("Save changes")'); await page.waitForFunction(() => document.querySelector('[data-testid=ss-ref-item]')?.textContent.includes('Simple menu'), null, { timeout: 8000 }); ok('edit saved (refs_update sent, text changed)', srv.refCalls.some((c) => c.action === 'refs_update')); // a second entry, then delete it with confirmation await page.click('[data-testid=ss-sites-i-like] button:has-text("Add a site I like")'); await page.fill('#ss-ref-url', 'second-example.test'); await page.fill('#ss-ref-likes', 'Nice footer'); await page.click('[data-testid=ss-ref-form] button:has-text("Add site")'); await page.waitForFunction(() => document.querySelectorAll('[data-testid=ss-ref-item]').length === 2, null, { timeout: 8000 }); const second = page.locator('[data-testid=ss-ref-item]', { hasText: 'second-example.test' }); await second.locator('button:has-text("Delete")').click(); ok('delete asks for confirmation first', (await page.locator('button:has-text("Confirm delete")').count()) === 1 && !srv.refCalls.some((c) => c.action === 'refs_delete')); await page.click('button:has-text("Confirm delete")'); await page.waitForFunction(() => document.querySelectorAll('[data-testid=ss-ref-item]').length === 1, null, { timeout: 8000 }); ok('delete removed the entry (refs_delete sent)', srv.refCalls.some((c) => c.action === 'refs_delete')); // select + copy + send await page.locator('[data-testid=ss-ref-item] input[type=checkbox]').check(); const copy = page.locator('[data-testid=ss-sites-copy]'); const copyTxt = (await copy.textContent()) ?? ''; ok('"does not open, visit or fetch it" copy visible', (await copy.isVisible()) && /does not open, visit or fetch it/.test(copyTxt), copyTxt); await modalShot(page, name, '04-sites-i-like.png'); await sendMsg(page, 'Use the sites I like'); await waitCanvas(page, 'Reference based heading'); const refId = srv.refCalls.find((c) => c.action === 'refs_add'); ok('start body reference_ids has the selected id', Array.isArray(srv.starts[0]?.reference_ids) && srv.starts[0].reference_ids.length === 1, JSON.stringify(srv.starts[0]?.reference_ids)); ok('the label was stored as given', !!refId && refId.body.url_label.includes('should-never-be-fetched'), JSON.stringify(refId?.body)); ok('mock never received a request for the label', !srv.requests.some((r) => r.path.includes('should-never-be-fetched'))); ok('the browser never requested the label', !page.__urls.some((u) => u.includes('should-never-be-fetched'))); }); scenario('5b', 'asset awareness', async ({ page, name, ok }) => { srv.setNextResult(pageResult('Asset aware heading')); await openModal(page); await expandSection(page, 'assets'); await page.waitForSelector('[data-testid=ss-asset-item]', { timeout: 8000 }); ok('three assets listed', (await page.locator('[data-testid=ss-asset-item]').count()) === 3); const txt = (await page.textContent('[data-testid=ss-assets]')) ?? ''; ok('captions shown', txt.includes('A wide blue banner photo') && txt.includes('No description yet'), txt.slice(0, 200)); await page.waitForFunction(() => [...document.querySelectorAll('[data-testid=ss-asset-item] img')].every((i) => i.complete && i.naturalWidth > 0), null, { timeout: 8000 }); ok('thumbnails loaded from same-origin URLs', true); await page.locator('[data-testid=ss-asset-item]').nth(2).locator('input[type=checkbox]').uncheck(); await modalShot(page, name, '05-asset-awareness.png'); await sendMsg(page, 'Use my pictures'); await waitCanvas(page, 'Asset aware heading'); const b = srv.starts[0] ?? {}; ok('use_assets stays true and the excluded asset is left out', b.use_assets === true && Array.isArray(b.asset_filenames) && b.asset_filenames.length === 2 && !b.asset_filenames.includes('logo.png'), JSON.stringify([b.use_assets, b.asset_filenames])); }); scenario('6', 'contrast repair', async ({ page, name, ok }) => { srv.setNextResult(pageResult('Contrast heading', { id: 'c-h', color: '#334155', bg: '#1e3a8a', size: '16px', weight: '400' })); await openModal(page); await sendMsg(page, 'Make a blue banner'); await waitCanvas(page, 'Contrast heading'); const c = await contrastIn(page, 'Contrast heading'); ok('heading found in canvas', !!c); ok('heading colour was changed from #334155', !!c && c.color !== 'rgb(51, 65, 85)', c?.color); ok('heading now passes 4.5:1 on its background', !!c && c.ratio >= 4.5, JSON.stringify(c)); const adj = page.locator('[data-testid=ss-adjust]'); await adj.waitFor({ timeout: 5000 }); const t = (await adj.textContent()) ?? ''; ok('adjustment summary visible and about readability', (await adj.isVisible()) && /text colou?rs? .*(adjusted|readab)|readab/i.test(t), t.slice(0, 200)); await modalShot(page, name, '06-contrast-summary.png'); }); scenario('7', 'vision unavailable', async ({ page, name, ok }) => { srv.setVision(false); srv.setNextResult(pageResult('Text only heading')); await openModal(page); await expandSection(page, 'mockups'); await page.waitForSelector('[data-testid=ss-vision-note]', { timeout: 8000 }); ok('vision note shown', await page.locator('[data-testid=ss-vision-note]').isVisible()); ok('file input disabled', await page.locator('input[data-testid=ss-mockup-input]').evaluate((e) => e.disabled)); ok('self-review toggle disabled', await page.locator('[data-testid=ss-selfreview-toggle] input').evaluate((e) => e.disabled && !e.checked)); await modalShot(page, name, '07-vision-unavailable.png'); await sendMsg(page, 'Just words please'); await waitCanvas(page, 'Text only heading'); ok('text-only send works end to end', srv.starts.length === 1 && !srv.starts[0].mockup_ids, JSON.stringify(srv.starts[0]?.mockup_ids)); }); scenario('8', 'self-review round trip (one round)', async ({ page, name, ok }) => { srv.setNextResult(pageResult('First pass heading', { id: 'fp-h' })); srv.setNextResult({ type: 'patch', message: 'Fixed spacing.', ops: [{ op: 'update_props', node_id: 'fp-h', props: { text: 'Reviewed heading' } }] }); await openModal(page); await sendMsg(page, 'Build and then check it'); await page.waitForSelector('[data-testid=ss-review-note]', { timeout: 45000 }); const note = (await page.textContent('[data-testid=ss-review-note]')) ?? ''; ok('review note says it made fixes', /made fixes/.test(note), note); await waitCanvas(page, 'Reviewed heading'); ok('patch from the second job applied', true); ok('two start jobs', srv.starts.length === 2, String(srv.starts.length)); const up = srv.uploads[0]; ok('render uploaded as a JPEG mockup named self-review-render.jpg, > 2000 bytes', !!up && up.mime === 'image/jpeg' && up.kind === 'mockup' && up.name === 'self-review-render.jpg' && up.size > 2000, JSON.stringify(up)); const s2 = srv.starts[1] ?? {}; ok('second job message starts "Review the attached screenshot"', typeof s2.message === 'string' && s2.message.startsWith('Review the attached screenshot'), String(s2.message).slice(0, 60)); ok('second job carries the upload id in mockup_ids', Array.isArray(s2.mockup_ids) && s2.mockup_ids.includes(up?.id), JSON.stringify(s2.mockup_ids)); ok('second job has use_assets false', s2.use_assets === false, String(s2.use_assets)); ok('render image deleted afterwards', srv.imageDeletes.includes(up?.id), JSON.stringify(srv.imageDeletes)); await page.waitForTimeout(2500); ok('no third job', srv.starts.length === 2, String(srv.starts.length)); await modalShot(page, name, '08-self-review-done.png'); }, { selfReview: true }); scenario('8b', 'self-review skipped when the upload fails', async ({ page, ok }) => { srv.setUploadFail(500); srv.setNextResult(pageResult('Kept first pass', { id: 'kp-h' })); await openModal(page); await sendMsg(page, 'Build it'); await page.waitForSelector('[data-testid=ss-review-note]', { timeout: 45000 }); const note = (await page.textContent('[data-testid=ss-review-note]')) ?? ''; ok('non-blocking "Self-review skipped" note', /Self-review skipped/.test(note), note); ok('first job content still applied', (await canvasText(page)).includes('Kept first pass')); ok('no second job', srv.starts.length === 1, String(srv.starts.length)); ok('no blocking error shown', (await page.locator('[data-testid=ss-error]').count()) === 0); }, { selfReview: true }); // ------------------------------------------------------------------ run async function run(name, launcher, exe) { const browser = await launcher.launch({ executablePath: exe || undefined, ...(name === 'chromium' ? { args: ['--no-sandbox'] } : {}) }); for (const sc of scenarios) { if (process.env.SITESMITH_SCENARIO && !process.env.SITESMITH_SCENARIO.split(',').includes(sc.id)) continue; srv.reset(); const label = `${sc.id} ${sc.title}`; let page; try { page = await newPage(browser, name, sc.opts); await sc.fn({ page, name, ok: (l, c, d) => log(name, label, !!c, l, d) }); await page.waitForTimeout(200); const bad = await page.evaluate(() => window.__bad); log(name, label, page.__errors.length === 0, 'no uncaught page errors', page.__errors.join(' | ')); log(name, label, bad.length === 0, 'no NetworkError / Failed to fetch text in the modal', JSON.stringify(bad)); const ext = page.__external.filter((u) => !/\.css(\?|$)|fonts\.(googleapis|gstatic)/.test(u)); if (ext.length) console.log(`[${name}] note ${label}: ${ext.length} external request(s) aborted locally`); } catch (e) { log(name, label, false, 'scenario completed', String(e?.message ?? e).split('\n').slice(0, 3).join(' ')); if (page && name === 'firefox') await page.screenshot({ path: path.join(shotsDir, `fail-${sc.id}.png`) }).catch(() => {}); } finally { await page?.__ctx.close().catch(() => {}); } } await browser.close(); } const only = process.env.SITESMITH_ONLY; if (only !== 'firefox') await run('chromium', chromium, process.env.CHROMIUM_PATH); if (only !== 'chromium') await run('firefox', firefox, process.env.FIREFOX_PATH); await srv.close(); console.log(failures ? `\n${failures} FAILURE(S)` : '\nALL PASS'); process.exit(failures ? 1 : 0);