site-builder/craft/src/panels/sitesmith/selfReview.test.ts

import { describe, test, expect, vi } from 'vitest';
import { runSelfReview, saysRenderMismatch, claimsAFix, REVIEW_MESSAGE, type SelfReviewDeps } from './selfReview';

const blob = () => new Blob([new Uint8Array(3000)], { type: 'image/jpeg' });
const deps = (o: Partial<SelfReviewDeps> = {}): SelfReviewDeps => ({
  composeHtml: () => '<html><body>x</body></html>',
  assetUrls: {}, allowUrls: [],
  resolveAssetUrls: (h) => h,
  capture: vi.fn(async () => ({ ok: true as const, blob: blob(), width: 1280, height: 900, cropped: false })),
  upload: vi.fn(async () => ({ ok: true as const, id: 'a'.repeat(32) })),
  review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message: 'm', ops: [{ op: 'delete_node' as const, node_id: 'n' }] } })),
  applyResult: vi.fn(async () => ({ ok: true })),
  removeImage: vi.fn(async () => ({})),
  ...o,
});

describe('runSelfReview', () => {
  test('reports the screenshot geometry; a cropped (very long) page says only its top was reviewed', async () => {
    const strips = await runSelfReview(deps({ capture: vi.fn(async () => ({ ok: true as const, blob: blob(), width: 2584, height: 2500, cropped: false, strips: 2, pageHeight: 5000 })) }));
    expect(strips.capture).toEqual({ strips: 2, cropped: false, pageHeight: 5000 });
    expect(strips.note).not.toMatch(/top 9000/);
    const cropped = await runSelfReview(deps({ capture: vi.fn(async () => ({ ok: true as const, blob: blob(), width: 3888, height: 3000, cropped: true, strips: 3, pageHeight: 14000 })) }));
    expect(cropped.status).toBe('fixed');
    expect(cropped.capture).toEqual({ strips: 3, cropped: true, pageHeight: 14000 });
    expect(cropped.note).toMatch(/only the top 9000 px of this 14000 px page/);
    const skipped = await runSelfReview(deps({ capture: vi.fn(async () => ({ ok: false as const, reason: 'capture timed out' })) }));
    expect(skipped.capture).toBeUndefined();
  });

  test('round trip: capture -> upload -> ONE review call -> apply -> render image deleted', async () => {
    const d = deps();
    const out = await runSelfReview(d);
    expect(out.status).toBe('fixed');
    expect(d.capture).toHaveBeenCalledTimes(1);
    expect(d.upload).toHaveBeenCalledTimes(1);
    expect(d.review).toHaveBeenCalledTimes(1);
    expect(d.review).toHaveBeenCalledWith(REVIEW_MESSAGE, 'a'.repeat(32));
    expect(d.applyResult).toHaveBeenCalledTimes(1);
    expect(d.removeImage).toHaveBeenCalledWith('a'.repeat(32));
    const file = (d.upload as any).mock.calls[0][0] as File;
    expect(file.type).toBe('image/jpeg');
  });
  test('capture failure skips with a note and calls nothing else', async () => {
    const d = deps({ capture: vi.fn(async () => ({ ok: false as const, reason: 'capture timed out' })) });
    const out = await runSelfReview(d);
    expect(out).toMatchObject({ status: 'skipped' });
    expect(out.note).toContain('capture timed out');
    expect(d.upload).not.toHaveBeenCalled();
    expect(d.review).not.toHaveBeenCalled();
    expect(d.removeImage).not.toHaveBeenCalled();
  });
  test('upload failure, review failure and apply failure all skip non-blockingly; the render is still cleaned up', async () => {
    expect((await runSelfReview(deps({ upload: vi.fn(async () => ({ ok: false as const, message: 'too big' })) }))).status).toBe('skipped');
    const d2 = deps({ review: vi.fn(async () => ({ ok: false as const, status: 'RATE_LIMITED' as const, message: 'Slow down' })) });
    const o2 = await runSelfReview(d2);
    expect(o2).toMatchObject({ status: 'skipped' });
    expect(o2.note).toContain('Slow down');
    expect(d2.removeImage).toHaveBeenCalled();
    const d3 = deps({ applyResult: vi.fn(async () => ({ ok: false, message: 'cannot apply' })) });
    expect((await runSelfReview(d3)).status).toBe('skipped');
    expect(d3.removeImage).toHaveBeenCalled();
  });
  test('a thrown error anywhere is a skip, never an exception', async () => {
    const out = await runSelfReview(deps({ capture: vi.fn(async () => { throw new Error('boom'); }) }));
    expect(out).toMatchObject({ status: 'skipped' });
    expect(await runSelfReview(deps({ composeHtml: () => { throw new Error('x'); } }))).toMatchObject({ status: 'skipped' });
  });
  test('an empty patch on the right page is "fine", and quotes what Sitesmith said', async () => {
    const d = deps({ review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message: 'Spacing and contrast look right.', ops: [] } })) });
    const out = await runSelfReview(d);
    expect(out.status).toBe('fine');
    expect(out.note).toMatch(/made no changes/);
    expect(out.note).toContain('Spacing and contrast look right.');
    expect(out.note).not.toMatch(/found nothing to fix/);
    expect(d.applyResult).not.toHaveBeenCalled();
  });
  test('an ask is not reported as a review that found nothing', async () => {
    const d2 = deps({ review: vi.fn(async () => ({ ok: true as const, response: { type: 'ask' as const, question: 'Which page?' } })) });
    const out = await runSelfReview(d2);
    expect(out.status).toBe('skipped');
    expect(out.note).toMatch(/asked a question instead of reviewing/);
    expect(out.note).toContain('Which page?');
    expect(d2.applyResult).not.toHaveBeenCalled();
  });
  // The five no-op replies the live evaluation got (the render was of the previous page).
  const EVAL_REPLIES = [
    "No changes made: the render (a 'MySite' nav, a hero with only a button, and two stacked contact forms) doesn't match the page I built, and the canvas ids (n2-n6) don't match my ai-* ids. I can't safely target nodes from this, so tell me which page to fix or ask me to rebuild the home page.",
    "No changes made. The render shows an empty hero with only a button, no brands, approach or values sections, and two duplicate contact forms. That doesn't match the page I built (hero, brands, approach, values, one contact form), so it looks like an older or partial render. I can't safely patch it, so I left your canvas untouched. Re-run the review once the new page has rendered.",
    "No fixes made. This render shows the old page (MySite nav, an empty hero with one button, two identical contact forms), not the OLS Digital page I built last turn, and its node ids (n2 to n5) don't match mine (ai-hero-1, ai-brands-1 and so on). I can't tell which of those nodes is which, so I left the canvas alone. If the old page is really still live, re-apply the rebuild or ask me to redo the home page.",
    "No changes made: the render (a 'MySite' nav, a hero with only a button, two stacked default contact forms) doesn't match the Tabletop Misfits page I built, and the canvas ids (n1-n6) don't match my ai-* ids, so I can't safely target nodes. Tell me which page to fix, or ask me to rebuild the home page.",
    "No changes made: the render (a 'MySite' nav, a hero showing only a 'Get Started' button, and two stacked default contact forms) doesn't match any page I built, and the canvas ids (n1-n6) don't match my ai-* ids, so I can't safely target nodes with update_props. Tell me which page to fix, or ask me to rebuild the home page.",
  ];
  test.each(EVAL_REPLIES)('model says the render is not its page -> "mismatch", never "found nothing to fix" (%#)', async (message) => {
    const out = await runSelfReview(deps({ review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message, ops: [] } })) }));
    expect(out.status).toBe('mismatch');
    expect(out.note).toMatch(/did not match the page it built/);
    expect(out.note).not.toMatch(/found nothing to fix|made no changes\./);
  });
  test('mismatch detection stays narrow: ordinary "looks fine" replies are not mismatches', () => {
    for (const m of [
      'Looks right: the page matches the brief, so no changes.',
      'Everything reads well; the button colour does not match the accent but that is a style choice, so I left it.',
      'The hero image is well cropped and nothing needs fixing.',
      '',
    ]) expect(saysRenderMismatch(m)).toBe(false);
  });
  test('ops that name nodes not on the page: "not-applied", not "made fixes"', async () => {
    const d = deps({ applyResult: vi.fn(async () => ({ ok: true, changed: 0, ops: 1 })) });
    const out = await runSelfReview(d);
    expect(out.status).toBe('not-applied');
    expect(out.note).toMatch(/none could be applied/);
    const d2 = deps({
      review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message: 'm', ops: [{ op: 'delete_node' as const, node_id: 'a' }, { op: 'delete_node' as const, node_id: 'b' }] } })),
      applyResult: vi.fn(async () => ({ ok: true, changed: 1, ops: 2 })),
    });
    const o2 = await runSelfReview(d2);
    expect(o2.status).toBe('fixed');
    expect(o2.note).toMatch(/1 of 2 suggested fixes were applied/);
  });
  test('canvas not showing the applied result: skipped with the reason, nothing captured or sent', async () => {
    const d = deps({ canvasMismatch: () => 'the canvas had not finished loading the new page' });
    const out = await runSelfReview(d);
    expect(out.status).toBe('skipped');
    expect(out.note).toContain('the canvas had not finished loading the new page');
    expect(d.capture).not.toHaveBeenCalled();
    expect(d.upload).not.toHaveBeenCalled();
    expect(d.review).not.toHaveBeenCalled();
  });
  test('assets are resolved before capture', async () => {
    const resolve = vi.fn((h: string) => h.replace('assets/a.png', '/api/a'));
    const d = deps({ composeHtml: () => '<img src="assets/a.png">', resolveAssetUrls: resolve, assetUrls: { 'assets/a.png': '/api/a' } });
    await runSelfReview(d);
    expect((d.capture as any).mock.calls[0][0]).toContain('/api/a');
  });
  // Live eval 2026-11-01 (ols-home-firefox): the model said "Fixed the hero" but sent ops: [].
  const CLAIMS = [
    'Fixed the hero: it rendered on a near-white background, so the headline, subtitle and outline button were almost invisible. It now sits on the navy brand colour with white text. The rest of the page reads fine, so I left it alone.',
    'I darkened the hero overlay and increased the card padding.',
    'Changed the CTA button to the accent colour.',
  ];
  test.each(CLAIMS)('a reply that claims a fix but sends no ops is reported honestly, never as fixed (%#)', async (message) => {
    const d = deps({ review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message, ops: [] } })) });
    const out = await runSelfReview(d);
    expect(out.status).toBe('claimed-no-ops');
    expect(out.note).toMatch(/sent no changes/);
    expect(out.note).toMatch(/nothing on your page was changed/i);
    expect(d.applyResult).not.toHaveBeenCalled();
  });
  test('claim detection stays narrow: "no fixes needed" style replies are not claims', () => {
    for (const m of [
      'No fixes needed: contrast, spacing, alignment and the mobile-stacking column setup all look right. The hero photo is still a grey placeholder, so upload a team photo to replace it.',
      'The page already looks right, so no changes. Text contrast holds on the dark bands and the red band, the show cards are even, spacing follows one rhythm, and the icons and form render properly.',
      'Nothing needed to be changed.',
      'I have not changed anything; it looks right.',
      'I made no changes because everything reads well.',
      'Changed nothing, the page looks right.',
      '',
    ]) expect(claimsAFix(m)).toBe(false);
  });
  test('structured render_mismatch flag wins over wording (and an old reply without it still works)', async () => {
    const d = deps({ review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message: 'Screenshot mismatch: the hero is navy on the page I built.', ops: [], render_mismatch: true } })) });
    const out = await runSelfReview(d);
    expect(out.status).toBe('mismatch');
    expect(out.note).toMatch(/screenshot/i);
    const d2 = deps({ review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message: 'Looks right.', ops: [] } })) });
    expect((await runSelfReview(d2)).status).toBe('fine');
  });
  test('render_mismatch with ops still applies the ops (the flag never blocks real fixes)', async () => {
    const d = deps({ review: vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message: 'm', ops: [{ op: 'delete_node' as const, node_id: 'n' }], render_mismatch: true } })) });
    expect((await runSelfReview(d)).status).toBe('fixed');
  });
});

// ---- round 4 item 6: honest wording when the server's safety checks dropped the review's changes
describe('runSelfReview: changes dropped by the safety checks', () => {
  const LIVE = 'Image band narrowed and the copy moved up so the hero reads first.';
  const reply = (ops: any[], reviewOps?: { proposed: number; kept: number }, message = LIVE) =>
    vi.fn(async () => ({ ok: true as const, response: { type: 'patch' as const, message, ops }, ...(reviewOps ? { reviewOps } : {}) }));
  test('all proposed changes dropped: never "made no changes. Sitesmith said: <a fix>"', async () => {
    const out = await runSelfReview(deps({ review: reply([], { proposed: 3, kept: 0 }) }));
    expect(out.status).toBe('dropped');
    expect(out.note).toContain('Sitesmith proposed 3 changes, but none passed the safety checks, so nothing on your page was changed.');
    expect(out.note).not.toMatch(/made no changes|Sitesmith said:/);
    expect(out.note).toContain(`What it proposed (not applied): "${LIVE}"`);
  });
  test('singular', async () => {
    const out = await runSelfReview(deps({ review: reply([], { proposed: 1, kept: 0 }) }));
    expect(out.note).toContain('Sitesmith proposed 1 change, but it did not pass the safety checks');
  });
  test('some dropped, the rest applied: says how many were dropped, and that the description may include them', async () => {
    const ops = [{ op: 'update_props' as const, node_id: 'a', props: { style: { color: '#000' } } }];
    const out = await runSelfReview(deps({ review: reply(ops, { proposed: 3, kept: 1 }), applyResult: vi.fn(async () => ({ ok: true, changed: 1, ops: 1 })) }));
    expect(out.status).toBe('fixed');
    expect(out.note).toContain('Sitesmith proposed 3 changes; 2 did not pass the safety checks and were dropped.');
    expect(out.note).toContain(`Sitesmith's description (it may include the dropped changes): "${LIVE}"`);
  });
  test('nothing dropped: unchanged wording (fine / fixed)', async () => {
    const fine = await runSelfReview(deps({ review: reply([], { proposed: 0, kept: 0 }, 'Looks right.') }));
    expect(fine.status).toBe('fine');
    expect(fine.note).toMatch(/made no changes\. Sitesmith said: "Looks right\."/);
    const old = await runSelfReview(deps({ review: reply([], undefined, 'Looks right.') }));   // older backend
    expect(old.status).toBe('fine');
  });
  test('ops sent but none applied in the editor: the quote is labelled as not applied', async () => {
    const out = await runSelfReview(deps({ applyResult: vi.fn(async () => ({ ok: true, changed: 0, ops: 1 })) }));
    expect(out.status).toBe('not-applied');
    expect(out.note).toMatch(/What it proposed \(not applied\): "m"/);
    expect(out.note).not.toMatch(/Sitesmith said:/);
  });
  test('claimed a fix but sent no ops: the quote is labelled as not applied', async () => {
    const out = await runSelfReview(deps({ review: reply([], { proposed: 0, kept: 0 }, 'Fixed the hero contrast.') }));
    expect(out.status).toBe('claimed-no-ops');
    expect(out.note).toContain('Its reply (nothing was applied): "Fixed the hero contrast."');
  });
});