fix: centre search excerpts on rare query terms, match simple inflections #29

Merged
jknapp merged 1 commits from fix/excerpt-centering into main 2026-11-14 19:14:27 +00:00
2 changed files with 71 additions and 7 deletions

No files matched your search

+20
View File
@@ -53,4 +53,24 @@ describe("makeExcerpt", () => {
expect(excerpt.endsWith("needle")).toBe(true);
expect(excerpt.length).toBeGreaterThan(EXCERPT_CHARS - 20);
});
// Regression from the live OLS memory: "WHP" and "partner" also appear
// in an earlier sentence ("business partners … (1) WHP commercial
// launch"), which tied the Founding Partner passage on raw term count
// and won by coming first. Rare terms must outweigh common ones.
test("weights rare query terms above ones that appear all over the memory", () => {
const ols =
'OLS Digital LLC ("One Last Shot"): Josh\'s solo parent company (alongside a full-time SWE job, 10–20 hrs/week) for AnHonestHost (boutique hosting since 2014), WHP (Web Hosting Platform: containerized hosting with a Go control panel), Streamer Tools (TTRPG streaming toolkit, labor of love) and myTppy (NFC card plus link-in-bio). AI agents are treated as business partners: honest, direct, push back when something won\'t work.\n\n' +
"Q1 2027 priorities: (1) WHP commercial launch early January 2027, goal 10 licensed installs at hosting providers; (2) AnHonestHost \"own your brand\" campaign for creators and small businesses, promoted by trusted creator friends on commission.\n\n" +
'WHP decisions (as of 2026-11-14): $20 or $25 per server (TBD), the same for physical or VM, no per-account tiers; includes a webmail client for Poste.io (hosts license Poste.io themselves, about $380/yr). x86 and ARM64 with identical features. Self-serve cPanel import built in (don\'t use "cPanel" in feature names, trademark). Founding Partner Program: first 10 hosts get 90 days free on 1 server, one-on-one onboarding (setup, test migrations, WHMCS integration) and a locked price; multi-server partners get the standard 30-day free period.';
const { excerpt } = makeExcerpt(ols, "WHP founding partner pricing");
expect(excerpt).toContain("Founding Partner Program");
// "pricing" should reach "locked price" in the same window.
expect(excerpt).toContain("locked price");
});
test("matches simple inflections of a query word (pricing → price)", () => {
const content = `${words(100)} we settled on a flat price per server ${words(100)}`;
expect(makeExcerpt(content, "pricing").excerpt).toContain("flat price per server");
});
});
+51 -7
View File
@@ -7,10 +7,20 @@
* `truncated`, and the caller fetches the whole body with memory.get when
* the excerpt says it's worth it.
*
* With a query, the excerpt is the window matching the most distinct
* query terms (case-insensitive substring match, so "launch" also finds
* "launched"). Without one — or when no term appears, e.g. a vector-only
* hit — it is the start of the memory, which is usually its summary line.
* With a query, the excerpt is the window with the highest weighted score
* over the query terms it contains. Each term is weighted by how RARE it
* is in this memory (1 / occurrences): a word that appears once points at
* one passage, while a word sprinkled throughout (often the project's own
* name) says little about where to look. Without the weighting, a common
* term plus one coincidental hit tied the real passage and won by coming
* first. Terms are matched case-insensitively as prefixes after light
* suffix stripping, so "pricing" finds "price" and "launch" finds
* "launched". Without a query — or when no term appears, e.g. a
* vector-only hit — the excerpt is the start of the memory, which is
* usually its summary line.
*
* Postgres ts_headline was tried for this and picked the same wrong
* passage: it also weighs every query word equally.
*/
export const EXCERPT_CHARS = 200;
@@ -35,16 +45,50 @@ export interface Excerpt {
truncated: boolean;
}
/** Suffixes stripped so a query word matches its simple inflections. */
const SUFFIXES = ["ing", "ed", "es", "s", "e"];
/**
* Crude stem: strip one common suffix as long as 4+ chars remain, so the
* result works as a prefix ("pricing"/"price" → "pric", "partners" →
* "partner"). Not a real stemmer — a missed inflection only costs a
* slightly worse window, never a wrong result.
*/
function stem(term: string): string {
for (const suf of SUFFIXES) {
if (term.endsWith(suf) && term.length - suf.length >= 4) {
return term.slice(0, -suf.length);
}
}
return term;
}
function queryTerms(query: string): string[] {
const terms = query
.toLowerCase()
.split(/[^\p{L}\p{N}]+/u)
.filter((t) => t.length >= 3 && !STOPWORDS.has(t));
.filter((t) => t.length >= 3 && !STOPWORDS.has(t))
.map(stem);
return [...new Set(terms)];
}
/** Occurrences of `term` in `lower`, capped. */
function countOccurrences(lower: string, term: string): number {
let n = 0;
for (let i = lower.indexOf(term); i !== -1 && n < MAX_OCCURRENCES_PER_TERM; i = lower.indexOf(term, i + 1)) n++;
return n;
}
/** Start offset of the best-matching window, or 0 when nothing matches. */
function bestWindowStart(lower: string, terms: string[], max: number): number {
function bestWindowStart(lower: string, allTerms: string[], max: number): number {
// Rarity weight per term; terms absent from the memory drop out.
const weights = new Map<string, number>();
for (const t of allTerms) {
const n = countOccurrences(lower, t);
if (n > 0) weights.set(t, 1 / n);
}
const terms = [...weights.keys()];
let best = 0;
let bestScore = 0;
for (const term of terms) {
@@ -54,7 +98,7 @@ function bestWindowStart(lower: string, terms: string[], max: number): number {
if (i === -1) break;
const start = Math.max(0, i - LEAD_CHARS);
const window = lower.slice(start, start + max);
const score = terms.reduce((s, t) => s + (window.includes(t) ? 1 : 0), 0);
const score = terms.reduce((s, t) => s + (window.includes(t) ? weights.get(t)! : 0), 0);
if (score > bestScore || (score === bestScore && start < best)) {
best = start;
bestScore = score;