fix: centre search excerpts on rare query terms, match simple inflections #29
No files matched your search
@@ -53,4 +53,24 @@ describe("makeExcerpt", () => {
|
||||
expect(excerpt.endsWith("needle")).toBe(true);
|
||||
expect(excerpt.length).toBeGreaterThan(EXCERPT_CHARS - 20);
|
||||
});
|
||||
|
||||
// Regression from the live OLS memory: "WHP" and "partner" also appear
|
||||
// in an earlier sentence ("business partners … (1) WHP commercial
|
||||
// launch"), which tied the Founding Partner passage on raw term count
|
||||
// and won by coming first. Rare terms must outweigh common ones.
|
||||
test("weights rare query terms above ones that appear all over the memory", () => {
|
||||
const ols =
|
||||
'OLS Digital LLC ("One Last Shot"): Josh\'s solo parent company (alongside a full-time SWE job, 10–20 hrs/week) for AnHonestHost (boutique hosting since 2014), WHP (Web Hosting Platform: containerized hosting with a Go control panel), Streamer Tools (TTRPG streaming toolkit, labor of love) and myTppy (NFC card plus link-in-bio). AI agents are treated as business partners: honest, direct, push back when something won\'t work.\n\n' +
|
||||
"Q1 2027 priorities: (1) WHP commercial launch early January 2027, goal 10 licensed installs at hosting providers; (2) AnHonestHost \"own your brand\" campaign for creators and small businesses, promoted by trusted creator friends on commission.\n\n" +
|
||||
'WHP decisions (as of 2026-11-14): $20 or $25 per server (TBD), the same for physical or VM, no per-account tiers; includes a webmail client for Poste.io (hosts license Poste.io themselves, about $380/yr). x86 and ARM64 with identical features. Self-serve cPanel import built in (don\'t use "cPanel" in feature names, trademark). Founding Partner Program: first 10 hosts get 90 days free on 1 server, one-on-one onboarding (setup, test migrations, WHMCS integration) and a locked price; multi-server partners get the standard 30-day free period.';
|
||||
const { excerpt } = makeExcerpt(ols, "WHP founding partner pricing");
|
||||
expect(excerpt).toContain("Founding Partner Program");
|
||||
// "pricing" should reach "locked price" in the same window.
|
||||
expect(excerpt).toContain("locked price");
|
||||
});
|
||||
|
||||
test("matches simple inflections of a query word (pricing → price)", () => {
|
||||
const content = `${words(100)} we settled on a flat price per server ${words(100)}`;
|
||||
expect(makeExcerpt(content, "pricing").excerpt).toContain("flat price per server");
|
||||
});
|
||||
});
|
||||
@@ -7,10 +7,20 @@
|
||||
* `truncated`, and the caller fetches the whole body with memory.get when
|
||||
* the excerpt says it's worth it.
|
||||
*
|
||||
* With a query, the excerpt is the window matching the most distinct
|
||||
* query terms (case-insensitive substring match, so "launch" also finds
|
||||
* "launched"). Without one — or when no term appears, e.g. a vector-only
|
||||
* hit — it is the start of the memory, which is usually its summary line.
|
||||
* With a query, the excerpt is the window with the highest weighted score
|
||||
* over the query terms it contains. Each term is weighted by how RARE it
|
||||
* is in this memory (1 / occurrences): a word that appears once points at
|
||||
* one passage, while a word sprinkled throughout (often the project's own
|
||||
* name) says little about where to look. Without the weighting, a common
|
||||
* term plus one coincidental hit tied the real passage and won by coming
|
||||
* first. Terms are matched case-insensitively as prefixes after light
|
||||
* suffix stripping, so "pricing" finds "price" and "launch" finds
|
||||
* "launched". Without a query — or when no term appears, e.g. a
|
||||
* vector-only hit — the excerpt is the start of the memory, which is
|
||||
* usually its summary line.
|
||||
*
|
||||
* Postgres ts_headline was tried for this and picked the same wrong
|
||||
* passage: it also weighs every query word equally.
|
||||
*/
|
||||
|
||||
export const EXCERPT_CHARS = 200;
|
||||
@@ -35,16 +45,50 @@ export interface Excerpt {
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/** Suffixes stripped so a query word matches its simple inflections. */
|
||||
const SUFFIXES = ["ing", "ed", "es", "s", "e"];
|
||||
|
||||
/**
|
||||
* Crude stem: strip one common suffix as long as 4+ chars remain, so the
|
||||
* result works as a prefix ("pricing"/"price" → "pric", "partners" →
|
||||
* "partner"). Not a real stemmer — a missed inflection only costs a
|
||||
* slightly worse window, never a wrong result.
|
||||
*/
|
||||
function stem(term: string): string {
|
||||
for (const suf of SUFFIXES) {
|
||||
if (term.endsWith(suf) && term.length - suf.length >= 4) {
|
||||
return term.slice(0, -suf.length);
|
||||
}
|
||||
}
|
||||
return term;
|
||||
}
|
||||
|
||||
function queryTerms(query: string): string[] {
|
||||
const terms = query
|
||||
.toLowerCase()
|
||||
.split(/[^\p{L}\p{N}]+/u)
|
||||
.filter((t) => t.length >= 3 && !STOPWORDS.has(t));
|
||||
.filter((t) => t.length >= 3 && !STOPWORDS.has(t))
|
||||
.map(stem);
|
||||
return [...new Set(terms)];
|
||||
}
|
||||
|
||||
/** Occurrences of `term` in `lower`, capped. */
|
||||
function countOccurrences(lower: string, term: string): number {
|
||||
let n = 0;
|
||||
for (let i = lower.indexOf(term); i !== -1 && n < MAX_OCCURRENCES_PER_TERM; i = lower.indexOf(term, i + 1)) n++;
|
||||
return n;
|
||||
}
|
||||
|
||||
/** Start offset of the best-matching window, or 0 when nothing matches. */
|
||||
function bestWindowStart(lower: string, terms: string[], max: number): number {
|
||||
function bestWindowStart(lower: string, allTerms: string[], max: number): number {
|
||||
// Rarity weight per term; terms absent from the memory drop out.
|
||||
const weights = new Map<string, number>();
|
||||
for (const t of allTerms) {
|
||||
const n = countOccurrences(lower, t);
|
||||
if (n > 0) weights.set(t, 1 / n);
|
||||
}
|
||||
const terms = [...weights.keys()];
|
||||
|
||||
let best = 0;
|
||||
let bestScore = 0;
|
||||
for (const term of terms) {
|
||||
@@ -54,7 +98,7 @@ function bestWindowStart(lower: string, terms: string[], max: number): number {
|
||||
if (i === -1) break;
|
||||
const start = Math.max(0, i - LEAD_CHARS);
|
||||
const window = lower.slice(start, start + max);
|
||||
const score = terms.reduce((s, t) => s + (window.includes(t) ? 1 : 0), 0);
|
||||
const score = terms.reduce((s, t) => s + (window.includes(t) ? weights.get(t)! : 0), 0);
|
||||
if (score > bestScore || (score === bestScore && start < best)) {
|
||||
best = start;
|
||||
bestScore = score;
|
||||
|
||||
Reference in new issue
Block a user