From 2bef06d108e9db28cd2a8667ff6bf1ed56a5e8cd Mon Sep 17 00:00:00 2001 From: Lee Hanken Date: Sun, 22 Nov 2026 11:07:28 +0000 Subject: [PATCH] Add politeness delay between requests, fix two more marker gaps found by smoke-testing - config.py/fetch.py: enforce a minimum gap between any requests to HPR's servers (fetch._wait_for_politeness_gap()), regardless of which function issues them. Not needed for someone who already understands HPR's infrastructure, but a real consideration for a future user who doesn't -- a sweep makes several requests per episode across thousands of episodes. - markers.json (version 3): found via the same random-sample-then- visualize technique used for the earlier hpr4775 fix. hpr2157's era apparently used the bare 'HPR episode' rather than 'Hacker Public Radio episode', which fell all the way through to a much weaker sibling- correlation fallback (3.25s) instead of a proper marker match (now 15.1s); Whisper tiny mis-hearing 'Today's' as 'To its' on hpr3356 broke the same marker a different way (8.0s -> 26.6s). hpr3356 itself is an unusual 282-minute special episode that remains correctly flagged for manual review even after the fix. - docs/DESIGN.md: corrected an overstated claim ('retry with backoff') that was never actually implemented; documented the real smoke test (SIGINT mid-episode, resumed, interrupted again, resumed to completion -- no lost work, no stray subprocesses, no ledger corruption) and a real full-corpus timing estimate (32.1s/episode average on the hardest episode category, so on the order of 1-2 days for the full archive, not weeks). - New regression tests for the politeness delay and both marker fixes. --- docs/DESIGN.md | 76 +++++++++++++++++++++++++++------------ hpr_stripper/config.py | 9 ++++- hpr_stripper/fetch.py | 19 ++++++++++ hpr_stripper/markers.json | 7 ++-- tests/test_markers.py | 32 +++++++++++++++++ tests/test_politeness.py | 28 +++++++++++++++ 6 files changed, 145 insertions(+), 26 deletions(-) create mode 100644 tests/test_politeness.py diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 903ca45..660712a 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -161,8 +161,15 @@ special-cased per status code. a *private* origin-to-mirror-node design: volunteer-run edge nodes `rsync` over SSH from a private origin, not open to the public, with directory browsing explicitly disabled. The practical approach is a polite, -resumable, per-episode HTTP fetch loop (skip-if-cached, rate-limited, -retry with backoff) — not an attempt to find or build a bulk-sync path. +resumable, per-episode HTTP fetch loop — not an attempt to find or build a +bulk-sync path. Concretely: every request to HPR's servers goes through a +single gate (`fetch._wait_for_politeness_gap()`) enforcing a minimum +`config.REQUEST_DELAY_SECONDS` (default 0.5s) between requests, regardless +of which function issued them, and `fetch_audio`/`fetch_official_srt` +skip the network entirely once a file is cached. Retry-with-backoff on a +transient failure is not implemented — a failed request is currently +treated the same as "not available," falling through to the next +detection signal rather than being retried. **Official transcripts/SRT exist, but coverage is patchy — confirmed by direct testing, not assumed.** Even episodes from 2024–2025 sometimes @@ -334,30 +341,30 @@ expected to keep growing. hpr-intro-stripper/ ├── README.md # minimal: usage, requirements, brief how-it-works ├── docs/ -│ └── DESIGN.md # this file -├── references/ # vendored from HPR's own hpr-tools repo (CC0/CC-BY-SA, not this repo's code license) +│ └── DESIGN.md # this file +├── references/ # vendored from HPR's own hpr-tools repo (CC0/CC-BY-SA, not this repo's code license) │ ├── theme.flac │ ├── outro.flac │ ├── intro.srt │ ├── outro.srt -│ ├── mysql2sqlite # the same AWK converter HPR's own tooling uses +│ ├── mysql2sqlite # the same AWK converter HPR's own tooling uses │ └── LICENSE_NOTICE.md ├── hpr_stripper/ -│ ├── config.py # the one place to change defaults -│ ├── markers.json # versioned, plain-text boilerplate phrase library -│ ├── markers.py # fuzzy matching against markers.json -│ ├── metadata.py # canonical hpr.sql fetch + sqlite conversion + queries -│ ├── fetch.py # audio/transcript/SRT fetch via ccdn.php only -│ ├── srt.py # minimal SRT parser (stdlib only) -│ ├── audio.py # ffmpeg/ffprobe wrappers, cross-correlation -│ ├── asr.py # pluggable ASR backend dispatch -│ ├── detect.py # the layered detection pipeline -│ ├── ledger.py # resumable per-episode progress tracking -│ ├── trim.py # atomic trim + license gating + sidecar metadata -│ └── cli.py # detect / sweep / random-sample / trim / report +│ ├── config.py # the one place to change defaults +│ ├── markers.json # versioned, plain-text boilerplate phrase library +│ ├── markers.py # fuzzy matching against markers.json +│ ├── metadata.py # canonical hpr.sql fetch + sqlite conversion + queries +│ ├── fetch.py # audio/transcript/SRT fetch via ccdn.php only +│ ├── srt.py # minimal SRT parser (stdlib only) +│ ├── audio.py # ffmpeg/ffprobe wrappers, cross-correlation +│ ├── asr.py # pluggable ASR backend dispatch +│ ├── detect.py # the layered detection pipeline +│ ├── ledger.py # resumable per-episode progress tracking +│ ├── trim.py # atomic trim + license gating + sidecar metadata +│ └── cli.py # detect / sweep / random-sample / trim / report ├── tools/ -│ └── visualize_clip.sh # renders a waveform/spectrogram for a clip, for inspecting a cut point directly -└── cache/ # gitignored: hpr.db, downloaded audio, transcripts, ledger.db, trimmed output +│ └── visualize_clip.sh # renders a waveform/spectrogram for a clip, for inspecting a cut point directly +└── cache/ # gitignored: hpr.db, downloaded audio, transcripts, ledger.db, trimmed output ``` **What lives in git:** code, the marker-phrase library, the vendored @@ -381,9 +388,20 @@ Two first-class test modes, not ad hoc spot checks: ~2020+) so a lucky or unlucky sample can't hide an era-specific regression. Fast, for day-to-day development checks. - **Full-corpus sweep test** (`cli.py sweep`): runs detection (not - trimming) across every episode in the archive. Slow — potentially hours - once ASR fallback is needed for a meaningful fraction of 4,500+ episodes - — and explicitly designed to be interrupted. + trimming) across every episode in the archive. Slow — a real, + continuous run of 21 episodes from the hardest category (pre-2015, no + official transcript, full ASR + sibling-correlation fallback) averaged + 32.1 seconds/episode on the hardware this was developed on; scaled to + the full ~4,750-episode archive that's on the order of a day or two of + continuous wall-clock time, not weeks, though a weaker CPU or the + politeness delay (see "Data acquisition") would push it higher — and + explicitly designed to be interrupted. +- **Interrupt/resume was smoke-tested for real, not just unit-tested**: + a `sweep` was started, sent a real `SIGINT` mid-episode, resumed (and + confirmed it skipped the already-completed episodes rather than + reprocessing them), interrupted again, and resumed to completion. No + lost work, no stray subprocesses left running after either interrupt, + no corrupted ledger state. **Resumability is a hard requirement**, because a half-hour-plus batch run *will* get interrupted (Ctrl+C, laptop sleep, SSH drop): @@ -478,7 +496,19 @@ from the generation code itself, not inferred: set didn't recognize yet, so matching stopped two segments early. Fixed in `markers.json` (bumped to version 2); see `tests/test_markers.py`'s `test_2026_wording_variant_is_recognized` for the locked-in regression - test. + test. The same technique (a `random-sample` run flagging a + `metadata_arithmetic` disagreement, then rendering the clip to check + which side was actually right) found two more gaps in one pass: + hpr2157's era apparently used the bare abbreviation "HPR episode" rather + than "Hacker Public Radio episode" (intro_end was landing at 3.25s via + the much weaker sibling-correlation fallback, instead of a proper + marker match), and Whisper `tiny` mis-hearing "Today's" as "To its" on + hpr3356 broke the same marker a different way. Both fixed in + `markers.json` (version 3). hpr3356 itself (a 282-minute New Year's Eve + marathon episode) remains an open case even after the fix — its + structure doesn't cleanly match the standard template, and it's still + correctly flagged for manual review rather than given a confident wrong + answer. - **The theme-correlation fallback can't bridge a genuine silence gap** — also found via the hpr4775 waveform inspection above, and still open. `generate_intro()` mixes TTS speech additively onto `theme.flac` diff --git a/hpr_stripper/config.py b/hpr_stripper/config.py index e097353..e3cce10 100644 --- a/hpr_stripper/config.py +++ b/hpr_stripper/config.py @@ -16,6 +16,13 @@ CCDN_BASE = "https://hub.hackerpublicradio.org/ccdn.php" # pulls from. HPR_SQL_URL = "https://www.hackerpublicradio.org/hpr.sql" +# Minimum gap between requests to HPR's servers. Anyone running this who +# already understands HPR's infrastructure can lower it; the default has +# to assume a future user who doesn't, and a `sweep` over the full archive +# makes several requests per episode across thousands of episodes, so this +# is a real, not hypothetical, consideration. +REQUEST_DELAY_SECONDS = 0.5 + # Try formats in this order; fall back to the next on failure. All three are # always published per-episode. AUDIO_FORMAT_PREFERENCE = ("opus", "ogg", "mp3") @@ -41,7 +48,7 @@ XCORR_MATCH_THRESHOLD = 0.9 # Marker-phrase library version. Bump this whenever markers.json changes; # the ledger uses it to know when a cached "no signal found" result needs # re-checking against a newer marker set. -MARKERS_VERSION = 2 +MARKERS_VERSION = 3 # Licenses that forbid producing/redistributing a derivative without extra # permission. Trimming is refused for these unless explicitly overridden. diff --git a/hpr_stripper/fetch.py b/hpr_stripper/fetch.py index 395cc40..2f9937f 100644 --- a/hpr_stripper/fetch.py +++ b/hpr_stripper/fetch.py @@ -9,6 +9,8 @@ same outcome: "not available right now" -- callers fall back to the next detection signal rather than special-casing any one status code. """ import os +import threading +import time import urllib.error import urllib.request @@ -16,6 +18,22 @@ from . import config MIN_PLAUSIBLE_BYTES = 16 # anything smaller than this is treated as a placeholder +_last_request_lock = threading.Lock() +_last_request_at = 0.0 + + +def _wait_for_politeness_gap(): + """Enforces config.REQUEST_DELAY_SECONDS between any two requests this + process makes to HPR's servers, regardless of which function issued + them (audio, SRT, txt, or a neighbour's audio during sibling + correlation all share this gate).""" + global _last_request_at + with _last_request_lock: + wait = config.REQUEST_DELAY_SECONDS - (time.monotonic() - _last_request_at) + if wait > 0: + time.sleep(wait) + _last_request_at = time.monotonic() + # A byte-length check alone isn't enough -- an error response large enough # to clear MIN_PLAUSIBLE_BYTES (an HTML 404 page, an S3-style AccessDenied # XML body) would otherwise get saved under an audio/SRT filename and fail @@ -35,6 +53,7 @@ def _looks_like_error_page(data): def _fetch_bytes(url, timeout=30): + _wait_for_politeness_gap() try: with urllib.request.urlopen(url, timeout=timeout) as resp: data = resp.read() diff --git a/hpr_stripper/markers.json b/hpr_stripper/markers.json index c01f727..5f7fa1a 100644 --- a/hpr_stripper/markers.json +++ b/hpr_stripper/markers.json @@ -1,13 +1,16 @@ { - "version": 2, - "note": "Boilerplate phrase library for the station-added intro/outro. HPR's wording has drifted at least 4 times over its history (pre-2015: no spoken announcement at all; ~2015-2020: a simpler template; ~2020-2025ish: a fuller template with explicit date/series fields; ~2026: reworded again, e.g. 'the host is X and the duration is Y' / 'the flag is Y, and the license is Z' -- found via direct waveform/spectrogram inspection of hpr4775's real official SRT, where the old marker set stopped matching after segment 2 and silently under-trimmed) and will keep drifting. Add new phrasing here as it's spotted -- never hardcode it elsewhere. 'strong' markers must be specific enough that a host could not plausibly say them by coincidence in their own recording; 'greeting' markers can only extend an already-established run, never start one on their own (see hpr0500 in docs/DESIGN.md for why that distinction exists).", + "version": 3, + "note": "Boilerplate phrase library for the station-added intro/outro. HPR's wording has drifted at least 4 times over its history (pre-2015: no spoken announcement at all; ~2015-2020: a simpler template, apparently using the bare abbreviation 'HPR episode' rather than the full 'Hacker Public Radio episode'; ~2020-2025ish: a fuller template with explicit date/series fields; ~2026: reworded again, e.g. 'the host is X and the duration is Y' / 'the flag is Y, and the license is Z' -- found via direct waveform/spectrogram inspection of hpr4775's real official SRT, where the old marker set stopped matching after segment 2 and silently under-trimmed) and will keep drifting. Add new phrasing here as it's spotted -- never hardcode it elsewhere. Some additions here are ASR mis-hearings rather than genuine wording changes (e.g. Whisper tiny hearing \"Today's\" as \"To its\", or \"is\" as \"in\") -- found the same way, by rendering a clip with tools/visualize_clip.sh and comparing a fresh transcript against the marker set on a real disagreement flag (hpr2157, hpr3356). 'strong' markers must be specific enough that a host could not plausibly say them by coincidence in their own recording; 'greeting' markers can only extend an already-established run, never start one on their own (see hpr0500 in docs/DESIGN.md for why that distinction exists).", "strong_intro_markers": [ "hacker public radio episode", "hbr episode", + "hpr episode", "todays show is entitled", + "to its show is entitled", "todays show is titled", "been titled", "is part of the series", + "in part of the series", "is hosted by", "it is posted by", "the host is", diff --git a/tests/test_markers.py b/tests/test_markers.py index 5cd8bef..30c5255 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -43,6 +43,38 @@ class TestIntroDetection(unittest.TestCase): ] self.assertEqual(markers.find_intro_end(segments), 27.32) + def test_2015_2020_template_with_bare_hpr_abbreviation(self): + # hpr2157, real Whisper tiny transcription. This era's template + # apparently says the bare "HPR episode" rather than "Hacker + # Public Radio episode"; Whisper also mis-heard "is part" as "in + # part" here specifically. Found via a waveform inspection after a + # metadata_arithmetic disagreement flag (84.2s vs 3.2s) pointed at + # this episode; the old marker set matched nothing in segment 0 + # and returned None immediately, falling through to a much worse + # sibling-correlation guess (3.25s) instead of the real ~37.6s + # boundary. + segments = [ + ("This in HPR episode 2,157 entitled Markham Manchester Part 3, and in part of the series, Interviews.", 0.0, 8.1), + ("It is posted by Tony H1,212, and in about 8 minutes long.", 8.1, 14.0), + ("The summary is.", 14.0, 15.1), + ("This in an Interview with Alononovo, which I did at Markham Manchester.", 15.1, 19.8), + ("This episode of HPR is brought to you by An Honest Host at Come.", 19.8, 24.3), + ("Get 15% discount on all shared hosting with the offer code HPR-15.", 24.3, 30.6), + ("That's HPR-15.", 30.6, 33.1), + ("Better web hosting that's Honest and Fair at An Honest Host.com.", 33.1, 37.6), + ] + self.assertIsNotNone(markers.find_intro_end(segments)) + + def test_todays_misheard_as_to_its(self): + # hpr3356: Whisper tiny heard "Today's show is entitled" as "To + # its show is entitled", breaking the marker on segment 1 even + # though segment 0 matched fine. + segments = [ + ("This is Hacker Public Radio episode, 3,356 Formundi, 14 June 2021.", 0.0, 8.0), + ("To its show is entitled, HPR 2020, 2021 New Year's Eve, Show Episode 5.", 8.0, 15.5), + ] + self.assertEqual(markers.find_intro_end(segments), 15.5) + def test_organic_cold_open_is_not_mistaken_for_tts_boilerplate(self): # hpr0500: the host's own greeting happens to say "this is Hacker # Public Radio", but there is no actual TTS metadata announcement. diff --git a/tests/test_politeness.py b/tests/test_politeness.py new file mode 100644 index 0000000..58eb8f8 --- /dev/null +++ b/tests/test_politeness.py @@ -0,0 +1,28 @@ +import time +import unittest +from unittest import mock + +from hpr_stripper import config, fetch + + +class TestPolitenessGap(unittest.TestCase): + def test_enforces_minimum_gap_between_requests(self): + fetch._last_request_at = 0.0 + with mock.patch.object(config, "REQUEST_DELAY_SECONDS", 0.05): + start = time.monotonic() + fetch._wait_for_politeness_gap() + fetch._wait_for_politeness_gap() + elapsed = time.monotonic() - start + self.assertGreaterEqual(elapsed, 0.05) + + def test_no_wait_once_the_gap_has_already_elapsed(self): + with mock.patch.object(config, "REQUEST_DELAY_SECONDS", 0.01): + fetch._last_request_at = time.monotonic() - 10 + start = time.monotonic() + fetch._wait_for_politeness_gap() + elapsed = time.monotonic() - start + self.assertLess(elapsed, 0.01) + + +if __name__ == "__main__": + unittest.main()