Files
Lee Hanken a3844addc5 Retry transient network failures instead of treating them as 404
Found via a real user bug report: a first detect run on an episode fell
through to a worse, flagged signal; a forced re-run with no code or data
changes resolved cleanly. The only explanation is a transient fetch
failure on the first run -- _fetch_bytes caught URLError/HTTPError/
TimeoutError identically and returned None for all of them, so a dropped
connection or a momentary timeout was silently treated the same as the
content genuinely not existing. That false conclusion then got written
to the ledger and stayed there, since cli.py's ledger.is_done() check
treats any prior result (flagged or not) as done until --force.

Now retries a couple of times (short delay between attempts) on
URLError/TimeoutError, which mean the request never got a real answer
from the server at all. HTTPError is never retried -- the server did
answer, so that's real signal (403/404/etc), not noise.
2026-11-22 15:45:19 +00:00

72 lines
3.2 KiB
Python

import unittest
import urllib.error
from unittest import mock
from hpr_stripper import fetch
class TestErrorPageDetection(unittest.TestCase):
def test_rejects_html_404_page(self):
body = b"<html>\n<head><title>404 Not Found</title></head>\n<body>...</body></html>"
self.assertTrue(fetch._looks_like_error_page(body))
def test_rejects_s3_access_denied_xml(self):
body = (
b'<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code>'
b'<Message></Message><BucketName>hpr</BucketName></Error>'
)
self.assertTrue(fetch._looks_like_error_page(body))
def test_accepts_real_srt_content(self):
body = b"1\n00:00:00,000 --> 00:00:05,000\nThis is real subtitle text.\n"
self.assertFalse(fetch._looks_like_error_page(body))
def test_accepts_binary_audio_looking_data(self):
# A real audio file's first bytes are arbitrary binary, not markup.
body = bytes([0xFF, 0xFB, 0x90, 0x00]) + b"\x00" * 100
self.assertFalse(fetch._looks_like_error_page(body))
class TestFetchBytesTransientRetry(unittest.TestCase):
"""Regression test for a real bug report: a user's first `detect` on
an episode fell through to a worse signal and got flagged; a forced
re-run with no code or data changes resolved cleanly. The only
explanation is a transient fetch failure (a dropped connection, a
timeout) on the first run that _fetch_bytes used to treat identically
to a definitive 404 -- silently, with no retry -- and that bad
conclusion then got permanently written to the ledger."""
def test_retries_on_transient_url_error_then_succeeds(self):
mock_resp = mock.MagicMock()
mock_resp.read.return_value = b"x" * 100
mock_resp.__enter__.return_value = mock_resp
with mock.patch("hpr_stripper.fetch._wait_for_politeness_gap"), \
mock.patch("time.sleep"), \
mock.patch("urllib.request.urlopen",
side_effect=[urllib.error.URLError("connection reset"), mock_resp]) as urlopen_mock:
result = fetch._fetch_bytes("http://example.test/x")
self.assertEqual(result, b"x" * 100)
self.assertEqual(urlopen_mock.call_count, 2)
def test_gives_up_after_exhausting_retries(self):
with mock.patch("hpr_stripper.fetch._wait_for_politeness_gap"), \
mock.patch("time.sleep"), \
mock.patch("urllib.request.urlopen", side_effect=TimeoutError("timed out")) as urlopen_mock:
result = fetch._fetch_bytes("http://example.test/x")
self.assertIsNone(result)
self.assertEqual(urlopen_mock.call_count, fetch.TRANSIENT_FETCH_RETRIES + 1)
def test_definitive_http_error_is_never_retried(self):
with mock.patch("hpr_stripper.fetch._wait_for_politeness_gap"), \
mock.patch("time.sleep") as sleep_mock, \
mock.patch("urllib.request.urlopen",
side_effect=urllib.error.HTTPError("http://x", 404, "Not Found", {}, None)) as urlopen_mock:
result = fetch._fetch_bytes("http://example.test/x")
self.assertIsNone(result)
self.assertEqual(urlopen_mock.call_count, 1)
sleep_mock.assert_not_called()
if __name__ == "__main__":
unittest.main()