Found via a real user bug report: a first detect run on an episode fell through to a worse, flagged signal; a forced re-run with no code or data changes resolved cleanly. The only explanation is a transient fetch failure on the first run -- _fetch_bytes caught URLError/HTTPError/ TimeoutError identically and returned None for all of them, so a dropped connection or a momentary timeout was silently treated the same as the content genuinely not existing. That false conclusion then got written to the ledger and stayed there, since cli.py's ledger.is_done() check treats any prior result (flagged or not) as done until --force. Now retries a couple of times (short delay between attempts) on URLError/TimeoutError, which mean the request never got a real answer from the server at all. HTTPError is never retried -- the server did answer, so that's real signal (403/404/etc), not noise.
72 lines
3.2 KiB
Python
72 lines
3.2 KiB
Python
import unittest
|
|
import urllib.error
|
|
from unittest import mock
|
|
|
|
from hpr_stripper import fetch
|
|
|
|
|
|
class TestErrorPageDetection(unittest.TestCase):
|
|
def test_rejects_html_404_page(self):
|
|
body = b"<html>\n<head><title>404 Not Found</title></head>\n<body>...</body></html>"
|
|
self.assertTrue(fetch._looks_like_error_page(body))
|
|
|
|
def test_rejects_s3_access_denied_xml(self):
|
|
body = (
|
|
b'<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code>'
|
|
b'<Message></Message><BucketName>hpr</BucketName></Error>'
|
|
)
|
|
self.assertTrue(fetch._looks_like_error_page(body))
|
|
|
|
def test_accepts_real_srt_content(self):
|
|
body = b"1\n00:00:00,000 --> 00:00:05,000\nThis is real subtitle text.\n"
|
|
self.assertFalse(fetch._looks_like_error_page(body))
|
|
|
|
def test_accepts_binary_audio_looking_data(self):
|
|
# A real audio file's first bytes are arbitrary binary, not markup.
|
|
body = bytes([0xFF, 0xFB, 0x90, 0x00]) + b"\x00" * 100
|
|
self.assertFalse(fetch._looks_like_error_page(body))
|
|
|
|
|
|
class TestFetchBytesTransientRetry(unittest.TestCase):
|
|
"""Regression test for a real bug report: a user's first `detect` on
|
|
an episode fell through to a worse signal and got flagged; a forced
|
|
re-run with no code or data changes resolved cleanly. The only
|
|
explanation is a transient fetch failure (a dropped connection, a
|
|
timeout) on the first run that _fetch_bytes used to treat identically
|
|
to a definitive 404 -- silently, with no retry -- and that bad
|
|
conclusion then got permanently written to the ledger."""
|
|
|
|
def test_retries_on_transient_url_error_then_succeeds(self):
|
|
mock_resp = mock.MagicMock()
|
|
mock_resp.read.return_value = b"x" * 100
|
|
mock_resp.__enter__.return_value = mock_resp
|
|
with mock.patch("hpr_stripper.fetch._wait_for_politeness_gap"), \
|
|
mock.patch("time.sleep"), \
|
|
mock.patch("urllib.request.urlopen",
|
|
side_effect=[urllib.error.URLError("connection reset"), mock_resp]) as urlopen_mock:
|
|
result = fetch._fetch_bytes("http://example.test/x")
|
|
self.assertEqual(result, b"x" * 100)
|
|
self.assertEqual(urlopen_mock.call_count, 2)
|
|
|
|
def test_gives_up_after_exhausting_retries(self):
|
|
with mock.patch("hpr_stripper.fetch._wait_for_politeness_gap"), \
|
|
mock.patch("time.sleep"), \
|
|
mock.patch("urllib.request.urlopen", side_effect=TimeoutError("timed out")) as urlopen_mock:
|
|
result = fetch._fetch_bytes("http://example.test/x")
|
|
self.assertIsNone(result)
|
|
self.assertEqual(urlopen_mock.call_count, fetch.TRANSIENT_FETCH_RETRIES + 1)
|
|
|
|
def test_definitive_http_error_is_never_retried(self):
|
|
with mock.patch("hpr_stripper.fetch._wait_for_politeness_gap"), \
|
|
mock.patch("time.sleep") as sleep_mock, \
|
|
mock.patch("urllib.request.urlopen",
|
|
side_effect=urllib.error.HTTPError("http://x", 404, "Not Found", {}, None)) as urlopen_mock:
|
|
result = fetch._fetch_bytes("http://example.test/x")
|
|
self.assertIsNone(result)
|
|
self.assertEqual(urlopen_mock.call_count, 1)
|
|
sleep_mock.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|