zhoujie
ea69788d0c
新增 scripts/http_utils.py 统一封装 HTTP 请求的限流重试(429/5xx 指数退避, 优先遵守服务端 Retry-After),search_arxiv/crossref/semantic_scholar.py 和 verify_citation.py 都已接入,不再各自裸调 urllib。 新增 scripts/search_openalex.py 作为第四个检索源:免费、无需 API key,覆盖面 比单独的 Crossref 更广,还能拿到开放获取PDF直链;已接入 literature_search.py 的主检索流程。verify_citation.py 的跨源标题核查同步改为同时查 Semantic Scholar 和 OpenAlex 两个独立源、任一命中相似度达标即通过,不再单点依赖 S2—— 这是针对"S2 被限流导致整批候选退化成 unverified"这个实际发生过的问题的直接 修复,已用真实网络请求验证:复测中 S2 确实当场返回了 429,靠 OpenAlex 兜底 最终判定仍然是 verified。 顺带修了 archive_references.py 的 slugify(),之前中文主题名会被正则全部 过滤掉、退化成通用的 "references",导致不同中文主题的归档目录互相冲突。 scripts/tests/ 下补了 37 个 unittest(全部 mock 网络请求,不发真实请求), 覆盖 verify_citation 的三档判定和双源核查合并逻辑、archive_references 的 bib 解析边界情况(嵌套花括号、中文主题名)、literature_search 的候选去重 合并与 BibTeX 生成、http_utils 的重试逻辑。用标准库 unittest 而不是 pytest, 和这些脚本本身不引入第三方依赖的原则保持一致。 CLAUDE.md 的下一步计划里,这一项已标记为完成。 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
104 lines
3.8 KiB
Python
104 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Unit tests for http_utils.fetch's retry/backoff behavior. All network calls
|
|
are mocked -- these never touch the real network.
|
|
|
|
Run: python -m unittest discover -s .claude/skills/literature-search-verify/scripts
|
|
"""
|
|
import os
|
|
import sys
|
|
import unittest
|
|
import urllib.error
|
|
import urllib.request
|
|
from email.message import Message
|
|
from unittest import mock
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
|
|
import http_utils
|
|
|
|
|
|
def _http_error(code, retry_after=None):
|
|
hdrs = Message()
|
|
if retry_after is not None:
|
|
hdrs["Retry-After"] = str(retry_after)
|
|
return urllib.error.HTTPError("http://example.com", code, "err", hdrs, None)
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, data):
|
|
self._data = data
|
|
|
|
def read(self):
|
|
return self._data
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc_info):
|
|
return False
|
|
|
|
|
|
class TestFetch(unittest.TestCase):
|
|
@mock.patch("urllib.request.urlopen")
|
|
@mock.patch("http_utils.time.sleep", return_value=None)
|
|
def test_succeeds_first_try_without_sleeping(self, mock_sleep, mock_urlopen):
|
|
mock_urlopen.return_value = FakeResponse(b"ok")
|
|
result = http_utils.fetch("http://example.com")
|
|
self.assertEqual(result, b"ok")
|
|
mock_sleep.assert_not_called()
|
|
|
|
@mock.patch("urllib.request.urlopen")
|
|
@mock.patch("http_utils.time.sleep", return_value=None)
|
|
def test_retries_on_429_then_succeeds(self, mock_sleep, mock_urlopen):
|
|
mock_urlopen.side_effect = [_http_error(429), FakeResponse(b"ok")]
|
|
result = http_utils.fetch("http://example.com", max_retries=3)
|
|
self.assertEqual(result, b"ok")
|
|
self.assertEqual(mock_urlopen.call_count, 2)
|
|
mock_sleep.assert_called_once()
|
|
|
|
@mock.patch("urllib.request.urlopen")
|
|
@mock.patch("http_utils.time.sleep", return_value=None)
|
|
def test_retries_on_5xx(self, mock_sleep, mock_urlopen):
|
|
mock_urlopen.side_effect = [_http_error(503), FakeResponse(b"ok")]
|
|
result = http_utils.fetch("http://example.com", max_retries=3)
|
|
self.assertEqual(result, b"ok")
|
|
|
|
@mock.patch("urllib.request.urlopen")
|
|
@mock.patch("http_utils.time.sleep", return_value=None)
|
|
def test_raises_after_exhausting_retries(self, mock_sleep, mock_urlopen):
|
|
mock_urlopen.side_effect = [_http_error(429)] * 10
|
|
with self.assertRaises(urllib.error.HTTPError):
|
|
http_utils.fetch("http://example.com", max_retries=2)
|
|
self.assertEqual(mock_urlopen.call_count, 3) # initial attempt + 2 retries
|
|
|
|
@mock.patch("urllib.request.urlopen")
|
|
@mock.patch("http_utils.time.sleep", return_value=None)
|
|
def test_non_retryable_status_raises_immediately(self, mock_sleep, mock_urlopen):
|
|
mock_urlopen.side_effect = _http_error(404)
|
|
with self.assertRaises(urllib.error.HTTPError):
|
|
http_utils.fetch("http://example.com")
|
|
self.assertEqual(mock_urlopen.call_count, 1)
|
|
mock_sleep.assert_not_called()
|
|
|
|
|
|
class TestWaitSeconds(unittest.TestCase):
|
|
def test_honors_retry_after_header(self):
|
|
err = _http_error(429, retry_after=5)
|
|
self.assertEqual(http_utils._wait_seconds(err, attempt=0), 5.0)
|
|
|
|
def test_falls_back_to_exponential_backoff_without_header(self):
|
|
err = _http_error(429)
|
|
self.assertAlmostEqual(http_utils._wait_seconds(err, attempt=2, base=1.0), 4.0)
|
|
|
|
def test_caps_wait_time(self):
|
|
err = _http_error(429, retry_after=999)
|
|
self.assertEqual(http_utils._wait_seconds(err, attempt=0, cap=20.0), 20.0)
|
|
|
|
def test_ignores_unparseable_retry_after(self):
|
|
err = _http_error(429, retry_after="not-a-number")
|
|
self.assertAlmostEqual(http_utils._wait_seconds(err, attempt=0, base=1.0), 1.0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|