Files
dsh 0824e0ef83 v1.2.0: 事件流对账、增量 update、ZIM 发布与表情渲染
从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
2026-11-18 11:41:25 +08:00

1401 lines
59 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""离线单元测试:不联网、不读真实归档。
跑法(在 tieba-archiver 目录下)::
C:\\Users\\peng0\\DSH\\tieba-archive\\.venv\\Scripts\\python -m unittest discover -s tests -v
覆盖点:
* Cookie 解析:整条请求头(RT 值里有分号)、JSON、cURL、Netscape、缺 BDUSS;
* 回复对象正则;
* schema 建库 + 旧库 ALTER 补列;
* importer:3 条楼层的小样本、幂等、rto 优先、正文昵称匹配、v1 记录兼容;
* export:thread → threads.jsonl;
* images:URL 还原、v2/v1 图片 URL 收集、index.jsonl → image 表;
* crawler:用假 client 跑枚举、抓帖、已删帖、连续空结果、页数上限、楼中楼补抓。
crawler 的测试用 ``FakeClient`` 代替 aiotieba,所以不需要网络;只有
``import_all`` / ``stats`` 会碰临时 SQLite 文件。
"""
from __future__ import annotations
import asyncio
import contextlib
import io
import json
import os
import shutil
import sqlite3
import sys
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
TMP_ROOT = ROOT / ".tmp-tests"
class _WorkspaceTempDir:
"""测试临时目录:``Path.mkdir`` + 自建文件名,不用 ``tempfile.mkdtemp``。
Windows 上 ``mkdtemp`` 会以 mode=0700 建目录,某些受限令牌/沙箱下
进程反而写不进去(PermissionError);普通 ``mkdir`` 继承父目录 ACL,
在仓库内和系统 TEMP 都能正常读写。
"""
def __init__(self, prefix: str = "tmp-"):
TMP_ROOT.mkdir(parents=True, exist_ok=True)
self.path = TMP_ROOT / f"{prefix}{os.getpid()}-{time.time_ns()}"
self.path.mkdir()
self.name = str(self.path)
def cleanup(self) -> None:
shutil.rmtree(self.path, ignore_errors=True)
def __enter__(self) -> str:
return str(self.path)
def __exit__(self, *exc_info) -> None:
self.cleanup()
def make_temp_dir():
return _WorkspaceTempDir()
from tieba_archiver import (crawler, db, events, export, images, importer, # noqa: E402
rebuild, replay, restore, update, verify, zim)
from tieba_archiver.config import Config # noqa: E402
from tieba_archiver.cookies import ( # noqa: E402
CookieError,
mask_value,
parse_cookie_file,
parse_cookie_text,
save_cookie_json,
split_header_segments,
validate_cookies,
)
# ================================================================ Cookie
class CookieTests(unittest.TestCase):
HEADER = (
"BAIDUID=740939311064F9A9B7DA652338D7C8CF:FG=1;"
"BDUSS_BFESS=bfess-value;"
"BAIDUID_BFESS=740939311064F9A9B7DA652338D7C8CF:FG=1;"
"BDUSS=bduss-value;"
"STOKEN=stoken-value;"
'RT="z=1&dm=baidu.com&si=abc;co=def;st=ghi";'
"XFT=xft-value"
)
def test_header_with_quoted_semicolons(self):
jar = parse_cookie_text(self.HEADER)
self.assertEqual(
set(jar),
{"BAIDUID", "BDUSS_BFESS", "BAIDUID_BFESS", "BDUSS", "STOKEN", "RT", "XFT"},
)
self.assertEqual(jar["BDUSS"], "bduss-value")
# RT 里面的分号不能被切开,否则后半段会变成独立键
self.assertIn("co=def", jar["RT"])
self.assertNotIn("co", jar)
self.assertTrue(jar["RT"].startswith('"') and jar["RT"].endswith('"'))
def test_header_with_spaces_and_newline(self):
jar = parse_cookie_text(" BDUSS=a ; STOKEN=b ;\n XFT=c ")
self.assertEqual(jar, {"BDUSS": "a", "STOKEN": "b", "XFT": "c"})
def test_outer_quotes_are_stripped(self):
jar = parse_cookie_text("'BDUSS=a;STOKEN=b;RT=\"x;y\"'")
self.assertEqual(jar["BDUSS"], "a")
self.assertEqual(jar["STOKEN"], "b")
self.assertIn("x;y", jar["RT"])
def test_json_object(self):
jar = parse_cookie_text('{"BDUSS": "x", "STOKEN": "y", "N": 3}')
self.assertEqual(jar, {"BDUSS": "x", "STOKEN": "y", "N": "3"})
def test_json_list_from_extension(self):
raw = '[{"name": "BDUSS", "value": "x"}, {"name": "STOKEN", "value": "y"}]'
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
def test_curl_header(self):
raw = (
"curl 'https://tieba.baidu.com/' "
"-H 'Cookie: BAIDUID=a; BDUSS=b; RT=\"x;y\"' "
"-H 'User-Agent: test'"
)
jar = parse_cookie_text(raw)
self.assertEqual(jar["BDUSS"], "b")
self.assertIn("x;y", jar["RT"])
def test_curl_b_flag(self):
raw = "curl -b 'BDUSS=x; STOKEN=y' https://tieba.baidu.com/"
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
def test_plain_cookie_header(self):
raw = "Cookie: BDUSS=x; STOKEN=y"
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
def test_netscape_file(self):
raw = (
"# Netscape HTTP Cookie File\n"
".baidu.com\tTRUE\t/\tFALSE\t0\tBDUSS\tx\n"
".baidu.com\tTRUE\t/\tFALSE\t0\tSTOKEN\ty\n"
)
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
def test_missing_bduss_raises(self):
with self.assertRaises(CookieError):
validate_cookies({"STOKEN": "y"})
warnings = validate_cookies({"BDUSS": "x"})
self.assertTrue(any("STOKEN" in warning for warning in warnings))
def test_mask_value(self):
masked = mask_value("0123456789abcdef")
self.assertIn("***", masked)
self.assertNotIn("4567", masked)
self.assertEqual(mask_value("短"), "******")
def test_save_cookie_json_roundtrip(self):
with make_temp_dir() as tmp:
path = Path(tmp) / "cookies.txt"
save_cookie_json(path, {"BDUSS": "x", "STOKEN": "y"})
self.assertEqual(json.loads(path.read_text(encoding="utf-8")), {"BDUSS": "x", "STOKEN": "y"})
self.assertEqual(parse_cookie_file(path), {"BDUSS": "x", "STOKEN": "y"})
def test_split_header_segments(self):
parts = split_header_segments('A=1; B="x;y"; C=3')
self.assertEqual([part.strip() for part in parts], ["A=1", 'B="x;y"', "C=3"])
# ================================================================ 回复正则
class ReplyRegexTests(unittest.TestCase):
def test_common_forms(self):
cases = {
"回复 张三 : 你好": "张三",
"回复 张三: 你好": "张三",
"回复张三:你好": "张三",
"回复 张三 : 你好": "张三",
"回复 张三:\n多行正文": "张三",
"回复 ABC : 全角": "ABC",
}
for text, expected in cases.items():
with self.subTest(text=text):
self.assertEqual(importer.parse_reply_name(text), expected)
def test_non_matches(self):
for text in ("", "你好", "回复:没有昵称", "=========="):
with self.subTest(text=text):
self.assertIsNone(importer.parse_reply_name(text))
def test_name_too_long(self):
self.assertIsNone(importer.parse_reply_name("回复 " + "一" * 31 + " : x"))
self.assertEqual(importer.parse_reply_name("回复 " + "一" * 30 + " : x"), "一" * 30)
# ================================================================ schema
class SchemaTests(unittest.TestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
def tearDown(self):
self.tmp.cleanup()
def test_columns_match_existing_archive(self):
conn = db.connect(self.cfg)
try:
db.init_db(conn)
post_cols = {row[1] for row in conn.execute("PRAGMA table_info(post)")}
# 新 schema 必须是旧库列的超集(旧库靠 ensure_compat 补新列)
self.assertLessEqual(
{
"pid", "tid", "floor", "author", "author_id", "content",
"content_html", "post_time", "comment_num", "imgs", "is_lz",
"fetched_at", "agree", "disagree", "sign", "frags",
},
post_cols,
)
self.assertLessEqual({"status", "recycled_at", "recycle_reason", "content_hash"}, post_cols)
comment_cols = {row[1] for row in conn.execute("PRAGMA table_info(comment)")}
self.assertLessEqual(
{
"cid", "pid", "tid", "author", "author_id", "content", "post_time",
"fetched_at", "floor", "agree", "disagree", "rto",
"reply_to_name", "reply_to_uid", "frags",
},
comment_cols,
)
self.assertLessEqual({"status", "recycled_at", "recycle_reason"}, comment_cols)
thread_cols = {row[1] for row in conn.execute("PRAGMA table_info(thread)")}
self.assertLessEqual(
{
"tid", "fid", "title", "author", "author_id", "reply_num",
"is_good", "is_top", "pages", "pages_done", "status", "err",
"updated_at",
},
thread_cols,
)
self.assertIn("deleted_at", thread_cols)
self.assertEqual(
{row[1] for row in conn.execute("PRAGMA table_info(user)")},
{
"uid", "uname", "nick", "portrait", "level", "glevel", "gender",
"ip", "bawu", "vip", "icons", "updated_at",
},
)
self.assertEqual(
{row[1] for row in conn.execute("PRAGMA table_info(image)")},
{
"url", "tid", "pid", "md5", "size", "path", "status", "tries",
"code", "err", "fetched_at",
},
)
finally:
conn.close()
def test_init_is_idempotent(self):
conn = db.connect(self.cfg)
try:
db.init_db(conn)
db.init_db(conn)
tables = {
row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
}
self.assertLessEqual(
{"thread", "post", "comment", "image", "user", "event", "batch",
"thread_seen", "post_stat_history", "verify_suspect", "thread_suspect"},
tables,
)
# 反复 init 不应该重复建表
before = len(tables)
db.init_db(conn)
after = len({row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")})
self.assertEqual(before, after)
finally:
conn.close()
def test_ensure_compat_upgrades_v1_db(self):
# 手工造一个 v1 库:没有 v2 列,也没有 user 表
conn = sqlite3.connect(str(self.cfg.db_path))
conn.executescript(
"""
CREATE TABLE post(
pid INTEGER PRIMARY KEY, tid INTEGER NOT NULL, floor INTEGER,
author TEXT, author_id TEXT, content TEXT, content_html TEXT,
post_time TEXT, comment_num INTEGER DEFAULT 0, imgs TEXT,
is_lz INTEGER DEFAULT 0, fetched_at TEXT);
CREATE TABLE comment(
cid INTEGER PRIMARY KEY, pid INTEGER, tid INTEGER, author TEXT,
author_id TEXT, content TEXT, post_time TEXT, fetched_at TEXT);
INSERT INTO post(pid,tid,content) VALUES(1, 2, '旧数据');
"""
)
conn.commit()
conn.close()
conn = db.connect(self.cfg)
try:
db.ensure_compat(conn)
db.ensure_compat(conn) # 幂等
self.assertTrue({"agree", "disagree", "sign", "frags"} <= db.table_columns(conn, "post"))
self.assertTrue(
{"floor", "agree", "disagree", "rto", "reply_to_name", "reply_to_uid", "frags"}
<= db.table_columns(conn, "comment")
)
self.assertIn("user", {row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'")})
self.assertEqual(conn.execute("SELECT content FROM post WHERE pid=1").fetchone()[0], "旧数据")
finally:
conn.close()
# ================================================================ config
class ConfigTests(unittest.TestCase):
def test_defaults_and_validation(self):
with make_temp_dir() as tmp:
cfg = Config(kw="东方年代记", data_dir=tmp)
self.assertEqual(cfg.rate, 1.3)
self.assertEqual(cfg.page_cap, 400)
self.assertEqual(cfg.db_path, Path(tmp) / "tieba.db")
self.assertEqual(cfg.comments_path, Path(tmp) / "raw" / "comments.jsonl")
with self.assertRaises(ValueError):
Config(kw="x", data_dir=tmp, rate=0)
with self.assertRaises(ValueError):
Config(kw="", data_dir=tmp)
with self.assertRaises(ValueError):
Config(kw="x", data_dir=tmp, comment_rn=51)
def test_save_load_roundtrip(self):
with make_temp_dir() as tmp:
cfg = Config(kw="东方年代记", data_dir=tmp, rate=2.5, workers=3)
cfg.fid = 2288117
cfg.forum_name = "东方年代记"
path = cfg.save()
self.assertTrue(path.exists())
self.assertNotIn("data_dir", json.loads(path.read_text(encoding="utf-8")))
loaded = Config.load(tmp)
self.assertEqual(loaded.kw, "东方年代记")
self.assertEqual(loaded.rate, 2.5)
self.assertEqual(loaded.workers, 3)
self.assertEqual(loaded.fid, 2288117)
def test_load_missing_uses_defaults(self):
with make_temp_dir() as tmp:
cfg = Config.load(tmp, kw="临时吧")
self.assertEqual(cfg.kw, "临时吧")
self.assertEqual(cfg.rate, 1.3)
def test_coerce_string_numbers(self):
with make_temp_dir() as tmp:
(Path(tmp) / "config.json").write_text(
json.dumps({"kw": "x", "rate": "0.5", "workers": "4"}), encoding="utf-8"
)
cfg = Config.load(tmp)
self.assertEqual(cfg.rate, 0.5)
self.assertEqual(cfg.workers, 4)
def test_bad_config(self):
with make_temp_dir() as tmp:
(Path(tmp) / "config.json").write_text("{not json", encoding="utf-8")
with self.assertRaises(ValueError):
Config.load(tmp)
# ================================================================ fixture / importer
def write_fixture(cfg: Config, *, v1: bool = False) -> None:
"""3 条楼层 + 3 条楼中楼 + 2 个主题的小样本。"""
cfg.ensure_dirs()
if v1:
threads = [
{"tid": 1001, "title": "v1 帖", "reply_num": 1, "is_good": 0, "is_top": 0}
]
posts = [
{
"tid": 1001, "pid": 5001, "floor": 1, "author": "旧楼主",
"author_id": 11, "text": "v1 正文", "time": 1700000000,
"reply_num": 0, "imgs": ["https://tiebapic.baidu.com/forum/pic/item/old.jpg"],
}
]
comments = [
{
"cid": 9001, "pid": 5001, "tid": 1001, "author": "旧路人",
"author_id": 22, "text": "v1 楼中楼", "time": 1700000100,
}
]
users = []
else:
threads = [
{
"tid": 1001, "fid": 2288117, "title": "测试帖 1", "author": "楼主甲",
"author_id": 11, "reply_num": 2, "is_good": 0, "is_top": 0,
},
{
"tid": 1002, "fid": 2288117, "title": "测试帖 2", "author": "路人乙",
"author_id": 22, "reply_num": 9, "is_good": 0, "is_top": 0,
},
]
posts = [
{
"tid": 1001, "pid": 5001, "floor": 1, "uid": 11, "uname": "楼主甲",
"agree": 3, "disagree": 0, "sign": "小尾巴", "rn": 2,
"ts": 1700000000, "lz": True, "text": "第一楼",
"frags": [{"t": "text", "text": "第一楼"}],
},
{
"tid": 1001, "pid": 5002, "floor": 2, "uid": 22, "uname": "路人乙",
"agree": 0, "disagree": 0, "sign": "", "rn": 0,
"ts": 1700000100, "lz": False, "text": "第二楼带图",
"frags": [
{"t": "text", "text": "第二楼带图"},
{"t": "img", "src": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg"},
],
},
{
"tid": 1001, "pid": 5003, "floor": 3, "uid": 11, "uname": "楼主甲",
"agree": 1, "disagree": 0, "sign": "", "rn": 1,
"ts": 1700000200, "lz": True, "text": "第三楼",
"frags": [{"t": "text", "text": "第三楼"}],
},
]
comments = [
{
"tid": 1001, "pid": 9001, "ppid": 5001, "floor": 1, "uid": 22,
"uname": "路人乙", "rto": 11, "agree": 1, "disagree": 0,
"ts": 1700000300, "lz": False, "text": "直接回复楼主",
"frags": [{"t": "text", "text": "直接回复楼主"}],
},
{
"tid": 1001, "pid": 9002, "ppid": 5003, "floor": 3, "uid": 33,
"uname": "路人丙", "rto": 0, "agree": 0, "disagree": 0,
"ts": 1700000400, "lz": False, "text": "回复 甲甲 : 正文匹配昵称",
"frags": [{"t": "text", "text": "回复 甲甲 : 正文匹配昵称"}],
},
{
"tid": 1001, "pid": 9003, "ppid": 5003, "floor": 3, "uid": 33,
"uname": "路人丙", "rto": 0, "agree": 0, "disagree": 0,
"ts": 1700000500, "lz": False, "text": "回复 查无此人 : 匹配不到",
"frags": [{"t": "text", "text": "回复 查无此人 : 匹配不到"}],
},
]
users = [
{
"uid": 11, "uname": "楼主甲", "nick": "甲甲", "portrait": "tb.1.aaa",
"level": 10, "glevel": 5, "gender": 1, "ip": "上海",
"bawu": False, "vip": True, "icons": ["vip"],
},
{
"uid": 22, "uname": "路人乙", "nick": "乙乙", "portrait": "",
"level": 3, "glevel": 2, "gender": 0, "ip": "江苏",
"bawu": False, "vip": False, "icons": [],
},
]
for path, records in (
(cfg.threads_path, threads),
(cfg.posts_path, posts),
(cfg.comments_path, comments),
(cfg.users_path, users),
):
with open(path, "w", encoding="utf-8", newline="\n") as fh:
for record in records:
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
cfg.done_path.write_text("1001\n", encoding="utf-8")
with open(cfg.image_index_path, "w", encoding="utf-8", newline="\n") as fh:
fh.write(
json.dumps(
{
"md5": "0" * 32,
"url": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg",
"size": 3,
"path": "imgs/00/00/" + "0" * 32 + ".jpg",
},
ensure_ascii=False,
)
+ "\n"
)
class ImporterTests(unittest.TestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
write_fixture(self.cfg)
with contextlib.redirect_stdout(io.StringIO()):
self.first = importer.import_all(self.cfg, quiet=True)
self.second = importer.import_all(self.cfg, quiet=True)
def tearDown(self):
self.tmp.cleanup()
def _rows(self, sql, params=()):
conn = db.connect(self.cfg, readonly=True)
try:
return conn.execute(sql, params).fetchall()
finally:
conn.close()
def test_counts(self):
self.assertEqual(self.first["threads"], 2)
self.assertEqual(self.first["posts"], 3)
self.assertEqual(self.first["comments"], 3)
self.assertEqual(self.first["users"], 2)
self.assertEqual(self.first["stats"]["posts"], 3)
self.assertEqual(self.first["stats"]["comments"], 3)
self.assertEqual(self.first["stats"]["posts_signed"], 1)
self.assertEqual(self.first["stats"]["posts_agreed"], 2)
self.assertEqual(self.first["stats"]["comments_rto"], 1)
self.assertEqual(self.first["stats"]["comments_named"], 2)
self.assertEqual(self.first["stats"]["comments_targeted"], 2)
def test_idempotent(self):
self.assertEqual(self.first["posts"], self.second["posts"])
self.assertEqual(self.first["comments"], self.second["comments"])
self.assertEqual(self.first["stats"], self.second["stats"])
def test_reply_targets(self):
rows = {
row["cid"]: (row["reply_to_name"], row["reply_to_uid"])
for row in self._rows("SELECT cid, reply_to_name, reply_to_uid FROM comment")
}
self.assertEqual(rows[9001], (None, 11)) # rto 优先
self.assertEqual(rows[9002], ("甲甲", 11)) # 正文昵称 → user.nick → 本帖作者
self.assertEqual(rows[9003], ("查无此人", None)) # 匹配不到就不猜
def test_thread_status_and_coverage(self):
rows = {
row["tid"]: (row["status"], row["pages"])
for row in self._rows("SELECT tid, status, pages FROM thread")
}
self.assertEqual(rows[1001][0], "done")
self.assertEqual(rows[1002][0], "new")
stats = self.second["stats"]
self.assertEqual(stats["threads_done"], 1)
self.assertEqual(stats["threads_with_posts"], 1)
self.assertAlmostEqual(db.coverage(stats), 50.0)
def test_frags_and_imgs_persisted(self):
row = self._rows("SELECT frags, imgs, sign, agree FROM post WHERE pid=5002")[0]
frags = json.loads(row["frags"])
self.assertEqual(frags[1]["t"], "img")
self.assertEqual(json.loads(row["imgs"]), [frags[1]["src"]])
sign = self._rows("SELECT sign FROM post WHERE pid=5001")[0]["sign"]
self.assertEqual(sign, "小尾巴")
def test_v1_records(self):
tmp = make_temp_dir()
try:
cfg = Config(kw="测试吧", data_dir=Path(tmp.name), rate=1000.0)
write_fixture(cfg, v1=True)
with contextlib.redirect_stdout(io.StringIO()):
importer.import_all(cfg, quiet=True)
conn = db.connect(cfg, readonly=True)
try:
post = conn.execute("SELECT content, author, author_id, imgs, frags FROM post").fetchone()
comment = conn.execute(
"SELECT pid, author, content, rto FROM comment"
).fetchone()
finally:
conn.close()
self.assertEqual(post["content"], "v1 正文")
self.assertEqual(post["author"], "旧楼主")
self.assertIn("old.jpg", post["imgs"])
self.assertEqual(json.loads(post["frags"])[0]["t"], "img")
self.assertEqual(comment["pid"], 5001) # v1 的 pid 是父楼层
self.assertEqual(comment["author"], "旧路人")
finally:
tmp.cleanup()
def test_missing_raw_is_ok(self):
tmp = make_temp_dir()
try:
cfg = Config(kw="空吧", data_dir=Path(tmp.name), rate=1000.0)
cfg.ensure_dirs()
with contextlib.redirect_stdout(io.StringIO()):
result = importer.import_all(cfg, quiet=True)
self.assertEqual(result["posts"], 0)
self.assertEqual(result["stats"]["threads"], 0)
finally:
tmp.cleanup()
class ExportTests(unittest.TestCase):
def test_export_threads(self):
with make_temp_dir() as tmp:
cfg = Config(kw="测试吧", data_dir=tmp, rate=1000.0)
write_fixture(cfg)
with contextlib.redirect_stdout(io.StringIO()):
importer.import_all(cfg, quiet=True)
cfg.threads_path.unlink() # 确认 export 是从库生成,而不是原样拷贝
count = export.export_threads(cfg, quiet=True)
self.assertEqual(count, 2)
lines = [json.loads(line) for line in cfg.threads_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual([row["tid"] for row in lines], [1001, 1002])
self.assertEqual(lines[0]["title"], "测试帖 1")
self.assertEqual(lines[0]["fid"], 2288117)
# ================================================================ images
class ImagesTests(unittest.TestCase):
def test_to_original(self):
compressed = (
"https://imgsrc.baidu.com/forum/w%3D720%3Bq%3D60%3Bg%3D0/"
"sign=abc123/1234567890.jpg"
)
self.assertEqual(
images.to_original(compressed),
"https://imgsrc.baidu.com/forum/pic/item/1234567890.jpg",
)
self.assertEqual(
images.to_original("https://tiebapic.baidu.com/forum/pic/item/abc.png"),
"https://imgsrc.baidu.com/forum/pic/item/abc.png",
)
c = "https://c.tieba.baidu.com/c/p/img?src=https%3A%2F%2Fexample.com%2Fa.jpg"
self.assertEqual(images.to_original(c), "https://example.com/a.jpg")
self.assertEqual(images.to_original("https://example.com/"), "https://example.com/")
def test_ext_for(self):
self.assertEqual(images.ext_for("image/jpeg", "http://x/a"), ".jpg")
self.assertEqual(images.ext_for("image/png; charset=x", "http://x/a"), ".png")
self.assertEqual(images.ext_for("", "http://x/a.webp"), ".webp")
self.assertEqual(images.ext_for("", "http://x/a"), ".jpg")
def test_collect_urls(self):
with make_temp_dir() as tmp:
cfg = Config(kw="x", data_dir=tmp, rate=1000.0)
cfg.ensure_dirs()
posts = [
{"frags": [{"t": "img", "src": "http://a/1.jpg"}, {"t": "text", "text": "x"}]},
{"frags": [{"t": "img", "src": "http://a/1.jpg"}]},
{"imgs": ["http://a/2.jpg"]},
]
with open(cfg.posts_path, "w", encoding="utf-8") as fh:
for post in posts:
fh.write(json.dumps(post) + "\n")
self.assertEqual(images.collect_urls(cfg), ["http://a/1.jpg", "http://a/2.jpg"])
def test_save_blob_buckets_by_md5(self):
with make_temp_dir() as tmp:
cfg = Config(kw="x", data_dir=tmp, rate=1000.0)
cfg.ensure_dirs()
md5 = "ab" + "0" * 30
path = images.save_blob(cfg, md5, b"hello", "image/png", "http://a/x")
self.assertEqual(path, f"imgs/ab/00/{md5}.png")
self.assertEqual((cfg.data_dir / path).read_bytes(), b"hello")
image = cfg.data_dir / path
image.write_bytes(b"hello") # 已存在时不重复写,也不报错
self.assertEqual(images.save_blob(cfg, md5, b"hello", "image/png", "http://a/x"), path)
def test_import_index(self):
with make_temp_dir() as tmp:
cfg = Config(kw="x", data_dir=tmp, rate=1000.0)
write_fixture(cfg)
with contextlib.redirect_stdout(io.StringIO()):
result = images.import_index(cfg, quiet=True)
again = images.import_index(cfg, quiet=True)
self.assertEqual(result["filled"], 1)
self.assertEqual(again["filled"], 1)
conn = db.connect(cfg, readonly=True)
try:
row = conn.execute(
"SELECT status, md5, size, path FROM image WHERE url=?",
("https://tiebapic.baidu.com/forum/pic/item/abc.jpg",),
).fetchone()
count = conn.execute("SELECT COUNT(*) FROM image").fetchone()[0]
finally:
conn.close()
self.assertEqual(row["status"], "done")
self.assertEqual(row["md5"], "0" * 32)
self.assertEqual(count, 1)
# ================================================================ crawler(假 client)
class FragText:
def __init__(self, text):
self.text = text
class FragImage:
def __init__(self, src):
self.src = src
self.big_src = ""
self.origin_src = ""
self.hash = ""
self.show_width = 100
self.show_height = 100
class Contents:
def __init__(self, *objs):
self.objs = list(objs)
def make_user(uid: int, name: str):
return SimpleNamespace(
user_id=uid, user_name=name, nick_name_new=name + "昵称", portrait="",
level=1, glevel=1, gender=0, ip="", is_bawu=False, is_vip=False, icons=[],
)
def make_post(pid, floor, uid, name, *, sign="", agree=0, reply_num=0, comments=None,
text="正文", image=None):
objs = [FragText(text)]
if image:
objs.append(FragImage(image))
return SimpleNamespace(
contents=Contents(*objs), sign=sign, comments=comments or [], is_aimeme=False,
fid=2288117, fname="测试吧", tid=2001, pid=pid, user=make_user(uid, name),
author_id=uid, floor=floor, reply_num=reply_num, agree=agree, disagree=0,
create_time=1700000000, is_thread_author=floor == 1,
)
def make_comment(cid, ppid, uid, name, *, rto=0, text="楼中楼", floor=1, agree=0):
return SimpleNamespace(
contents=Contents(FragText(text)), fid=2288117, fname="测试吧", tid=2001,
ppid=ppid, pid=cid, user=make_user(uid, name), reply_to_id=rto, floor=floor,
agree=agree, disagree=0, create_time=1700000100, is_thread_author=False,
)
class Page:
def __init__(self, has_more=False, total_page=1):
self.has_more = has_more
self.total_page = total_page
class Result:
def __init__(self, objs=None, *, err=None, has_more=False, total_page=1):
self.objs = objs or []
self.err = err
self.page = Page(has_more, total_page)
class FakeCrawlClient:
"""只实现 crawl_thread 用到的两个方法。"""
def __init__(self, post_pages, comment_pages=None):
self.post_pages = post_pages
self.comment_pages = comment_pages or {}
self.comment_calls = []
async def get_posts(self, tid, pn=1, rn=30, with_comments=True, comment_rn=50):
pages = self.post_pages.get(tid) or []
if pn <= len(pages):
return pages[pn - 1]
return pages[-1] if pages else Result([])
async def get_comments(self, tid, pid, pn=1):
self.comment_calls.append((tid, pid, pn))
pages = self.comment_pages.get(pid) or []
if pn <= len(pages):
return pages[pn - 1]
return pages[-1] if pages else Result([])
class FakeEnumClient:
def __init__(self, pages, threads_total=3):
self.pages = pages
self.threads_total = threads_total
self.forum = SimpleNamespace(
fid=2288117, fname="测试吧", thread_num=threads_total, member_num=1, post_num=1
)
async def get_forum(self, kw):
return self.forum
async def get_threads(self, kw, pn=1, rn=30):
if pn <= len(self.pages):
return self.pages[pn - 1]
return Result([])
class CrawlerTests(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(
kw="测试吧", data_dir=Path(self.tmp.name), rate=1_000_000.0,
tries=2, backoff=0.0, page_cap=400,
)
self.cfg.ensure_dirs()
crawler.set_rate(self.cfg.rate)
def tearDown(self):
self.tmp.cleanup()
def _jsonl(self, path):
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
async def test_enum_writes_threads_jsonl(self):
pages = [
Result(
[
SimpleNamespace(
tid=3001, title="主题 A", user=make_user(11, "甲"),
author_id=11, reply_num=5, is_good=True, is_top=False, fid=2288117,
),
SimpleNamespace(
tid=3002, title="主题 B", user=make_user(22, "乙"),
author_id=22, reply_num=0, is_good=False, is_top=False, fid=2288117,
),
],
has_more=True,
total_page=2,
),
Result(
[
SimpleNamespace(
tid=3003, title="主题 C", user=make_user(33, "丙"),
author_id=33, reply_num=1, is_good=False, is_top=False, fid=2288117,
)
],
has_more=False,
total_page=2,
),
]
client = FakeEnumClient(pages)
with contextlib.redirect_stdout(io.StringIO()):
result = await crawler.enumerate_with_client(client, self.cfg)
again = await crawler.enumerate_with_client(client, self.cfg)
self.assertEqual(result["added"], 3)
self.assertEqual(again["added"], 0)
rows = self._jsonl(self.cfg.threads_path)
self.assertEqual([row["tid"] for row in rows], [3001, 3002, 3003])
self.assertEqual(rows[0]["title"], "主题 A")
self.assertEqual(rows[0]["is_good"], 1)
self.assertEqual(json.loads(self.cfg.config_path.read_text(encoding="utf-8"))["fid"], 2288117)
async def test_crawl_thread_writes_records(self):
inline = [make_comment(9100, 5001, 33, "丙", rto=0, text="内联楼中楼")]
pages = {
2001: [
Result(
[
make_post(5001, 1, 11, "甲", sign="尾巴", agree=2, reply_num=1, comments=inline),
make_post(5002, 2, 22, "乙", text="带图", image="http://img/1.jpg"),
],
has_more=False,
)
]
}
client = FakeCrawlClient(pages)
writer = crawler.JsonlWriter(self.cfg)
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
lock = asyncio.Lock()
with contextlib.redirect_stdout(io.StringIO()):
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
writer.close()
posts = self._jsonl(self.cfg.posts_path)
comments = self._jsonl(self.cfg.comments_path)
users = self._jsonl(self.cfg.users_path)
self.assertEqual(len(posts), 2)
self.assertEqual(posts[0]["sign"], "尾巴")
self.assertEqual(posts[0]["agree"], 2)
self.assertEqual(posts[0]["text"], "正文")
self.assertEqual(posts[1]["frags"][1], {
"t": "img", "src": "http://img/1.jpg", "big": "", "origin": "",
"hash": "", "w": 100, "h": 100,
})
self.assertEqual(len(comments), 1)
self.assertEqual(comments[0]["ppid"], 5001)
self.assertEqual(comments[0]["pid"], 9100)
self.assertEqual(len(users), 3)
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
self.assertEqual(stats["threads"], 1)
async def test_deleted_thread_marks_done(self):
pages = {2001: [Result([], err=SimpleNamespace(code=4))]}
client = FakeCrawlClient(pages)
writer = crawler.JsonlWriter(self.cfg)
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
lock = asyncio.Lock()
with contextlib.redirect_stdout(io.StringIO()):
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
writer.close()
self.assertEqual(stats["deleted"], 1)
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
errors = self._jsonl(self.cfg.err_path)
self.assertEqual(errors[0]["code"], 4)
self.assertIn("deleted", errors[0]["err"])
async def test_empty_results_skip_without_done(self):
# 带错误的空结果(比如 429)才该重试/跳过,不能当成「翻到底了」
pages = {2001: [Result([], err=SimpleNamespace(code=429))]}
client = FakeCrawlClient(pages)
writer = crawler.JsonlWriter(self.cfg)
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
lock = asyncio.Lock()
with contextlib.redirect_stdout(io.StringIO()):
with self.assertRaises(crawler.CrawlError):
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
writer.close()
self.assertFalse(self.cfg.done_path.exists() and self.cfg.done_path.read_text().strip())
self.assertEqual(stats["threads"], 0)
async def test_empty_last_page_ends_thread(self):
# 真实遇到的情况:末页不足 30 楼,第 2 页返回空、err=None、has_more=False,
# 这帖其实已经抓完了,应该写 done 而不是当成 429 无限重试
pages = {
2001: [
Result([make_post(5001, 1, 11, "甲")], has_more=True),
Result([], has_more=False),
]
}
client = FakeCrawlClient(pages)
writer = crawler.JsonlWriter(self.cfg)
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
lock = asyncio.Lock()
with contextlib.redirect_stdout(io.StringIO()):
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
writer.close()
self.assertEqual(len(self._jsonl(self.cfg.posts_path)), 1)
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
self.assertEqual(stats["threads"], 1)
async def test_page_cap_stops_without_done(self):
self.cfg.page_cap = 1
pages = {
2001: [
Result([make_post(5001, 1, 11, "甲")], has_more=True),
Result([make_post(5002, 2, 11, "甲")], has_more=True),
]
}
client = FakeCrawlClient(pages)
writer = crawler.JsonlWriter(self.cfg)
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
lock = asyncio.Lock()
with contextlib.redirect_stdout(io.StringIO()):
with self.assertRaises(crawler.CrawlError) as ctx:
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
writer.close()
self.assertIn("页数上限", str(ctx.exception))
self.assertFalse(self.cfg.done_path.exists() and self.cfg.done_path.read_text().strip())
async def test_overflow_comments_are_fetched(self):
pages = {
2001: [
Result(
[
make_post(
5001, 1, 11, "甲",
comments=[make_comment(9100, 5001, 22, "乙", text="内联")],
reply_num=3,
)
],
has_more=False,
)
]
}
comments = {
5001: [
Result(
[
make_comment(9101, 5001, 22, "乙", text="补 1", agree=2),
make_comment(9102, 5001, 22, "乙", text="补 2"),
],
has_more=False,
)
]
}
client = FakeCrawlClient(pages, comments)
writer = crawler.JsonlWriter(self.cfg)
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
lock = asyncio.Lock()
with contextlib.redirect_stdout(io.StringIO()):
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
writer.close()
self.assertEqual(stats["overflow"], 1)
self.assertEqual(len(self._jsonl(self.cfg.comments_path)), 3)
self.assertEqual(client.comment_calls, [(2001, 5001, 1)])
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
def test_frags_conversion(self):
contents = Contents(
FragText("文字"),
FragImage("http://img/x.jpg"),
)
frags = crawler.frags_of(contents)
self.assertEqual(frags[0], {"t": "text", "text": "文字"})
self.assertEqual(frags[1]["t"], "img")
self.assertEqual(crawler.text_of(frags), "文字")
if __name__ == "__main__":
unittest.main()
# ================================================================ 事件流 / verify
class FakeVerifyClient:
"""只实现 verify_thread 用到的 get_posts。"""
def __init__(self, pages_by_tid):
self.pages_by_tid = pages_by_tid
async def get_posts(self, tid, pn=1, rn=30, with_comments=True, comment_rn=50):
pages = self.pages_by_tid.get(tid) or []
if pn <= len(pages):
return pages[pn - 1]
return Result([], has_more=False)
class VerifyTests(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1_000_000.0,
tries=2, backoff=0.0, page_cap=50)
self.cfg.ensure_dirs()
crawler.set_rate(self.cfg.rate)
self.conn = db.connect(self.cfg)
db.init_db(self.conn)
db.ensure_compat(self.conn)
self.conn.execute(
"INSERT INTO thread(tid, fid, title, reply_num, status) VALUES(1, 2, '帖', 3, 'done')"
)
self.conn.commit()
self.batch = events.start_batch(self.conn, "verify")
self.log = events.EventLog(self.conn, self.cfg, self.batch, "verify")
self.stats = {"ok": 0, "added": 0, "edited": 0, "deleted": 0, "suspect": 0,
"thread_deleted": 0, "thread_suspect": 0, "stat_samples": 0, "errors": 0}
def tearDown(self):
self.log.close()
self.conn.close()
self.tmp.cleanup()
def _seed(self, pid, floor, text, agree=0):
imgs = "[]"
self.conn.execute(
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, "
"agree, disagree, status, content_hash) VALUES(?,?,?,?,?,?,?,?,?, 'live', ?)",
(pid, 1, floor, "甲", "11", text, imgs, agree, 0,
events.content_hash(text, imgs, "甲")),
)
self.conn.commit()
async def _run(self, pages):
client = FakeVerifyClient({1: pages})
await verify.verify_thread(client, self.cfg, self.conn, self.log, 1, self.stats)
self.conn.commit()
async def test_add_new_post(self):
self._seed(100, 1, "旧楼")
old = make_post(100, 1, 11, "甲", text="旧楼")
post = make_post(101, 2, 11, "乙", text="新楼")
await self._run([Result([old, post], has_more=False)])
self.assertEqual(self.stats["added"], 1)
kind, = [r[0] for r in self.conn.execute("SELECT kind FROM event")]
self.assertEqual(kind, "post_add")
row = self.conn.execute("SELECT status FROM post WHERE pid=101").fetchone()
self.assertEqual(row[0], "live")
async def test_two_strike_delete(self):
self._seed(100, 1, "要被删的楼")
page = Result([], has_more=False) # 服务器这页什么都没有,但也没报错
# 第一次:只登记怀疑,不动数据
await self._run([Result([make_post(999, 1, 11, "别人")], has_more=False)])
self.assertEqual(self.stats["suspect"], 1)
status = self.conn.execute("SELECT status FROM post WHERE pid=100").fetchone()[0]
self.assertEqual(status, "live")
# 第二次:仍然缺 → 进回收站
await self._run([Result([make_post(999, 1, 11, "别人")], has_more=False)])
self.assertEqual(self.stats["deleted"], 1)
row = self.conn.execute("SELECT status, recycle_reason FROM post WHERE pid=100").fetchone()
self.assertEqual(tuple(row), ("recycled", "post_deleted"))
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event ORDER BY id")]
self.assertIn("post_del", kinds)
async def test_edit_detected(self):
self._seed(100, 1, "old text")
post = make_post(100, 1, 11, "甲", text="new text")
await self._run([Result([post], has_more=False)])
self.assertEqual(self.stats["edited"], 1)
content = self.conn.execute("SELECT content FROM post WHERE pid=100").fetchone()[0]
self.assertEqual(content, "new text")
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event")]
self.assertIn("post_edit", kinds)
async def test_thread_deleted_pushes_posts_to_bin(self):
self._seed(100, 1, "跟帖一起消失")
await self._run([Result([], err=SimpleNamespace(code=4))])
self.assertEqual(self.stats["thread_suspect"], 1)
await self._run([Result([], err=SimpleNamespace(code=4))])
self.assertEqual(self.stats["thread_deleted"], 1)
row = self.conn.execute("SELECT status, recycle_reason FROM post WHERE pid=100").fetchone()
self.assertEqual(tuple(row), ("recycled", "thread_deleted"))
self.assertIsNotNone(self.conn.execute("SELECT deleted_at FROM thread WHERE tid=1").fetchone()[0])
async def test_like_change_recorded(self):
self._seed(100, 1, "有赞的楼", agree=3)
post = make_post(100, 1, 11, "甲", text="有赞的楼")
post.agree = 5
await self._run([Result([post], has_more=False)])
self.assertEqual(self.stats["stat_samples"], 1)
row = self.conn.execute("SELECT agree FROM post WHERE pid=100").fetchone()
self.assertEqual(row[0], 5)
hist = self.conn.execute("SELECT agree FROM post_stat_history WHERE pid=100").fetchall()
self.assertEqual([h[0] for h in hist], [5])
async def test_comment_deleted_after_two_scans(self):
self._seed(100, 1, "有楼中楼")
self.conn.execute(
"INSERT INTO comment(cid, pid, tid, author, author_id, content, post_time, status) "
"VALUES(900, 100, 1, '乙', '22', '旧楼中楼', '2020-01-01 00:00:00', 'live')"
)
self.conn.commit()
post = make_post(100, 1, 11, "甲", text="有楼中楼")
post.reply_num = 0 # 内联 0 条 = 这一层的楼中楼被完整重取了
await self._run([Result([post], has_more=False)])
self.assertEqual(self.stats.get("comment_suspect", 0), 1)
await self._run([Result([post], has_more=False)])
self.assertEqual(self.stats.get("comment_deleted", 0), 1)
row = self.conn.execute("SELECT status FROM comment WHERE cid=900").fetchone()
self.assertEqual(row[0], "recycled")
# ================================================================ as-of 重放
class ReplayTests(unittest.TestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
self.cfg.ensure_dirs()
self.conn = db.connect(self.cfg)
db.init_db(self.conn)
db.ensure_compat(self.conn)
self.conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
"VALUES(1, 2, '帖', 3, 'done')")
self._seed(100, 1, "base")
self.conn.commit()
def tearDown(self):
self.conn.close()
self.tmp.cleanup()
def _seed(self, pid, floor, text):
self.conn.execute(
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
"VALUES(?,?,?,?,?,?, '[]', 'live')",
(pid, 1, floor, "甲", "11", text),
)
def _add_event(self, ts, kind, pid=None, payload=None, tid=1):
self.conn.execute(
"INSERT INTO event(batch, ts, kind, tid, pid, payload, source) VALUES(1,?,?,?,?,?,'verify')",
(ts, kind, tid, pid, json.dumps(payload, ensure_ascii=False) if payload else None),
)
ev = {"ts": ts, "kind": kind, "tid": tid, "pid": pid, "cid": None,
"payload": payload or {}}
events.apply_event(self.conn, ev)
self.conn.commit()
def test_new_post_rolls_back_before_add(self):
self._add_event("2026-11-17 10:00:00", "post_add", pid=101,
payload={"pid": 101, "tid": 1, "floor": 2, "author": "乙",
"content": "later", "status": "live"})
rows = replay.as_of_thread(self.conn, 1, "2026-11-16")
self.assertEqual([r["pid"] for r in rows], [100])
rows = replay.as_of_thread(self.conn, 1, "2026-11-17")
self.assertEqual([r["pid"] for r in rows], [100, 101])
def test_delete_rolls_back(self):
self._add_event("2026-11-18 10:00:00", "post_del", pid=100)
before = replay.as_of_thread(self.conn, 1, "2026-11-17")
self.assertEqual(before[0]["status"], "live")
after = replay.as_of_thread(self.conn, 1, "2026-11-18")
self.assertEqual(after[0]["status"], "recycled")
def test_edit_rolls_back_to_old_content(self):
self._add_event("2026-11-19 10:00:00", "post_edit", pid=100,
payload={"content": "new", "old_content": "base"})
before = replay.as_of_thread(self.conn, 1, "2026-11-18")
self.assertEqual(before[0]["content"], "base")
after = replay.as_of_thread(self.conn, 1, "2026-11-19")
self.assertEqual(after[0]["content"], "new")
def test_thread_deleted_rolls_back(self):
self._add_event("2026-11-20 10:00:00", "thread_deleted", tid=1)
before = replay.as_of_thread(self.conn, 1, "2026-11-19")
self.assertEqual(before[0]["status"], "live")
after = replay.as_of_thread(self.conn, 1, "2026-11-20")
self.assertEqual(after[0]["status"], "recycled")
# ================================================================ 增量 update
class FakeUpdateClient:
def __init__(self, thread_pages, post_pages):
self.thread_pages = thread_pages
self.post_pages = post_pages
async def get_threads(self, kw, pn=1, rn=30):
if pn <= len(self.thread_pages):
return self.thread_pages[pn - 1]
return Result([], has_more=False)
async def get_posts(self, tid, pn=1, rn=30, with_comments=True, comment_rn=50):
pages = self.post_pages.get(tid) or []
if pn <= len(pages):
return pages[pn - 1]
return Result([], has_more=False)
def _thread_obj(tid, reply_num=1, title="帖"):
return SimpleNamespace(tid=tid, title=title, user=make_user(11, "甲"), author_id=11,
reply_num=reply_num, is_good=False, is_top=False, fid=2)
class UpdateTests(unittest.TestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1_000_000.0,
tries=2, backoff=0.0, page_cap=50)
self.cfg.ensure_dirs()
crawler.set_rate(self.cfg.rate)
self.conn = db.connect(self.cfg)
db.init_db(self.conn)
db.ensure_compat(self.conn)
self.conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
"VALUES(1, 2, '老帖', 3, 'done')")
for pid, floor in ((100, 1), (101, 2)):
self.conn.execute(
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
"VALUES(?,?,?,?,?,?, '[]', 'live')",
(pid, 1, floor, "甲", "11", f"第{floor}楼"),
)
self.conn.execute("INSERT INTO thread_seen(tid, reply_num, max_floor, seen_at) "
"VALUES(1, 2, 2, '2026-11-16 00:00:00')")
self.conn.commit()
def tearDown(self):
self.conn.close()
self.tmp.cleanup()
def test_new_thread_and_tail_floor(self):
thread_pages = [Result([_thread_obj(1, reply_num=5), _thread_obj(2, reply_num=1)],
has_more=False)]
post_pages = {
1: [Result([make_post(100, 1, 11, "甲", text="第1楼"),
make_post(101, 2, 11, "甲", text="第2楼"),
make_post(102, 3, 11, "甲", text="第3楼",
comments=[make_comment(900, 102, 22, "乙",
text="回复 甲 : 你好")])],
has_more=False)],
2: [Result([make_post(200, 1, 11, "甲", text="新帖一楼")], has_more=False)],
}
client = FakeUpdateClient(thread_pages, post_pages)
stats = update.update(self.cfg, pages=1, client=client)
self.assertEqual(stats["new_threads"], 1)
self.assertEqual(stats["changed_threads"], 1)
self.assertEqual(stats["added"], 2)
self.assertEqual(stats["errors"], 0)
pids = {r[0] for r in self.conn.execute("SELECT pid FROM post")}
self.assertEqual(pids, {100, 101, 102, 200})
reply_num = self.conn.execute("SELECT reply_num FROM thread WHERE tid=1").fetchone()[0]
self.assertEqual(reply_num, 5)
max_floor = self.conn.execute("SELECT max_floor FROM thread_seen WHERE tid=1").fetchone()[0]
self.assertEqual(max_floor, 3)
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event ORDER BY id")]
self.assertEqual(kinds, ["post_add", "post_add", "comment_add"])
self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM comment").fetchone()[0], 1)
cname = self.conn.execute("SELECT reply_to_name FROM comment WHERE cid=900").fetchone()[0]
self.assertEqual(cname, "甲")
# ================================================================ restore / rebuild
class RestoreRebuildTests(unittest.TestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
self.cfg.ensure_dirs()
self.conn = db.connect(self.cfg)
db.init_db(self.conn)
db.ensure_compat(self.conn)
self.conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
"VALUES(1, 2, '帖', 1, 'done')")
self.conn.execute(
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
"VALUES(100, 1, 1, '甲', '11', '一楼', '[]', 'live')"
)
self.conn.commit()
def tearDown(self):
self.conn.close()
self.tmp.cleanup()
def _apply(self, kind, tid=None, pid=None):
batch = events.start_batch(self.conn, "manual")
log = events.EventLog(self.conn, self.cfg, batch, "manual")
ev = log.emit(kind, tid=tid, pid=pid, source="manual")
events.apply_event(self.conn, ev)
log.close()
self.conn.commit()
def test_restore_thread(self):
self._apply("thread_deleted", tid=1)
self.assertEqual(
self.conn.execute("SELECT status FROM post WHERE pid=100").fetchone()[0],
"recycled",
)
stats = restore.restore(self.cfg, tid=1)
self.assertEqual(stats["threads"], 1)
self.assertEqual(stats["posts"], 1)
self.assertEqual(
self.conn.execute("SELECT status FROM post WHERE pid=100").fetchone()[0],
"live",
)
self.assertIsNone(
self.conn.execute("SELECT deleted_at FROM thread WHERE tid=1").fetchone()[0]
)
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event ORDER BY id")]
self.assertIn("thread_restored", kinds)
def test_rebuild_clean(self):
self._apply("post_del", tid=1, pid=100)
stats = rebuild.check(self.cfg)
self.assertTrue(stats["idempotent"])
self.assertEqual(stats["recycled_without_event"], 0)
self.assertEqual(stats["add_events_without_post"], 0)
def test_rebuild_flags_missing_event(self):
# 手工把楼标成回收站但不发事件:rebuild 应该能发现
self.conn.execute("UPDATE post SET status='recycled' WHERE pid=100")
self.conn.commit()
stats = rebuild.check(self.cfg)
self.assertEqual(stats["recycled_without_event"], 1)
# ================================================================ ZIM
def _has_libzim() -> bool:
try:
import libzim # noqa: F401
return True
except ImportError:
return False
@unittest.skipUnless(_has_libzim(), "没装 libzim,跳过 ZIM 测试")
class ZimTests(unittest.TestCase):
def setUp(self):
self.tmp = make_temp_dir()
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
self.cfg.ensure_dirs()
conn = db.connect(self.cfg)
db.init_db(conn)
db.ensure_compat(conn)
conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
"VALUES(1, 2, '东方年代记 攻略', 2, 'done')")
for pid, floor, text in ((100, 1, "开局选东方年代记"), (101, 2, "结局怎么打")):
conn.execute(
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
"VALUES(?,?,?,?,?,?, '[]', 'live')",
(pid, 1, floor, "甲", "11", text),
)
conn.execute("INSERT INTO comment(cid, pid, tid, author, content, status) "
"VALUES(900, 100, 1, '乙', '楼中楼说结局', 'live')")
conn.commit()
conn.close()
def tearDown(self):
self.tmp.cleanup()
def test_parse_bytes_and_split(self):
self.assertEqual(zim.parse_bytes("3.5G"), int(3.5 * 1024 ** 3))
self.assertEqual(zim.parse_bytes("100M"), 100 * 1024 ** 2)
conn = db.connect(self.cfg)
try:
conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
"VALUES(2, 2, '第二帖', 3, 'done')")
for pid, floor in ((200, 1), (201, 2), (202, 3)):
conn.execute("INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
"VALUES(?,?,?,?,?,?, '[]', 'live')", (pid, 2, floor, "乙", "22", "x"))
conn.commit()
parts = zim.plan_split(conn, 1000) # 每帖 2~3 楼 × 300B,1000B 只放得下一帖
self.assertEqual([len(part) for part in parts], [1, 1])
finally:
conn.close()
def test_build_zim_and_search(self):
out = Path(self.tmp.name) / "test.zim"
stats = zim.build_zim(self.cfg, out, with_images=False, title="测试 ZIM")
self.assertTrue(out.exists())
self.assertGreaterEqual(stats["pages"], 2)
from libzim.reader import Archive
from libzim.search import Query, Searcher
z = Archive(str(out))
self.assertEqual(z.main_entry.get_item().path, "index.html")
page = bytes(z.get_entry_by_path("thread-1.html").get_item().content).decode("utf-8")
self.assertIn("开局选东方年代记", page)
self.assertIn("楼中楼说结局", page)
s = Searcher(z).search(Query().set_query("结局"))
self.assertGreaterEqual(s.get_estimated_matches(), 1)