从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
1401 lines
59 KiB
Python
1401 lines
59 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""离线单元测试:不联网、不读真实归档。
|
||
|
||
跑法(在 tieba-archiver 目录下)::
|
||
|
||
C:\\Users\\peng0\\DSH\\tieba-archive\\.venv\\Scripts\\python -m unittest discover -s tests -v
|
||
|
||
覆盖点:
|
||
|
||
* Cookie 解析:整条请求头(RT 值里有分号)、JSON、cURL、Netscape、缺 BDUSS;
|
||
* 回复对象正则;
|
||
* schema 建库 + 旧库 ALTER 补列;
|
||
* importer:3 条楼层的小样本、幂等、rto 优先、正文昵称匹配、v1 记录兼容;
|
||
* export:thread → threads.jsonl;
|
||
* images:URL 还原、v2/v1 图片 URL 收集、index.jsonl → image 表;
|
||
* crawler:用假 client 跑枚举、抓帖、已删帖、连续空结果、页数上限、楼中楼补抓。
|
||
|
||
crawler 的测试用 ``FakeClient`` 代替 aiotieba,所以不需要网络;只有
|
||
``import_all`` / ``stats`` 会碰临时 SQLite 文件。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import contextlib
|
||
import io
|
||
import json
|
||
import os
|
||
import shutil
|
||
import sqlite3
|
||
import sys
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
TMP_ROOT = ROOT / ".tmp-tests"
|
||
|
||
|
||
class _WorkspaceTempDir:
|
||
"""测试临时目录:``Path.mkdir`` + 自建文件名,不用 ``tempfile.mkdtemp``。
|
||
|
||
Windows 上 ``mkdtemp`` 会以 mode=0700 建目录,某些受限令牌/沙箱下
|
||
进程反而写不进去(PermissionError);普通 ``mkdir`` 继承父目录 ACL,
|
||
在仓库内和系统 TEMP 都能正常读写。
|
||
"""
|
||
|
||
def __init__(self, prefix: str = "tmp-"):
|
||
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
||
self.path = TMP_ROOT / f"{prefix}{os.getpid()}-{time.time_ns()}"
|
||
self.path.mkdir()
|
||
self.name = str(self.path)
|
||
|
||
def cleanup(self) -> None:
|
||
shutil.rmtree(self.path, ignore_errors=True)
|
||
|
||
def __enter__(self) -> str:
|
||
return str(self.path)
|
||
|
||
def __exit__(self, *exc_info) -> None:
|
||
self.cleanup()
|
||
|
||
|
||
def make_temp_dir():
|
||
return _WorkspaceTempDir()
|
||
|
||
|
||
from tieba_archiver import (crawler, db, events, export, images, importer, # noqa: E402
|
||
rebuild, replay, restore, update, verify, zim)
|
||
from tieba_archiver.config import Config # noqa: E402
|
||
from tieba_archiver.cookies import ( # noqa: E402
|
||
CookieError,
|
||
mask_value,
|
||
parse_cookie_file,
|
||
parse_cookie_text,
|
||
save_cookie_json,
|
||
split_header_segments,
|
||
validate_cookies,
|
||
)
|
||
|
||
|
||
# ================================================================ Cookie
|
||
class CookieTests(unittest.TestCase):
|
||
HEADER = (
|
||
"BAIDUID=740939311064F9A9B7DA652338D7C8CF:FG=1;"
|
||
"BDUSS_BFESS=bfess-value;"
|
||
"BAIDUID_BFESS=740939311064F9A9B7DA652338D7C8CF:FG=1;"
|
||
"BDUSS=bduss-value;"
|
||
"STOKEN=stoken-value;"
|
||
'RT="z=1&dm=baidu.com&si=abc;co=def;st=ghi";'
|
||
"XFT=xft-value"
|
||
)
|
||
|
||
def test_header_with_quoted_semicolons(self):
|
||
jar = parse_cookie_text(self.HEADER)
|
||
self.assertEqual(
|
||
set(jar),
|
||
{"BAIDUID", "BDUSS_BFESS", "BAIDUID_BFESS", "BDUSS", "STOKEN", "RT", "XFT"},
|
||
)
|
||
self.assertEqual(jar["BDUSS"], "bduss-value")
|
||
# RT 里面的分号不能被切开,否则后半段会变成独立键
|
||
self.assertIn("co=def", jar["RT"])
|
||
self.assertNotIn("co", jar)
|
||
self.assertTrue(jar["RT"].startswith('"') and jar["RT"].endswith('"'))
|
||
|
||
def test_header_with_spaces_and_newline(self):
|
||
jar = parse_cookie_text(" BDUSS=a ; STOKEN=b ;\n XFT=c ")
|
||
self.assertEqual(jar, {"BDUSS": "a", "STOKEN": "b", "XFT": "c"})
|
||
|
||
def test_outer_quotes_are_stripped(self):
|
||
jar = parse_cookie_text("'BDUSS=a;STOKEN=b;RT=\"x;y\"'")
|
||
self.assertEqual(jar["BDUSS"], "a")
|
||
self.assertEqual(jar["STOKEN"], "b")
|
||
self.assertIn("x;y", jar["RT"])
|
||
|
||
def test_json_object(self):
|
||
jar = parse_cookie_text('{"BDUSS": "x", "STOKEN": "y", "N": 3}')
|
||
self.assertEqual(jar, {"BDUSS": "x", "STOKEN": "y", "N": "3"})
|
||
|
||
def test_json_list_from_extension(self):
|
||
raw = '[{"name": "BDUSS", "value": "x"}, {"name": "STOKEN", "value": "y"}]'
|
||
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
|
||
|
||
def test_curl_header(self):
|
||
raw = (
|
||
"curl 'https://tieba.baidu.com/' "
|
||
"-H 'Cookie: BAIDUID=a; BDUSS=b; RT=\"x;y\"' "
|
||
"-H 'User-Agent: test'"
|
||
)
|
||
jar = parse_cookie_text(raw)
|
||
self.assertEqual(jar["BDUSS"], "b")
|
||
self.assertIn("x;y", jar["RT"])
|
||
|
||
def test_curl_b_flag(self):
|
||
raw = "curl -b 'BDUSS=x; STOKEN=y' https://tieba.baidu.com/"
|
||
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
|
||
|
||
def test_plain_cookie_header(self):
|
||
raw = "Cookie: BDUSS=x; STOKEN=y"
|
||
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
|
||
|
||
def test_netscape_file(self):
|
||
raw = (
|
||
"# Netscape HTTP Cookie File\n"
|
||
".baidu.com\tTRUE\t/\tFALSE\t0\tBDUSS\tx\n"
|
||
".baidu.com\tTRUE\t/\tFALSE\t0\tSTOKEN\ty\n"
|
||
)
|
||
self.assertEqual(parse_cookie_text(raw), {"BDUSS": "x", "STOKEN": "y"})
|
||
|
||
def test_missing_bduss_raises(self):
|
||
with self.assertRaises(CookieError):
|
||
validate_cookies({"STOKEN": "y"})
|
||
warnings = validate_cookies({"BDUSS": "x"})
|
||
self.assertTrue(any("STOKEN" in warning for warning in warnings))
|
||
|
||
def test_mask_value(self):
|
||
masked = mask_value("0123456789abcdef")
|
||
self.assertIn("***", masked)
|
||
self.assertNotIn("4567", masked)
|
||
self.assertEqual(mask_value("短"), "******")
|
||
|
||
def test_save_cookie_json_roundtrip(self):
|
||
with make_temp_dir() as tmp:
|
||
path = Path(tmp) / "cookies.txt"
|
||
save_cookie_json(path, {"BDUSS": "x", "STOKEN": "y"})
|
||
self.assertEqual(json.loads(path.read_text(encoding="utf-8")), {"BDUSS": "x", "STOKEN": "y"})
|
||
self.assertEqual(parse_cookie_file(path), {"BDUSS": "x", "STOKEN": "y"})
|
||
|
||
def test_split_header_segments(self):
|
||
parts = split_header_segments('A=1; B="x;y"; C=3')
|
||
self.assertEqual([part.strip() for part in parts], ["A=1", 'B="x;y"', "C=3"])
|
||
|
||
|
||
# ================================================================ 回复正则
|
||
class ReplyRegexTests(unittest.TestCase):
|
||
def test_common_forms(self):
|
||
cases = {
|
||
"回复 张三 : 你好": "张三",
|
||
"回复 张三: 你好": "张三",
|
||
"回复张三:你好": "张三",
|
||
"回复 张三 : 你好": "张三",
|
||
"回复 张三:\n多行正文": "张三",
|
||
"回复 ABC : 全角": "ABC",
|
||
}
|
||
for text, expected in cases.items():
|
||
with self.subTest(text=text):
|
||
self.assertEqual(importer.parse_reply_name(text), expected)
|
||
|
||
def test_non_matches(self):
|
||
for text in ("", "你好", "回复:没有昵称", "=========="):
|
||
with self.subTest(text=text):
|
||
self.assertIsNone(importer.parse_reply_name(text))
|
||
|
||
def test_name_too_long(self):
|
||
self.assertIsNone(importer.parse_reply_name("回复 " + "一" * 31 + " : x"))
|
||
self.assertEqual(importer.parse_reply_name("回复 " + "一" * 30 + " : x"), "一" * 30)
|
||
|
||
|
||
# ================================================================ schema
|
||
class SchemaTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def test_columns_match_existing_archive(self):
|
||
conn = db.connect(self.cfg)
|
||
try:
|
||
db.init_db(conn)
|
||
post_cols = {row[1] for row in conn.execute("PRAGMA table_info(post)")}
|
||
# 新 schema 必须是旧库列的超集(旧库靠 ensure_compat 补新列)
|
||
self.assertLessEqual(
|
||
{
|
||
"pid", "tid", "floor", "author", "author_id", "content",
|
||
"content_html", "post_time", "comment_num", "imgs", "is_lz",
|
||
"fetched_at", "agree", "disagree", "sign", "frags",
|
||
},
|
||
post_cols,
|
||
)
|
||
self.assertLessEqual({"status", "recycled_at", "recycle_reason", "content_hash"}, post_cols)
|
||
comment_cols = {row[1] for row in conn.execute("PRAGMA table_info(comment)")}
|
||
self.assertLessEqual(
|
||
{
|
||
"cid", "pid", "tid", "author", "author_id", "content", "post_time",
|
||
"fetched_at", "floor", "agree", "disagree", "rto",
|
||
"reply_to_name", "reply_to_uid", "frags",
|
||
},
|
||
comment_cols,
|
||
)
|
||
self.assertLessEqual({"status", "recycled_at", "recycle_reason"}, comment_cols)
|
||
thread_cols = {row[1] for row in conn.execute("PRAGMA table_info(thread)")}
|
||
self.assertLessEqual(
|
||
{
|
||
"tid", "fid", "title", "author", "author_id", "reply_num",
|
||
"is_good", "is_top", "pages", "pages_done", "status", "err",
|
||
"updated_at",
|
||
},
|
||
thread_cols,
|
||
)
|
||
self.assertIn("deleted_at", thread_cols)
|
||
self.assertEqual(
|
||
{row[1] for row in conn.execute("PRAGMA table_info(user)")},
|
||
{
|
||
"uid", "uname", "nick", "portrait", "level", "glevel", "gender",
|
||
"ip", "bawu", "vip", "icons", "updated_at",
|
||
},
|
||
)
|
||
self.assertEqual(
|
||
{row[1] for row in conn.execute("PRAGMA table_info(image)")},
|
||
{
|
||
"url", "tid", "pid", "md5", "size", "path", "status", "tries",
|
||
"code", "err", "fetched_at",
|
||
},
|
||
)
|
||
finally:
|
||
conn.close()
|
||
|
||
def test_init_is_idempotent(self):
|
||
conn = db.connect(self.cfg)
|
||
try:
|
||
db.init_db(conn)
|
||
db.init_db(conn)
|
||
tables = {
|
||
row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||
}
|
||
self.assertLessEqual(
|
||
{"thread", "post", "comment", "image", "user", "event", "batch",
|
||
"thread_seen", "post_stat_history", "verify_suspect", "thread_suspect"},
|
||
tables,
|
||
)
|
||
# 反复 init 不应该重复建表
|
||
before = len(tables)
|
||
db.init_db(conn)
|
||
after = len({row[0] for row in conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table'")})
|
||
self.assertEqual(before, after)
|
||
finally:
|
||
conn.close()
|
||
|
||
def test_ensure_compat_upgrades_v1_db(self):
|
||
# 手工造一个 v1 库:没有 v2 列,也没有 user 表
|
||
conn = sqlite3.connect(str(self.cfg.db_path))
|
||
conn.executescript(
|
||
"""
|
||
CREATE TABLE post(
|
||
pid INTEGER PRIMARY KEY, tid INTEGER NOT NULL, floor INTEGER,
|
||
author TEXT, author_id TEXT, content TEXT, content_html TEXT,
|
||
post_time TEXT, comment_num INTEGER DEFAULT 0, imgs TEXT,
|
||
is_lz INTEGER DEFAULT 0, fetched_at TEXT);
|
||
CREATE TABLE comment(
|
||
cid INTEGER PRIMARY KEY, pid INTEGER, tid INTEGER, author TEXT,
|
||
author_id TEXT, content TEXT, post_time TEXT, fetched_at TEXT);
|
||
INSERT INTO post(pid,tid,content) VALUES(1, 2, '旧数据');
|
||
"""
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
conn = db.connect(self.cfg)
|
||
try:
|
||
db.ensure_compat(conn)
|
||
db.ensure_compat(conn) # 幂等
|
||
self.assertTrue({"agree", "disagree", "sign", "frags"} <= db.table_columns(conn, "post"))
|
||
self.assertTrue(
|
||
{"floor", "agree", "disagree", "rto", "reply_to_name", "reply_to_uid", "frags"}
|
||
<= db.table_columns(conn, "comment")
|
||
)
|
||
self.assertIn("user", {row[0] for row in conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table'")})
|
||
self.assertEqual(conn.execute("SELECT content FROM post WHERE pid=1").fetchone()[0], "旧数据")
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
# ================================================================ config
|
||
class ConfigTests(unittest.TestCase):
|
||
def test_defaults_and_validation(self):
|
||
with make_temp_dir() as tmp:
|
||
cfg = Config(kw="东方年代记", data_dir=tmp)
|
||
self.assertEqual(cfg.rate, 1.3)
|
||
self.assertEqual(cfg.page_cap, 400)
|
||
self.assertEqual(cfg.db_path, Path(tmp) / "tieba.db")
|
||
self.assertEqual(cfg.comments_path, Path(tmp) / "raw" / "comments.jsonl")
|
||
with self.assertRaises(ValueError):
|
||
Config(kw="x", data_dir=tmp, rate=0)
|
||
with self.assertRaises(ValueError):
|
||
Config(kw="", data_dir=tmp)
|
||
with self.assertRaises(ValueError):
|
||
Config(kw="x", data_dir=tmp, comment_rn=51)
|
||
|
||
def test_save_load_roundtrip(self):
|
||
with make_temp_dir() as tmp:
|
||
cfg = Config(kw="东方年代记", data_dir=tmp, rate=2.5, workers=3)
|
||
cfg.fid = 2288117
|
||
cfg.forum_name = "东方年代记"
|
||
path = cfg.save()
|
||
self.assertTrue(path.exists())
|
||
self.assertNotIn("data_dir", json.loads(path.read_text(encoding="utf-8")))
|
||
loaded = Config.load(tmp)
|
||
self.assertEqual(loaded.kw, "东方年代记")
|
||
self.assertEqual(loaded.rate, 2.5)
|
||
self.assertEqual(loaded.workers, 3)
|
||
self.assertEqual(loaded.fid, 2288117)
|
||
|
||
def test_load_missing_uses_defaults(self):
|
||
with make_temp_dir() as tmp:
|
||
cfg = Config.load(tmp, kw="临时吧")
|
||
self.assertEqual(cfg.kw, "临时吧")
|
||
self.assertEqual(cfg.rate, 1.3)
|
||
|
||
def test_coerce_string_numbers(self):
|
||
with make_temp_dir() as tmp:
|
||
(Path(tmp) / "config.json").write_text(
|
||
json.dumps({"kw": "x", "rate": "0.5", "workers": "4"}), encoding="utf-8"
|
||
)
|
||
cfg = Config.load(tmp)
|
||
self.assertEqual(cfg.rate, 0.5)
|
||
self.assertEqual(cfg.workers, 4)
|
||
|
||
def test_bad_config(self):
|
||
with make_temp_dir() as tmp:
|
||
(Path(tmp) / "config.json").write_text("{not json", encoding="utf-8")
|
||
with self.assertRaises(ValueError):
|
||
Config.load(tmp)
|
||
|
||
|
||
# ================================================================ fixture / importer
|
||
def write_fixture(cfg: Config, *, v1: bool = False) -> None:
|
||
"""3 条楼层 + 3 条楼中楼 + 2 个主题的小样本。"""
|
||
cfg.ensure_dirs()
|
||
if v1:
|
||
threads = [
|
||
{"tid": 1001, "title": "v1 帖", "reply_num": 1, "is_good": 0, "is_top": 0}
|
||
]
|
||
posts = [
|
||
{
|
||
"tid": 1001, "pid": 5001, "floor": 1, "author": "旧楼主",
|
||
"author_id": 11, "text": "v1 正文", "time": 1700000000,
|
||
"reply_num": 0, "imgs": ["https://tiebapic.baidu.com/forum/pic/item/old.jpg"],
|
||
}
|
||
]
|
||
comments = [
|
||
{
|
||
"cid": 9001, "pid": 5001, "tid": 1001, "author": "旧路人",
|
||
"author_id": 22, "text": "v1 楼中楼", "time": 1700000100,
|
||
}
|
||
]
|
||
users = []
|
||
else:
|
||
threads = [
|
||
{
|
||
"tid": 1001, "fid": 2288117, "title": "测试帖 1", "author": "楼主甲",
|
||
"author_id": 11, "reply_num": 2, "is_good": 0, "is_top": 0,
|
||
},
|
||
{
|
||
"tid": 1002, "fid": 2288117, "title": "测试帖 2", "author": "路人乙",
|
||
"author_id": 22, "reply_num": 9, "is_good": 0, "is_top": 0,
|
||
},
|
||
]
|
||
posts = [
|
||
{
|
||
"tid": 1001, "pid": 5001, "floor": 1, "uid": 11, "uname": "楼主甲",
|
||
"agree": 3, "disagree": 0, "sign": "小尾巴", "rn": 2,
|
||
"ts": 1700000000, "lz": True, "text": "第一楼",
|
||
"frags": [{"t": "text", "text": "第一楼"}],
|
||
},
|
||
{
|
||
"tid": 1001, "pid": 5002, "floor": 2, "uid": 22, "uname": "路人乙",
|
||
"agree": 0, "disagree": 0, "sign": "", "rn": 0,
|
||
"ts": 1700000100, "lz": False, "text": "第二楼带图",
|
||
"frags": [
|
||
{"t": "text", "text": "第二楼带图"},
|
||
{"t": "img", "src": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg"},
|
||
],
|
||
},
|
||
{
|
||
"tid": 1001, "pid": 5003, "floor": 3, "uid": 11, "uname": "楼主甲",
|
||
"agree": 1, "disagree": 0, "sign": "", "rn": 1,
|
||
"ts": 1700000200, "lz": True, "text": "第三楼",
|
||
"frags": [{"t": "text", "text": "第三楼"}],
|
||
},
|
||
]
|
||
comments = [
|
||
{
|
||
"tid": 1001, "pid": 9001, "ppid": 5001, "floor": 1, "uid": 22,
|
||
"uname": "路人乙", "rto": 11, "agree": 1, "disagree": 0,
|
||
"ts": 1700000300, "lz": False, "text": "直接回复楼主",
|
||
"frags": [{"t": "text", "text": "直接回复楼主"}],
|
||
},
|
||
{
|
||
"tid": 1001, "pid": 9002, "ppid": 5003, "floor": 3, "uid": 33,
|
||
"uname": "路人丙", "rto": 0, "agree": 0, "disagree": 0,
|
||
"ts": 1700000400, "lz": False, "text": "回复 甲甲 : 正文匹配昵称",
|
||
"frags": [{"t": "text", "text": "回复 甲甲 : 正文匹配昵称"}],
|
||
},
|
||
{
|
||
"tid": 1001, "pid": 9003, "ppid": 5003, "floor": 3, "uid": 33,
|
||
"uname": "路人丙", "rto": 0, "agree": 0, "disagree": 0,
|
||
"ts": 1700000500, "lz": False, "text": "回复 查无此人 : 匹配不到",
|
||
"frags": [{"t": "text", "text": "回复 查无此人 : 匹配不到"}],
|
||
},
|
||
]
|
||
users = [
|
||
{
|
||
"uid": 11, "uname": "楼主甲", "nick": "甲甲", "portrait": "tb.1.aaa",
|
||
"level": 10, "glevel": 5, "gender": 1, "ip": "上海",
|
||
"bawu": False, "vip": True, "icons": ["vip"],
|
||
},
|
||
{
|
||
"uid": 22, "uname": "路人乙", "nick": "乙乙", "portrait": "",
|
||
"level": 3, "glevel": 2, "gender": 0, "ip": "江苏",
|
||
"bawu": False, "vip": False, "icons": [],
|
||
},
|
||
]
|
||
|
||
for path, records in (
|
||
(cfg.threads_path, threads),
|
||
(cfg.posts_path, posts),
|
||
(cfg.comments_path, comments),
|
||
(cfg.users_path, users),
|
||
):
|
||
with open(path, "w", encoding="utf-8", newline="\n") as fh:
|
||
for record in records:
|
||
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
cfg.done_path.write_text("1001\n", encoding="utf-8")
|
||
|
||
with open(cfg.image_index_path, "w", encoding="utf-8", newline="\n") as fh:
|
||
fh.write(
|
||
json.dumps(
|
||
{
|
||
"md5": "0" * 32,
|
||
"url": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg",
|
||
"size": 3,
|
||
"path": "imgs/00/00/" + "0" * 32 + ".jpg",
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
+ "\n"
|
||
)
|
||
|
||
|
||
class ImporterTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
|
||
write_fixture(self.cfg)
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
self.first = importer.import_all(self.cfg, quiet=True)
|
||
self.second = importer.import_all(self.cfg, quiet=True)
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _rows(self, sql, params=()):
|
||
conn = db.connect(self.cfg, readonly=True)
|
||
try:
|
||
return conn.execute(sql, params).fetchall()
|
||
finally:
|
||
conn.close()
|
||
|
||
def test_counts(self):
|
||
self.assertEqual(self.first["threads"], 2)
|
||
self.assertEqual(self.first["posts"], 3)
|
||
self.assertEqual(self.first["comments"], 3)
|
||
self.assertEqual(self.first["users"], 2)
|
||
self.assertEqual(self.first["stats"]["posts"], 3)
|
||
self.assertEqual(self.first["stats"]["comments"], 3)
|
||
self.assertEqual(self.first["stats"]["posts_signed"], 1)
|
||
self.assertEqual(self.first["stats"]["posts_agreed"], 2)
|
||
self.assertEqual(self.first["stats"]["comments_rto"], 1)
|
||
self.assertEqual(self.first["stats"]["comments_named"], 2)
|
||
self.assertEqual(self.first["stats"]["comments_targeted"], 2)
|
||
|
||
def test_idempotent(self):
|
||
self.assertEqual(self.first["posts"], self.second["posts"])
|
||
self.assertEqual(self.first["comments"], self.second["comments"])
|
||
self.assertEqual(self.first["stats"], self.second["stats"])
|
||
|
||
def test_reply_targets(self):
|
||
rows = {
|
||
row["cid"]: (row["reply_to_name"], row["reply_to_uid"])
|
||
for row in self._rows("SELECT cid, reply_to_name, reply_to_uid FROM comment")
|
||
}
|
||
self.assertEqual(rows[9001], (None, 11)) # rto 优先
|
||
self.assertEqual(rows[9002], ("甲甲", 11)) # 正文昵称 → user.nick → 本帖作者
|
||
self.assertEqual(rows[9003], ("查无此人", None)) # 匹配不到就不猜
|
||
|
||
def test_thread_status_and_coverage(self):
|
||
rows = {
|
||
row["tid"]: (row["status"], row["pages"])
|
||
for row in self._rows("SELECT tid, status, pages FROM thread")
|
||
}
|
||
self.assertEqual(rows[1001][0], "done")
|
||
self.assertEqual(rows[1002][0], "new")
|
||
stats = self.second["stats"]
|
||
self.assertEqual(stats["threads_done"], 1)
|
||
self.assertEqual(stats["threads_with_posts"], 1)
|
||
self.assertAlmostEqual(db.coverage(stats), 50.0)
|
||
|
||
def test_frags_and_imgs_persisted(self):
|
||
row = self._rows("SELECT frags, imgs, sign, agree FROM post WHERE pid=5002")[0]
|
||
frags = json.loads(row["frags"])
|
||
self.assertEqual(frags[1]["t"], "img")
|
||
self.assertEqual(json.loads(row["imgs"]), [frags[1]["src"]])
|
||
sign = self._rows("SELECT sign FROM post WHERE pid=5001")[0]["sign"]
|
||
self.assertEqual(sign, "小尾巴")
|
||
|
||
def test_v1_records(self):
|
||
tmp = make_temp_dir()
|
||
try:
|
||
cfg = Config(kw="测试吧", data_dir=Path(tmp.name), rate=1000.0)
|
||
write_fixture(cfg, v1=True)
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
importer.import_all(cfg, quiet=True)
|
||
conn = db.connect(cfg, readonly=True)
|
||
try:
|
||
post = conn.execute("SELECT content, author, author_id, imgs, frags FROM post").fetchone()
|
||
comment = conn.execute(
|
||
"SELECT pid, author, content, rto FROM comment"
|
||
).fetchone()
|
||
finally:
|
||
conn.close()
|
||
self.assertEqual(post["content"], "v1 正文")
|
||
self.assertEqual(post["author"], "旧楼主")
|
||
self.assertIn("old.jpg", post["imgs"])
|
||
self.assertEqual(json.loads(post["frags"])[0]["t"], "img")
|
||
self.assertEqual(comment["pid"], 5001) # v1 的 pid 是父楼层
|
||
self.assertEqual(comment["author"], "旧路人")
|
||
finally:
|
||
tmp.cleanup()
|
||
|
||
def test_missing_raw_is_ok(self):
|
||
tmp = make_temp_dir()
|
||
try:
|
||
cfg = Config(kw="空吧", data_dir=Path(tmp.name), rate=1000.0)
|
||
cfg.ensure_dirs()
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
result = importer.import_all(cfg, quiet=True)
|
||
self.assertEqual(result["posts"], 0)
|
||
self.assertEqual(result["stats"]["threads"], 0)
|
||
finally:
|
||
tmp.cleanup()
|
||
|
||
|
||
class ExportTests(unittest.TestCase):
|
||
def test_export_threads(self):
|
||
with make_temp_dir() as tmp:
|
||
cfg = Config(kw="测试吧", data_dir=tmp, rate=1000.0)
|
||
write_fixture(cfg)
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
importer.import_all(cfg, quiet=True)
|
||
cfg.threads_path.unlink() # 确认 export 是从库生成,而不是原样拷贝
|
||
count = export.export_threads(cfg, quiet=True)
|
||
self.assertEqual(count, 2)
|
||
lines = [json.loads(line) for line in cfg.threads_path.read_text(encoding="utf-8").splitlines()]
|
||
self.assertEqual([row["tid"] for row in lines], [1001, 1002])
|
||
self.assertEqual(lines[0]["title"], "测试帖 1")
|
||
self.assertEqual(lines[0]["fid"], 2288117)
|
||
|
||
|
||
# ================================================================ images
|
||
class ImagesTests(unittest.TestCase):
|
||
def test_to_original(self):
|
||
compressed = (
|
||
"https://imgsrc.baidu.com/forum/w%3D720%3Bq%3D60%3Bg%3D0/"
|
||
"sign=abc123/1234567890.jpg"
|
||
)
|
||
self.assertEqual(
|
||
images.to_original(compressed),
|
||
"https://imgsrc.baidu.com/forum/pic/item/1234567890.jpg",
|
||
)
|
||
self.assertEqual(
|
||
images.to_original("https://tiebapic.baidu.com/forum/pic/item/abc.png"),
|
||
"https://imgsrc.baidu.com/forum/pic/item/abc.png",
|
||
)
|
||
c = "https://c.tieba.baidu.com/c/p/img?src=https%3A%2F%2Fexample.com%2Fa.jpg"
|
||
self.assertEqual(images.to_original(c), "https://example.com/a.jpg")
|
||
self.assertEqual(images.to_original("https://example.com/"), "https://example.com/")
|
||
|
||
def test_ext_for(self):
|
||
self.assertEqual(images.ext_for("image/jpeg", "http://x/a"), ".jpg")
|
||
self.assertEqual(images.ext_for("image/png; charset=x", "http://x/a"), ".png")
|
||
self.assertEqual(images.ext_for("", "http://x/a.webp"), ".webp")
|
||
self.assertEqual(images.ext_for("", "http://x/a"), ".jpg")
|
||
|
||
def test_collect_urls(self):
|
||
with make_temp_dir() as tmp:
|
||
cfg = Config(kw="x", data_dir=tmp, rate=1000.0)
|
||
cfg.ensure_dirs()
|
||
posts = [
|
||
{"frags": [{"t": "img", "src": "http://a/1.jpg"}, {"t": "text", "text": "x"}]},
|
||
{"frags": [{"t": "img", "src": "http://a/1.jpg"}]},
|
||
{"imgs": ["http://a/2.jpg"]},
|
||
]
|
||
with open(cfg.posts_path, "w", encoding="utf-8") as fh:
|
||
for post in posts:
|
||
fh.write(json.dumps(post) + "\n")
|
||
self.assertEqual(images.collect_urls(cfg), ["http://a/1.jpg", "http://a/2.jpg"])
|
||
|
||
def test_save_blob_buckets_by_md5(self):
|
||
with make_temp_dir() as tmp:
|
||
cfg = Config(kw="x", data_dir=tmp, rate=1000.0)
|
||
cfg.ensure_dirs()
|
||
md5 = "ab" + "0" * 30
|
||
path = images.save_blob(cfg, md5, b"hello", "image/png", "http://a/x")
|
||
self.assertEqual(path, f"imgs/ab/00/{md5}.png")
|
||
self.assertEqual((cfg.data_dir / path).read_bytes(), b"hello")
|
||
image = cfg.data_dir / path
|
||
image.write_bytes(b"hello") # 已存在时不重复写,也不报错
|
||
self.assertEqual(images.save_blob(cfg, md5, b"hello", "image/png", "http://a/x"), path)
|
||
|
||
def test_import_index(self):
|
||
with make_temp_dir() as tmp:
|
||
cfg = Config(kw="x", data_dir=tmp, rate=1000.0)
|
||
write_fixture(cfg)
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
result = images.import_index(cfg, quiet=True)
|
||
again = images.import_index(cfg, quiet=True)
|
||
self.assertEqual(result["filled"], 1)
|
||
self.assertEqual(again["filled"], 1)
|
||
conn = db.connect(cfg, readonly=True)
|
||
try:
|
||
row = conn.execute(
|
||
"SELECT status, md5, size, path FROM image WHERE url=?",
|
||
("https://tiebapic.baidu.com/forum/pic/item/abc.jpg",),
|
||
).fetchone()
|
||
count = conn.execute("SELECT COUNT(*) FROM image").fetchone()[0]
|
||
finally:
|
||
conn.close()
|
||
self.assertEqual(row["status"], "done")
|
||
self.assertEqual(row["md5"], "0" * 32)
|
||
self.assertEqual(count, 1)
|
||
|
||
|
||
# ================================================================ crawler(假 client)
|
||
class FragText:
|
||
def __init__(self, text):
|
||
self.text = text
|
||
|
||
|
||
class FragImage:
|
||
def __init__(self, src):
|
||
self.src = src
|
||
self.big_src = ""
|
||
self.origin_src = ""
|
||
self.hash = ""
|
||
self.show_width = 100
|
||
self.show_height = 100
|
||
|
||
|
||
class Contents:
|
||
def __init__(self, *objs):
|
||
self.objs = list(objs)
|
||
|
||
|
||
def make_user(uid: int, name: str):
|
||
return SimpleNamespace(
|
||
user_id=uid, user_name=name, nick_name_new=name + "昵称", portrait="",
|
||
level=1, glevel=1, gender=0, ip="", is_bawu=False, is_vip=False, icons=[],
|
||
)
|
||
|
||
|
||
def make_post(pid, floor, uid, name, *, sign="", agree=0, reply_num=0, comments=None,
|
||
text="正文", image=None):
|
||
objs = [FragText(text)]
|
||
if image:
|
||
objs.append(FragImage(image))
|
||
return SimpleNamespace(
|
||
contents=Contents(*objs), sign=sign, comments=comments or [], is_aimeme=False,
|
||
fid=2288117, fname="测试吧", tid=2001, pid=pid, user=make_user(uid, name),
|
||
author_id=uid, floor=floor, reply_num=reply_num, agree=agree, disagree=0,
|
||
create_time=1700000000, is_thread_author=floor == 1,
|
||
)
|
||
|
||
|
||
def make_comment(cid, ppid, uid, name, *, rto=0, text="楼中楼", floor=1, agree=0):
|
||
return SimpleNamespace(
|
||
contents=Contents(FragText(text)), fid=2288117, fname="测试吧", tid=2001,
|
||
ppid=ppid, pid=cid, user=make_user(uid, name), reply_to_id=rto, floor=floor,
|
||
agree=agree, disagree=0, create_time=1700000100, is_thread_author=False,
|
||
)
|
||
|
||
|
||
class Page:
|
||
def __init__(self, has_more=False, total_page=1):
|
||
self.has_more = has_more
|
||
self.total_page = total_page
|
||
|
||
|
||
class Result:
|
||
def __init__(self, objs=None, *, err=None, has_more=False, total_page=1):
|
||
self.objs = objs or []
|
||
self.err = err
|
||
self.page = Page(has_more, total_page)
|
||
|
||
|
||
class FakeCrawlClient:
|
||
"""只实现 crawl_thread 用到的两个方法。"""
|
||
|
||
def __init__(self, post_pages, comment_pages=None):
|
||
self.post_pages = post_pages
|
||
self.comment_pages = comment_pages or {}
|
||
self.comment_calls = []
|
||
|
||
async def get_posts(self, tid, pn=1, rn=30, with_comments=True, comment_rn=50):
|
||
pages = self.post_pages.get(tid) or []
|
||
if pn <= len(pages):
|
||
return pages[pn - 1]
|
||
return pages[-1] if pages else Result([])
|
||
|
||
async def get_comments(self, tid, pid, pn=1):
|
||
self.comment_calls.append((tid, pid, pn))
|
||
pages = self.comment_pages.get(pid) or []
|
||
if pn <= len(pages):
|
||
return pages[pn - 1]
|
||
return pages[-1] if pages else Result([])
|
||
|
||
|
||
class FakeEnumClient:
|
||
def __init__(self, pages, threads_total=3):
|
||
self.pages = pages
|
||
self.threads_total = threads_total
|
||
self.forum = SimpleNamespace(
|
||
fid=2288117, fname="测试吧", thread_num=threads_total, member_num=1, post_num=1
|
||
)
|
||
|
||
async def get_forum(self, kw):
|
||
return self.forum
|
||
|
||
async def get_threads(self, kw, pn=1, rn=30):
|
||
if pn <= len(self.pages):
|
||
return self.pages[pn - 1]
|
||
return Result([])
|
||
|
||
|
||
class CrawlerTests(unittest.IsolatedAsyncioTestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(
|
||
kw="测试吧", data_dir=Path(self.tmp.name), rate=1_000_000.0,
|
||
tries=2, backoff=0.0, page_cap=400,
|
||
)
|
||
self.cfg.ensure_dirs()
|
||
crawler.set_rate(self.cfg.rate)
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _jsonl(self, path):
|
||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
|
||
|
||
async def test_enum_writes_threads_jsonl(self):
|
||
pages = [
|
||
Result(
|
||
[
|
||
SimpleNamespace(
|
||
tid=3001, title="主题 A", user=make_user(11, "甲"),
|
||
author_id=11, reply_num=5, is_good=True, is_top=False, fid=2288117,
|
||
),
|
||
SimpleNamespace(
|
||
tid=3002, title="主题 B", user=make_user(22, "乙"),
|
||
author_id=22, reply_num=0, is_good=False, is_top=False, fid=2288117,
|
||
),
|
||
],
|
||
has_more=True,
|
||
total_page=2,
|
||
),
|
||
Result(
|
||
[
|
||
SimpleNamespace(
|
||
tid=3003, title="主题 C", user=make_user(33, "丙"),
|
||
author_id=33, reply_num=1, is_good=False, is_top=False, fid=2288117,
|
||
)
|
||
],
|
||
has_more=False,
|
||
total_page=2,
|
||
),
|
||
]
|
||
client = FakeEnumClient(pages)
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
result = await crawler.enumerate_with_client(client, self.cfg)
|
||
again = await crawler.enumerate_with_client(client, self.cfg)
|
||
self.assertEqual(result["added"], 3)
|
||
self.assertEqual(again["added"], 0)
|
||
rows = self._jsonl(self.cfg.threads_path)
|
||
self.assertEqual([row["tid"] for row in rows], [3001, 3002, 3003])
|
||
self.assertEqual(rows[0]["title"], "主题 A")
|
||
self.assertEqual(rows[0]["is_good"], 1)
|
||
self.assertEqual(json.loads(self.cfg.config_path.read_text(encoding="utf-8"))["fid"], 2288117)
|
||
|
||
async def test_crawl_thread_writes_records(self):
|
||
inline = [make_comment(9100, 5001, 33, "丙", rto=0, text="内联楼中楼")]
|
||
pages = {
|
||
2001: [
|
||
Result(
|
||
[
|
||
make_post(5001, 1, 11, "甲", sign="尾巴", agree=2, reply_num=1, comments=inline),
|
||
make_post(5002, 2, 22, "乙", text="带图", image="http://img/1.jpg"),
|
||
],
|
||
has_more=False,
|
||
)
|
||
]
|
||
}
|
||
client = FakeCrawlClient(pages)
|
||
writer = crawler.JsonlWriter(self.cfg)
|
||
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
|
||
lock = asyncio.Lock()
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
|
||
writer.close()
|
||
|
||
posts = self._jsonl(self.cfg.posts_path)
|
||
comments = self._jsonl(self.cfg.comments_path)
|
||
users = self._jsonl(self.cfg.users_path)
|
||
self.assertEqual(len(posts), 2)
|
||
self.assertEqual(posts[0]["sign"], "尾巴")
|
||
self.assertEqual(posts[0]["agree"], 2)
|
||
self.assertEqual(posts[0]["text"], "正文")
|
||
self.assertEqual(posts[1]["frags"][1], {
|
||
"t": "img", "src": "http://img/1.jpg", "big": "", "origin": "",
|
||
"hash": "", "w": 100, "h": 100,
|
||
})
|
||
self.assertEqual(len(comments), 1)
|
||
self.assertEqual(comments[0]["ppid"], 5001)
|
||
self.assertEqual(comments[0]["pid"], 9100)
|
||
self.assertEqual(len(users), 3)
|
||
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
|
||
self.assertEqual(stats["threads"], 1)
|
||
|
||
async def test_deleted_thread_marks_done(self):
|
||
pages = {2001: [Result([], err=SimpleNamespace(code=4))]}
|
||
client = FakeCrawlClient(pages)
|
||
writer = crawler.JsonlWriter(self.cfg)
|
||
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
|
||
lock = asyncio.Lock()
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
|
||
writer.close()
|
||
self.assertEqual(stats["deleted"], 1)
|
||
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
|
||
errors = self._jsonl(self.cfg.err_path)
|
||
self.assertEqual(errors[0]["code"], 4)
|
||
self.assertIn("deleted", errors[0]["err"])
|
||
|
||
async def test_empty_results_skip_without_done(self):
|
||
# 带错误的空结果(比如 429)才该重试/跳过,不能当成「翻到底了」
|
||
pages = {2001: [Result([], err=SimpleNamespace(code=429))]}
|
||
client = FakeCrawlClient(pages)
|
||
writer = crawler.JsonlWriter(self.cfg)
|
||
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
|
||
lock = asyncio.Lock()
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
with self.assertRaises(crawler.CrawlError):
|
||
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
|
||
writer.close()
|
||
self.assertFalse(self.cfg.done_path.exists() and self.cfg.done_path.read_text().strip())
|
||
self.assertEqual(stats["threads"], 0)
|
||
|
||
async def test_empty_last_page_ends_thread(self):
|
||
# 真实遇到的情况:末页不足 30 楼,第 2 页返回空、err=None、has_more=False,
|
||
# 这帖其实已经抓完了,应该写 done 而不是当成 429 无限重试
|
||
pages = {
|
||
2001: [
|
||
Result([make_post(5001, 1, 11, "甲")], has_more=True),
|
||
Result([], has_more=False),
|
||
]
|
||
}
|
||
client = FakeCrawlClient(pages)
|
||
writer = crawler.JsonlWriter(self.cfg)
|
||
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
|
||
lock = asyncio.Lock()
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
|
||
writer.close()
|
||
self.assertEqual(len(self._jsonl(self.cfg.posts_path)), 1)
|
||
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
|
||
self.assertEqual(stats["threads"], 1)
|
||
|
||
async def test_page_cap_stops_without_done(self):
|
||
self.cfg.page_cap = 1
|
||
pages = {
|
||
2001: [
|
||
Result([make_post(5001, 1, 11, "甲")], has_more=True),
|
||
Result([make_post(5002, 2, 11, "甲")], has_more=True),
|
||
]
|
||
}
|
||
client = FakeCrawlClient(pages)
|
||
writer = crawler.JsonlWriter(self.cfg)
|
||
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
|
||
lock = asyncio.Lock()
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
with self.assertRaises(crawler.CrawlError) as ctx:
|
||
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
|
||
writer.close()
|
||
self.assertIn("页数上限", str(ctx.exception))
|
||
self.assertFalse(self.cfg.done_path.exists() and self.cfg.done_path.read_text().strip())
|
||
|
||
async def test_overflow_comments_are_fetched(self):
|
||
pages = {
|
||
2001: [
|
||
Result(
|
||
[
|
||
make_post(
|
||
5001, 1, 11, "甲",
|
||
comments=[make_comment(9100, 5001, 22, "乙", text="内联")],
|
||
reply_num=3,
|
||
)
|
||
],
|
||
has_more=False,
|
||
)
|
||
]
|
||
}
|
||
comments = {
|
||
5001: [
|
||
Result(
|
||
[
|
||
make_comment(9101, 5001, 22, "乙", text="补 1", agree=2),
|
||
make_comment(9102, 5001, 22, "乙", text="补 2"),
|
||
],
|
||
has_more=False,
|
||
)
|
||
]
|
||
}
|
||
client = FakeCrawlClient(pages, comments)
|
||
writer = crawler.JsonlWriter(self.cfg)
|
||
stats = {"threads": 0, "deleted": 0, "skipped": 0, "errors": 0, "overflow": 0, "overflow_err": 0}
|
||
lock = asyncio.Lock()
|
||
with contextlib.redirect_stdout(io.StringIO()):
|
||
await crawler.crawl_thread(client, self.cfg, writer, {"tid": 2001}, stats, lock)
|
||
writer.close()
|
||
self.assertEqual(stats["overflow"], 1)
|
||
self.assertEqual(len(self._jsonl(self.cfg.comments_path)), 3)
|
||
self.assertEqual(client.comment_calls, [(2001, 5001, 1)])
|
||
self.assertIn("2001", self.cfg.done_path.read_text(encoding="utf-8"))
|
||
|
||
def test_frags_conversion(self):
|
||
contents = Contents(
|
||
FragText("文字"),
|
||
FragImage("http://img/x.jpg"),
|
||
)
|
||
frags = crawler.frags_of(contents)
|
||
self.assertEqual(frags[0], {"t": "text", "text": "文字"})
|
||
self.assertEqual(frags[1]["t"], "img")
|
||
self.assertEqual(crawler.text_of(frags), "文字")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|
||
|
||
|
||
# ================================================================ 事件流 / verify
|
||
class FakeVerifyClient:
|
||
"""只实现 verify_thread 用到的 get_posts。"""
|
||
|
||
def __init__(self, pages_by_tid):
|
||
self.pages_by_tid = pages_by_tid
|
||
|
||
async def get_posts(self, tid, pn=1, rn=30, with_comments=True, comment_rn=50):
|
||
pages = self.pages_by_tid.get(tid) or []
|
||
if pn <= len(pages):
|
||
return pages[pn - 1]
|
||
return Result([], has_more=False)
|
||
|
||
|
||
class VerifyTests(unittest.IsolatedAsyncioTestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1_000_000.0,
|
||
tries=2, backoff=0.0, page_cap=50)
|
||
self.cfg.ensure_dirs()
|
||
crawler.set_rate(self.cfg.rate)
|
||
self.conn = db.connect(self.cfg)
|
||
db.init_db(self.conn)
|
||
db.ensure_compat(self.conn)
|
||
self.conn.execute(
|
||
"INSERT INTO thread(tid, fid, title, reply_num, status) VALUES(1, 2, '帖', 3, 'done')"
|
||
)
|
||
self.conn.commit()
|
||
self.batch = events.start_batch(self.conn, "verify")
|
||
self.log = events.EventLog(self.conn, self.cfg, self.batch, "verify")
|
||
self.stats = {"ok": 0, "added": 0, "edited": 0, "deleted": 0, "suspect": 0,
|
||
"thread_deleted": 0, "thread_suspect": 0, "stat_samples": 0, "errors": 0}
|
||
|
||
def tearDown(self):
|
||
self.log.close()
|
||
self.conn.close()
|
||
self.tmp.cleanup()
|
||
|
||
def _seed(self, pid, floor, text, agree=0):
|
||
imgs = "[]"
|
||
self.conn.execute(
|
||
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, "
|
||
"agree, disagree, status, content_hash) VALUES(?,?,?,?,?,?,?,?,?, 'live', ?)",
|
||
(pid, 1, floor, "甲", "11", text, imgs, agree, 0,
|
||
events.content_hash(text, imgs, "甲")),
|
||
)
|
||
self.conn.commit()
|
||
|
||
async def _run(self, pages):
|
||
client = FakeVerifyClient({1: pages})
|
||
await verify.verify_thread(client, self.cfg, self.conn, self.log, 1, self.stats)
|
||
self.conn.commit()
|
||
|
||
async def test_add_new_post(self):
|
||
self._seed(100, 1, "旧楼")
|
||
old = make_post(100, 1, 11, "甲", text="旧楼")
|
||
post = make_post(101, 2, 11, "乙", text="新楼")
|
||
await self._run([Result([old, post], has_more=False)])
|
||
self.assertEqual(self.stats["added"], 1)
|
||
kind, = [r[0] for r in self.conn.execute("SELECT kind FROM event")]
|
||
self.assertEqual(kind, "post_add")
|
||
row = self.conn.execute("SELECT status FROM post WHERE pid=101").fetchone()
|
||
self.assertEqual(row[0], "live")
|
||
|
||
async def test_two_strike_delete(self):
|
||
self._seed(100, 1, "要被删的楼")
|
||
page = Result([], has_more=False) # 服务器这页什么都没有,但也没报错
|
||
# 第一次:只登记怀疑,不动数据
|
||
await self._run([Result([make_post(999, 1, 11, "别人")], has_more=False)])
|
||
self.assertEqual(self.stats["suspect"], 1)
|
||
status = self.conn.execute("SELECT status FROM post WHERE pid=100").fetchone()[0]
|
||
self.assertEqual(status, "live")
|
||
# 第二次:仍然缺 → 进回收站
|
||
await self._run([Result([make_post(999, 1, 11, "别人")], has_more=False)])
|
||
self.assertEqual(self.stats["deleted"], 1)
|
||
row = self.conn.execute("SELECT status, recycle_reason FROM post WHERE pid=100").fetchone()
|
||
self.assertEqual(tuple(row), ("recycled", "post_deleted"))
|
||
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event ORDER BY id")]
|
||
self.assertIn("post_del", kinds)
|
||
|
||
async def test_edit_detected(self):
|
||
self._seed(100, 1, "old text")
|
||
post = make_post(100, 1, 11, "甲", text="new text")
|
||
await self._run([Result([post], has_more=False)])
|
||
self.assertEqual(self.stats["edited"], 1)
|
||
content = self.conn.execute("SELECT content FROM post WHERE pid=100").fetchone()[0]
|
||
self.assertEqual(content, "new text")
|
||
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event")]
|
||
self.assertIn("post_edit", kinds)
|
||
|
||
async def test_thread_deleted_pushes_posts_to_bin(self):
|
||
self._seed(100, 1, "跟帖一起消失")
|
||
await self._run([Result([], err=SimpleNamespace(code=4))])
|
||
self.assertEqual(self.stats["thread_suspect"], 1)
|
||
await self._run([Result([], err=SimpleNamespace(code=4))])
|
||
self.assertEqual(self.stats["thread_deleted"], 1)
|
||
row = self.conn.execute("SELECT status, recycle_reason FROM post WHERE pid=100").fetchone()
|
||
self.assertEqual(tuple(row), ("recycled", "thread_deleted"))
|
||
self.assertIsNotNone(self.conn.execute("SELECT deleted_at FROM thread WHERE tid=1").fetchone()[0])
|
||
|
||
async def test_like_change_recorded(self):
|
||
self._seed(100, 1, "有赞的楼", agree=3)
|
||
post = make_post(100, 1, 11, "甲", text="有赞的楼")
|
||
post.agree = 5
|
||
await self._run([Result([post], has_more=False)])
|
||
self.assertEqual(self.stats["stat_samples"], 1)
|
||
row = self.conn.execute("SELECT agree FROM post WHERE pid=100").fetchone()
|
||
self.assertEqual(row[0], 5)
|
||
hist = self.conn.execute("SELECT agree FROM post_stat_history WHERE pid=100").fetchall()
|
||
self.assertEqual([h[0] for h in hist], [5])
|
||
|
||
|
||
async def test_comment_deleted_after_two_scans(self):
|
||
self._seed(100, 1, "有楼中楼")
|
||
self.conn.execute(
|
||
"INSERT INTO comment(cid, pid, tid, author, author_id, content, post_time, status) "
|
||
"VALUES(900, 100, 1, '乙', '22', '旧楼中楼', '2020-01-01 00:00:00', 'live')"
|
||
)
|
||
self.conn.commit()
|
||
post = make_post(100, 1, 11, "甲", text="有楼中楼")
|
||
post.reply_num = 0 # 内联 0 条 = 这一层的楼中楼被完整重取了
|
||
await self._run([Result([post], has_more=False)])
|
||
self.assertEqual(self.stats.get("comment_suspect", 0), 1)
|
||
await self._run([Result([post], has_more=False)])
|
||
self.assertEqual(self.stats.get("comment_deleted", 0), 1)
|
||
row = self.conn.execute("SELECT status FROM comment WHERE cid=900").fetchone()
|
||
self.assertEqual(row[0], "recycled")
|
||
|
||
|
||
# ================================================================ as-of 重放
|
||
class ReplayTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
|
||
self.cfg.ensure_dirs()
|
||
self.conn = db.connect(self.cfg)
|
||
db.init_db(self.conn)
|
||
db.ensure_compat(self.conn)
|
||
self.conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
|
||
"VALUES(1, 2, '帖', 3, 'done')")
|
||
self._seed(100, 1, "base")
|
||
self.conn.commit()
|
||
|
||
def tearDown(self):
|
||
self.conn.close()
|
||
self.tmp.cleanup()
|
||
|
||
def _seed(self, pid, floor, text):
|
||
self.conn.execute(
|
||
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
|
||
"VALUES(?,?,?,?,?,?, '[]', 'live')",
|
||
(pid, 1, floor, "甲", "11", text),
|
||
)
|
||
|
||
def _add_event(self, ts, kind, pid=None, payload=None, tid=1):
|
||
self.conn.execute(
|
||
"INSERT INTO event(batch, ts, kind, tid, pid, payload, source) VALUES(1,?,?,?,?,?,'verify')",
|
||
(ts, kind, tid, pid, json.dumps(payload, ensure_ascii=False) if payload else None),
|
||
)
|
||
ev = {"ts": ts, "kind": kind, "tid": tid, "pid": pid, "cid": None,
|
||
"payload": payload or {}}
|
||
events.apply_event(self.conn, ev)
|
||
self.conn.commit()
|
||
|
||
def test_new_post_rolls_back_before_add(self):
|
||
self._add_event("2026-11-17 10:00:00", "post_add", pid=101,
|
||
payload={"pid": 101, "tid": 1, "floor": 2, "author": "乙",
|
||
"content": "later", "status": "live"})
|
||
rows = replay.as_of_thread(self.conn, 1, "2026-11-16")
|
||
self.assertEqual([r["pid"] for r in rows], [100])
|
||
rows = replay.as_of_thread(self.conn, 1, "2026-11-17")
|
||
self.assertEqual([r["pid"] for r in rows], [100, 101])
|
||
|
||
def test_delete_rolls_back(self):
|
||
self._add_event("2026-11-18 10:00:00", "post_del", pid=100)
|
||
before = replay.as_of_thread(self.conn, 1, "2026-11-17")
|
||
self.assertEqual(before[0]["status"], "live")
|
||
after = replay.as_of_thread(self.conn, 1, "2026-11-18")
|
||
self.assertEqual(after[0]["status"], "recycled")
|
||
|
||
def test_edit_rolls_back_to_old_content(self):
|
||
self._add_event("2026-11-19 10:00:00", "post_edit", pid=100,
|
||
payload={"content": "new", "old_content": "base"})
|
||
before = replay.as_of_thread(self.conn, 1, "2026-11-18")
|
||
self.assertEqual(before[0]["content"], "base")
|
||
after = replay.as_of_thread(self.conn, 1, "2026-11-19")
|
||
self.assertEqual(after[0]["content"], "new")
|
||
|
||
def test_thread_deleted_rolls_back(self):
|
||
self._add_event("2026-11-20 10:00:00", "thread_deleted", tid=1)
|
||
before = replay.as_of_thread(self.conn, 1, "2026-11-19")
|
||
self.assertEqual(before[0]["status"], "live")
|
||
after = replay.as_of_thread(self.conn, 1, "2026-11-20")
|
||
self.assertEqual(after[0]["status"], "recycled")
|
||
|
||
|
||
# ================================================================ 增量 update
|
||
class FakeUpdateClient:
|
||
def __init__(self, thread_pages, post_pages):
|
||
self.thread_pages = thread_pages
|
||
self.post_pages = post_pages
|
||
|
||
async def get_threads(self, kw, pn=1, rn=30):
|
||
if pn <= len(self.thread_pages):
|
||
return self.thread_pages[pn - 1]
|
||
return Result([], has_more=False)
|
||
|
||
async def get_posts(self, tid, pn=1, rn=30, with_comments=True, comment_rn=50):
|
||
pages = self.post_pages.get(tid) or []
|
||
if pn <= len(pages):
|
||
return pages[pn - 1]
|
||
return Result([], has_more=False)
|
||
|
||
|
||
def _thread_obj(tid, reply_num=1, title="帖"):
|
||
return SimpleNamespace(tid=tid, title=title, user=make_user(11, "甲"), author_id=11,
|
||
reply_num=reply_num, is_good=False, is_top=False, fid=2)
|
||
|
||
|
||
class UpdateTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1_000_000.0,
|
||
tries=2, backoff=0.0, page_cap=50)
|
||
self.cfg.ensure_dirs()
|
||
crawler.set_rate(self.cfg.rate)
|
||
self.conn = db.connect(self.cfg)
|
||
db.init_db(self.conn)
|
||
db.ensure_compat(self.conn)
|
||
self.conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
|
||
"VALUES(1, 2, '老帖', 3, 'done')")
|
||
for pid, floor in ((100, 1), (101, 2)):
|
||
self.conn.execute(
|
||
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
|
||
"VALUES(?,?,?,?,?,?, '[]', 'live')",
|
||
(pid, 1, floor, "甲", "11", f"第{floor}楼"),
|
||
)
|
||
self.conn.execute("INSERT INTO thread_seen(tid, reply_num, max_floor, seen_at) "
|
||
"VALUES(1, 2, 2, '2026-11-16 00:00:00')")
|
||
self.conn.commit()
|
||
|
||
def tearDown(self):
|
||
self.conn.close()
|
||
self.tmp.cleanup()
|
||
|
||
def test_new_thread_and_tail_floor(self):
|
||
thread_pages = [Result([_thread_obj(1, reply_num=5), _thread_obj(2, reply_num=1)],
|
||
has_more=False)]
|
||
post_pages = {
|
||
1: [Result([make_post(100, 1, 11, "甲", text="第1楼"),
|
||
make_post(101, 2, 11, "甲", text="第2楼"),
|
||
make_post(102, 3, 11, "甲", text="第3楼",
|
||
comments=[make_comment(900, 102, 22, "乙",
|
||
text="回复 甲 : 你好")])],
|
||
has_more=False)],
|
||
2: [Result([make_post(200, 1, 11, "甲", text="新帖一楼")], has_more=False)],
|
||
}
|
||
client = FakeUpdateClient(thread_pages, post_pages)
|
||
stats = update.update(self.cfg, pages=1, client=client)
|
||
|
||
self.assertEqual(stats["new_threads"], 1)
|
||
self.assertEqual(stats["changed_threads"], 1)
|
||
self.assertEqual(stats["added"], 2)
|
||
self.assertEqual(stats["errors"], 0)
|
||
|
||
pids = {r[0] for r in self.conn.execute("SELECT pid FROM post")}
|
||
self.assertEqual(pids, {100, 101, 102, 200})
|
||
reply_num = self.conn.execute("SELECT reply_num FROM thread WHERE tid=1").fetchone()[0]
|
||
self.assertEqual(reply_num, 5)
|
||
max_floor = self.conn.execute("SELECT max_floor FROM thread_seen WHERE tid=1").fetchone()[0]
|
||
self.assertEqual(max_floor, 3)
|
||
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event ORDER BY id")]
|
||
self.assertEqual(kinds, ["post_add", "post_add", "comment_add"])
|
||
self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM comment").fetchone()[0], 1)
|
||
cname = self.conn.execute("SELECT reply_to_name FROM comment WHERE cid=900").fetchone()[0]
|
||
self.assertEqual(cname, "甲")
|
||
|
||
|
||
# ================================================================ restore / rebuild
|
||
class RestoreRebuildTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
|
||
self.cfg.ensure_dirs()
|
||
self.conn = db.connect(self.cfg)
|
||
db.init_db(self.conn)
|
||
db.ensure_compat(self.conn)
|
||
self.conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
|
||
"VALUES(1, 2, '帖', 1, 'done')")
|
||
self.conn.execute(
|
||
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
|
||
"VALUES(100, 1, 1, '甲', '11', '一楼', '[]', 'live')"
|
||
)
|
||
self.conn.commit()
|
||
|
||
def tearDown(self):
|
||
self.conn.close()
|
||
self.tmp.cleanup()
|
||
|
||
def _apply(self, kind, tid=None, pid=None):
|
||
batch = events.start_batch(self.conn, "manual")
|
||
log = events.EventLog(self.conn, self.cfg, batch, "manual")
|
||
ev = log.emit(kind, tid=tid, pid=pid, source="manual")
|
||
events.apply_event(self.conn, ev)
|
||
log.close()
|
||
self.conn.commit()
|
||
|
||
def test_restore_thread(self):
|
||
self._apply("thread_deleted", tid=1)
|
||
self.assertEqual(
|
||
self.conn.execute("SELECT status FROM post WHERE pid=100").fetchone()[0],
|
||
"recycled",
|
||
)
|
||
stats = restore.restore(self.cfg, tid=1)
|
||
self.assertEqual(stats["threads"], 1)
|
||
self.assertEqual(stats["posts"], 1)
|
||
self.assertEqual(
|
||
self.conn.execute("SELECT status FROM post WHERE pid=100").fetchone()[0],
|
||
"live",
|
||
)
|
||
self.assertIsNone(
|
||
self.conn.execute("SELECT deleted_at FROM thread WHERE tid=1").fetchone()[0]
|
||
)
|
||
kinds = [r[0] for r in self.conn.execute("SELECT kind FROM event ORDER BY id")]
|
||
self.assertIn("thread_restored", kinds)
|
||
|
||
def test_rebuild_clean(self):
|
||
self._apply("post_del", tid=1, pid=100)
|
||
stats = rebuild.check(self.cfg)
|
||
self.assertTrue(stats["idempotent"])
|
||
self.assertEqual(stats["recycled_without_event"], 0)
|
||
self.assertEqual(stats["add_events_without_post"], 0)
|
||
|
||
def test_rebuild_flags_missing_event(self):
|
||
# 手工把楼标成回收站但不发事件:rebuild 应该能发现
|
||
self.conn.execute("UPDATE post SET status='recycled' WHERE pid=100")
|
||
self.conn.commit()
|
||
stats = rebuild.check(self.cfg)
|
||
self.assertEqual(stats["recycled_without_event"], 1)
|
||
|
||
|
||
# ================================================================ ZIM
|
||
def _has_libzim() -> bool:
|
||
try:
|
||
import libzim # noqa: F401
|
||
return True
|
||
except ImportError:
|
||
return False
|
||
|
||
|
||
@unittest.skipUnless(_has_libzim(), "没装 libzim,跳过 ZIM 测试")
|
||
class ZimTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = make_temp_dir()
|
||
self.cfg = Config(kw="测试吧", data_dir=Path(self.tmp.name), rate=1000.0)
|
||
self.cfg.ensure_dirs()
|
||
conn = db.connect(self.cfg)
|
||
db.init_db(conn)
|
||
db.ensure_compat(conn)
|
||
conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
|
||
"VALUES(1, 2, '东方年代记 攻略', 2, 'done')")
|
||
for pid, floor, text in ((100, 1, "开局选东方年代记"), (101, 2, "结局怎么打")):
|
||
conn.execute(
|
||
"INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
|
||
"VALUES(?,?,?,?,?,?, '[]', 'live')",
|
||
(pid, 1, floor, "甲", "11", text),
|
||
)
|
||
conn.execute("INSERT INTO comment(cid, pid, tid, author, content, status) "
|
||
"VALUES(900, 100, 1, '乙', '楼中楼说结局', 'live')")
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def test_parse_bytes_and_split(self):
|
||
self.assertEqual(zim.parse_bytes("3.5G"), int(3.5 * 1024 ** 3))
|
||
self.assertEqual(zim.parse_bytes("100M"), 100 * 1024 ** 2)
|
||
conn = db.connect(self.cfg)
|
||
try:
|
||
conn.execute("INSERT INTO thread(tid, fid, title, reply_num, status) "
|
||
"VALUES(2, 2, '第二帖', 3, 'done')")
|
||
for pid, floor in ((200, 1), (201, 2), (202, 3)):
|
||
conn.execute("INSERT INTO post(pid, tid, floor, author, author_id, content, imgs, status) "
|
||
"VALUES(?,?,?,?,?,?, '[]', 'live')", (pid, 2, floor, "乙", "22", "x"))
|
||
conn.commit()
|
||
parts = zim.plan_split(conn, 1000) # 每帖 2~3 楼 × 300B,1000B 只放得下一帖
|
||
self.assertEqual([len(part) for part in parts], [1, 1])
|
||
finally:
|
||
conn.close()
|
||
|
||
def test_build_zim_and_search(self):
|
||
out = Path(self.tmp.name) / "test.zim"
|
||
stats = zim.build_zim(self.cfg, out, with_images=False, title="测试 ZIM")
|
||
self.assertTrue(out.exists())
|
||
self.assertGreaterEqual(stats["pages"], 2)
|
||
from libzim.reader import Archive
|
||
from libzim.search import Query, Searcher
|
||
z = Archive(str(out))
|
||
self.assertEqual(z.main_entry.get_item().path, "index.html")
|
||
page = bytes(z.get_entry_by_path("thread-1.html").get_item().content).decode("utf-8")
|
||
self.assertIn("开局选东方年代记", page)
|
||
self.assertIn("楼中楼说结局", page)
|
||
s = Searcher(z).search(Query().set_query("结局"))
|
||
self.assertGreaterEqual(s.get_estimated_matches(), 1)
|