从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
805 lines
29 KiB
Python
805 lines
29 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""aiotieba 采集器:枚举主题 + 逐帖抓楼层/楼中楼/用户资料。
|
||
|
||
只追加 JSONL,不直接写 SQLite。理由:
|
||
|
||
1. aiotieba 是 asyncio 并发,SQLite 同一时刻只允许一个写者。多个 worker 抢锁
|
||
会把「61 楼/秒」拖回个位数,还容易出现 ``database is locked``;
|
||
2. 进程被强杀时,JSONL 最多丢最后半行,而数据库事务会回滚掉一整批;
|
||
3. 抓取和导入解耦:导入逻辑改坏了可以重放 JSONL,不用重新联网;
|
||
4. 旧归档已经证明这条路可行(见旧备份 README 的 raw/ 与 raw2/)。
|
||
|
||
全局限速(:func:`pace`)是所有 worker 共享的令牌桶。aiotieba 遇到 429 时
|
||
**不会抛异常**,而是返回空结果并挂一个 ``err``,所以「空结果」必须当成
|
||
「可能被限流」处理:退避重试,连续多次空才跳过该帖。
|
||
|
||
断点续传靠 ``raw/done.txt``:一个 tid 抓完才写进去。已删帖(错误码 4)也写
|
||
done 并记 err,避免每轮重试;连续空结果、超过页数上限则**不写** done,下一轮
|
||
还会再试。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import math
|
||
import time
|
||
import traceback
|
||
from pathlib import Path
|
||
from typing import Any, Iterable
|
||
|
||
from .config import Config, read_tid_set
|
||
from .cookies import CookieError, parse_cookie_file, validate_cookies
|
||
|
||
|
||
class CrawlError(RuntimeError):
|
||
"""可以跳过当前帖、但不应该杀死整个任务的错误。"""
|
||
|
||
|
||
# ---------------------------------------------------------------- 限速
|
||
_rate = 1.3
|
||
_pace_lock: asyncio.Lock | None = None
|
||
_pace_loop: asyncio.AbstractEventLoop | None = None
|
||
_last_request = 0.0
|
||
|
||
|
||
def set_rate(rate: float) -> None:
|
||
"""设置全局限速(请求/秒)。在 asyncio.run 之前调用。"""
|
||
global _rate
|
||
if float(rate) <= 0:
|
||
raise ValueError("rate 必须大于 0")
|
||
_rate = float(rate)
|
||
|
||
|
||
async def pace() -> None:
|
||
"""全局令牌桶:所有 worker 共用一个节奏。
|
||
|
||
单个请求再快也没用,服务端看的是整体速率。用 ``time.monotonic`` 而不是
|
||
``time.time``:系统时间被同步盘/NTP 调整时不会突然睡很久。
|
||
|
||
Lock 按事件循环懒创建:测试里会连续跑多个 ``asyncio.run``,模块导入时
|
||
创建的 Lock 绑定第一个循环,第二个循环再用就会炸。
|
||
"""
|
||
global _pace_lock, _pace_loop, _last_request
|
||
loop = asyncio.get_running_loop()
|
||
if _pace_lock is None or _pace_loop is not loop:
|
||
_pace_lock = asyncio.Lock()
|
||
_pace_loop = loop
|
||
_last_request = 0.0
|
||
async with _pace_lock:
|
||
now = time.monotonic()
|
||
wait = _last_request + 1.0 / _rate - now
|
||
if wait > 0:
|
||
await asyncio.sleep(wait)
|
||
_last_request = time.monotonic()
|
||
|
||
|
||
# ---------------------------------------------------------------- JSON 化工具
|
||
def _jsonable(value: Any) -> Any:
|
||
"""把 protobuf 枚举/URL 容器之类转成 JSON 能存的东西。"""
|
||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||
return value
|
||
return str(value)
|
||
|
||
|
||
def _ts(value: Any) -> int:
|
||
try:
|
||
return int(value)
|
||
except (TypeError, ValueError):
|
||
return 0
|
||
|
||
|
||
def _user_name(user: Any) -> str:
|
||
if user is None:
|
||
return ""
|
||
return str(getattr(user, "user_name", "") or getattr(user, "nick_name_new", "") or "")
|
||
|
||
|
||
def frags_of(contents: Any) -> list[dict]:
|
||
"""把 aiotieba 的 fragment 列表转成可长期保存的 JSON 结构。
|
||
|
||
为什么原样存 fragment 而不是只存纯文本:表情、@、链接、视频/语音的
|
||
元数据(时长、封面、尺寸)丢了就再也补不回来,而纯文本随时能从 fragment
|
||
重新生成。字段名尽量短(t/src/w/h),因为一份归档里可能有上百万条。
|
||
"""
|
||
out: list[dict] = []
|
||
for frag in getattr(contents, "objs", None) or []:
|
||
name = type(frag).__name__
|
||
if name == "FragText":
|
||
out.append({"t": "text", "text": _jsonable(frag.text)})
|
||
elif name == "FragEmoji":
|
||
out.append(
|
||
{"t": "emoji", "id": _jsonable(frag.id), "desc": _jsonable(frag.desc)}
|
||
)
|
||
elif name.startswith("FragImage"):
|
||
out.append(
|
||
{
|
||
"t": "img",
|
||
"src": _jsonable(frag.src),
|
||
"big": _jsonable(getattr(frag, "big_src", "")),
|
||
"origin": _jsonable(getattr(frag, "origin_src", "")),
|
||
"hash": _jsonable(getattr(frag, "hash", "")),
|
||
"w": _jsonable(getattr(frag, "show_width", 0)),
|
||
"h": _jsonable(getattr(frag, "show_height", 0)),
|
||
}
|
||
)
|
||
elif name == "FragAt":
|
||
out.append(
|
||
{
|
||
"t": "at",
|
||
"text": _jsonable(getattr(frag, "text", "")),
|
||
"uid": _jsonable(getattr(frag, "user_id", 0)),
|
||
}
|
||
)
|
||
elif name == "FragLink":
|
||
out.append(
|
||
{
|
||
"t": "link",
|
||
"text": _jsonable(getattr(frag, "text", "")),
|
||
"title": _jsonable(getattr(frag, "title", "")),
|
||
"url": _jsonable(getattr(frag, "raw_url", "")),
|
||
}
|
||
)
|
||
elif name.startswith("FragVideo"):
|
||
out.append(
|
||
{
|
||
"t": "video",
|
||
"src": _jsonable(getattr(frag, "src", "")),
|
||
"cover": _jsonable(getattr(frag, "cover_src", "")),
|
||
"duration": _jsonable(getattr(frag, "duration", 0)),
|
||
"w": _jsonable(getattr(frag, "width", 0)),
|
||
"h": _jsonable(getattr(frag, "height", 0)),
|
||
"view": _jsonable(getattr(frag, "view_num", 0)),
|
||
}
|
||
)
|
||
elif name == "FragVoice":
|
||
out.append(
|
||
{
|
||
"t": "voice",
|
||
"md5": _jsonable(getattr(frag, "md5", "")),
|
||
"duration": _jsonable(getattr(frag, "duration", 0)),
|
||
}
|
||
)
|
||
else:
|
||
# 未知类型别丢:至少留个名字,方便以后写迁移脚本
|
||
out.append({"t": name, "raw": _jsonable(frag)})
|
||
return out
|
||
|
||
|
||
def text_of(frags: Iterable[dict]) -> str:
|
||
"""从 fragment 还原纯文本(检索/正则解析回复对象都用它)。"""
|
||
parts: list[str] = []
|
||
for frag in frags:
|
||
kind = frag.get("t")
|
||
if kind == "text":
|
||
parts.append(frag.get("text") or "")
|
||
elif kind == "at":
|
||
parts.append(frag.get("text") or "")
|
||
elif kind == "link":
|
||
parts.append(frag.get("text") or frag.get("title") or "")
|
||
return "".join(parts).strip()
|
||
|
||
|
||
def user_of(user: Any) -> dict:
|
||
"""用户资料落一份独立记录;首页/楼层只带昵称,这里才是完整信息。"""
|
||
if user is None:
|
||
return {}
|
||
return {
|
||
"uid": _jsonable(getattr(user, "user_id", 0)),
|
||
"uname": _jsonable(getattr(user, "user_name", "")),
|
||
"nick": _jsonable(getattr(user, "nick_name_new", ""))
|
||
or _jsonable(getattr(user, "user_name", "")),
|
||
"portrait": _jsonable(getattr(user, "portrait", "")),
|
||
"level": _jsonable(getattr(user, "level", 0)),
|
||
"glevel": _jsonable(getattr(user, "glevel", 0)),
|
||
"gender": _jsonable(getattr(user, "gender", 0)),
|
||
"ip": _jsonable(getattr(user, "ip", "")),
|
||
"bawu": bool(getattr(user, "is_bawu", False)),
|
||
"vip": bool(getattr(user, "is_vip", False)),
|
||
"icons": [_jsonable(x) for x in (getattr(user, "icons", None) or [])],
|
||
}
|
||
|
||
|
||
def thread_record(thread: Any) -> dict:
|
||
"""主题列表 → threads.jsonl 的一行。"""
|
||
return {
|
||
"tid": _ts(getattr(thread, "tid", 0)),
|
||
"title": _jsonable(getattr(thread, "title", "")) or "",
|
||
"author": _user_name(getattr(thread, "user", None)),
|
||
"author_id": str(getattr(thread, "author_id", 0) or ""),
|
||
"reply_num": _ts(getattr(thread, "reply_num", 0)),
|
||
"is_good": 1 if getattr(thread, "is_good", False) else 0,
|
||
"is_top": 1 if getattr(thread, "is_top", False) else 0,
|
||
"fid": _ts(getattr(thread, "fid", 0)),
|
||
}
|
||
|
||
|
||
def post_record(tid: int, post: Any) -> dict:
|
||
"""一个楼层 → posts.jsonl 的一行(字段名与旧 raw2/posts2.jsonl 一致)。"""
|
||
frags = frags_of(getattr(post, "contents", None))
|
||
return {
|
||
"tid": tid,
|
||
"pid": _jsonable(getattr(post, "pid", 0)),
|
||
"floor": _jsonable(getattr(post, "floor", 0)),
|
||
"uid": _jsonable(getattr(post, "author_id", 0)),
|
||
"uname": _user_name(getattr(post, "user", None)),
|
||
"agree": _jsonable(getattr(post, "agree", 0)),
|
||
"disagree": _jsonable(getattr(post, "disagree", 0)),
|
||
"sign": _jsonable(getattr(post, "sign", "")),
|
||
"rn": _jsonable(getattr(post, "reply_num", 0)),
|
||
"ts": _ts(getattr(post, "create_time", 0)),
|
||
"lz": bool(getattr(post, "is_thread_author", False)),
|
||
"text": text_of(frags),
|
||
"frags": frags,
|
||
}
|
||
|
||
|
||
def comment_record(tid: int, comment: Any, parent_pid: int = 0) -> dict:
|
||
"""一条楼中楼 → comments.jsonl 的一行。
|
||
|
||
``ppid`` 是所在楼层的 pid;协议偶尔不给(0),就用父楼层补上。cid 存在
|
||
``pid`` 字段里,和旧 raw2/comments2.jsonl 保持一致。
|
||
"""
|
||
frags = frags_of(getattr(comment, "contents", None))
|
||
return {
|
||
"tid": tid,
|
||
"pid": _jsonable(getattr(comment, "pid", 0)),
|
||
"ppid": _jsonable(getattr(comment, "ppid", parent_pid) or parent_pid),
|
||
"floor": _jsonable(getattr(comment, "floor", 0)),
|
||
"uid": _jsonable(getattr(comment, "author_id", 0)),
|
||
"uname": _user_name(getattr(comment, "user", None)),
|
||
"rto": _jsonable(getattr(comment, "reply_to_id", 0)),
|
||
"agree": _jsonable(getattr(comment, "agree", 0)),
|
||
"disagree": _jsonable(getattr(comment, "disagree", 0)),
|
||
"ts": _ts(getattr(comment, "create_time", 0)),
|
||
"lz": bool(getattr(comment, "is_thread_author", False)),
|
||
"text": text_of(frags),
|
||
"frags": frags,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------- 落盘
|
||
def _dumps(record: dict) -> str:
|
||
return json.dumps(record, ensure_ascii=False) + "\n"
|
||
|
||
|
||
class JsonlWriter:
|
||
"""追加写 raw/*.jsonl。
|
||
|
||
用户资料先攒在内存:每层/每条楼中楼都带作者信息,如果每条都写一行,
|
||
users.jsonl 会膨胀到几百万行且大量重复。攒够 500 个不同用户、或距上次
|
||
落盘超过 60 秒,就批量写一次——被强杀最多丢这一分钟的新用户资料,
|
||
下次抓帖还会再遇到他们。
|
||
"""
|
||
|
||
def __init__(self, cfg: Config):
|
||
cfg.raw_dir.mkdir(parents=True, exist_ok=True)
|
||
self.cfg = cfg
|
||
self._posts = open(cfg.posts_path, "a", encoding="utf-8")
|
||
self._comments = open(cfg.comments_path, "a", encoding="utf-8")
|
||
self._users = open(cfg.users_path, "a", encoding="utf-8")
|
||
self._done = open(cfg.done_path, "a", encoding="utf-8")
|
||
self._err = open(cfg.err_path, "a", encoding="utf-8")
|
||
self.n_posts = 0
|
||
self.n_comments = 0
|
||
self.n_users = 0
|
||
self.users: dict[Any, dict] = {}
|
||
self._last_dump = time.monotonic()
|
||
self._since_flush = 0
|
||
|
||
# -- 楼层 / 楼中楼 -------------------------------------------------
|
||
def post(self, record: dict) -> None:
|
||
self._posts.write(_dumps(record))
|
||
self.n_posts += 1
|
||
self._tick()
|
||
|
||
def comment(self, record: dict) -> None:
|
||
self._comments.write(_dumps(record))
|
||
self.n_comments += 1
|
||
self._tick()
|
||
|
||
# -- 用户 ----------------------------------------------------------
|
||
def user(self, record: dict) -> None:
|
||
uid = record.get("uid")
|
||
if not uid or uid in self.users:
|
||
return
|
||
self.users[uid] = record
|
||
if (
|
||
len(self.users) >= self.cfg.user_dump_every
|
||
or time.monotonic() - self._last_dump >= self.cfg.user_dump_seconds
|
||
):
|
||
self.dump_users()
|
||
|
||
def dump_users(self) -> None:
|
||
if not self.users:
|
||
return
|
||
for record in self.users.values():
|
||
self._users.write(_dumps(record))
|
||
self.n_users += 1
|
||
self.users.clear()
|
||
self._users.flush()
|
||
self._last_dump = time.monotonic()
|
||
|
||
# -- 断点 / 错误 ----------------------------------------------------
|
||
def finish_thread(self, tid: int) -> None:
|
||
"""先 flush 数据再记 done。
|
||
|
||
反过来的话,进程在两者之间被强杀,下次会以为这帖抓完了,
|
||
实际 JSONL 缓冲区里还有几百楼没落盘。
|
||
"""
|
||
self.flush()
|
||
self._done.write(f"{int(tid)}\n")
|
||
self._done.flush()
|
||
|
||
def err(self, record: dict) -> None:
|
||
self._err.write(_dumps(record))
|
||
self._err.flush()
|
||
|
||
# -- 刷盘 / 关闭 ----------------------------------------------------
|
||
def _tick(self) -> None:
|
||
self._since_flush += 1
|
||
if self._since_flush >= self.cfg.flush_every:
|
||
self.flush()
|
||
|
||
def flush(self) -> None:
|
||
self._posts.flush()
|
||
self._comments.flush()
|
||
self._since_flush = 0
|
||
|
||
def close(self) -> None:
|
||
try:
|
||
self.dump_users()
|
||
finally:
|
||
for handle in (self._posts, self._comments, self._users, self._done, self._err):
|
||
try:
|
||
handle.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
# ---------------------------------------------------------------- 认证 / 客户端
|
||
def read_auth(cfg: Config) -> tuple[str, str]:
|
||
"""从 ``<data>/cookies.txt`` 读 BDUSS/STOKEN。
|
||
|
||
只认这个位置:命令行的 ``--data`` 已经把「用哪套 Cookie」表达清楚了,
|
||
再允许随便指路径,容易出现抓 A 吧用 B 号的情况。
|
||
"""
|
||
cookies = parse_cookie_file(cfg.cookies_path)
|
||
if not cookies:
|
||
raise CookieError(
|
||
f"没有可用的 Cookie:{cfg.cookies_path}\n"
|
||
f"把浏览器里的 Cookie 整条粘进这个文件,再跑 `python tieba.py check-cookie --data {cfg.data_dir}`"
|
||
)
|
||
warnings = validate_cookies(cookies)
|
||
for warning in warnings:
|
||
print(f"[!] {warning}")
|
||
return cookies["BDUSS"], cookies.get("STOKEN", "")
|
||
|
||
|
||
def require_aiotieba():
|
||
"""延迟导入:``--help`` / ``stats`` / ``import`` 不需要 aiotieba 也能跑。"""
|
||
try:
|
||
import aiotieba # noqa: F401
|
||
except ImportError as exc: # pragma: no cover - 只在没装依赖时触发
|
||
raise RuntimeError(
|
||
"采集需要 aiotieba:`pip install -r requirements.txt`"
|
||
) from exc
|
||
return aiotieba
|
||
|
||
|
||
def new_client(aiotieba, cfg: Config, bduss: str, stoken: str):
|
||
from aiotieba.config import TimeoutConfig
|
||
|
||
timeout = TimeoutConfig(
|
||
http_read=float(cfg.timeout), http_connect=min(float(cfg.timeout), 10.0)
|
||
)
|
||
return aiotieba.Client(BDUSS=bduss, STOKEN=stoken or "", timeout=timeout)
|
||
|
||
|
||
# ---------------------------------------------------------------- 枚举
|
||
def load_threads(path: Path) -> list[dict]:
|
||
"""读 threads.jsonl,坏行跳过;同一个 tid 只保留第一次出现。"""
|
||
if not path.exists():
|
||
raise FileNotFoundError(
|
||
f"缺少主题清单:{path}\n先跑 `python tieba.py enum --kw <吧名> --pages 1` 试水,"
|
||
f"或去掉 --pages 全量枚举"
|
||
)
|
||
rows: list[dict] = []
|
||
seen: set[int] = set()
|
||
bad = 0
|
||
with open(path, encoding="utf-8", errors="ignore") as fh:
|
||
for line in fh:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
row = json.loads(line)
|
||
tid = int(row["tid"])
|
||
except (json.JSONDecodeError, KeyError, TypeError, ValueError):
|
||
bad += 1
|
||
continue
|
||
if tid in seen:
|
||
continue
|
||
seen.add(tid)
|
||
row["tid"] = tid
|
||
rows.append(row)
|
||
if bad:
|
||
print(f"[!] threads.jsonl 有 {bad} 行无法解析,已跳过")
|
||
return rows
|
||
|
||
|
||
async def enumerate_with_client(client, cfg: Config, pages: int = 0) -> dict:
|
||
"""枚举主题列表,追加写 threads.jsonl;返回统计。
|
||
|
||
每次都从第 1 页开始扫,靠 tid 去重。为什么不从「已有条数/30」续:
|
||
主题列表按最后回复时间排序,删帖/新回复都会让分页漂移,用页码当游标
|
||
会漏帖。重扫的代价只是多花十几分钟请求。
|
||
"""
|
||
set_rate(cfg.rate)
|
||
cfg.ensure_dirs()
|
||
|
||
existing: set[int] = set()
|
||
if cfg.threads_path.exists():
|
||
try:
|
||
existing = {int(row["tid"]) for row in load_threads(cfg.threads_path)}
|
||
except (FileNotFoundError, KeyError, TypeError, ValueError):
|
||
existing = set()
|
||
|
||
try:
|
||
forum = await client.get_forum(cfg.kw)
|
||
except Exception as exc:
|
||
raise CrawlError(
|
||
f"获取贴吧信息失败(吧名或 Cookie 可能不对):{type(exc).__name__}: {exc}"
|
||
) from exc
|
||
|
||
fid = _ts(getattr(forum, "fid", 0))
|
||
thread_num = _ts(getattr(forum, "thread_num", 0))
|
||
fname = str(getattr(forum, "fname", "") or cfg.kw)
|
||
print(f"[i] {fname} fid={fid} 主题={thread_num:,}")
|
||
|
||
expected_pages = (
|
||
math.ceil(thread_num / cfg.thread_rn) if thread_num and cfg.thread_rn else 0
|
||
)
|
||
added = seen = page_no = 0
|
||
pn = 1
|
||
|
||
with open(cfg.threads_path, "a", encoding="utf-8") as fh:
|
||
while True:
|
||
page_no += 1
|
||
page = await _enum_page(client, cfg, pn, expected_pages)
|
||
objs = getattr(page, "objs", None) or []
|
||
if not objs:
|
||
print(f"\n[i] 第 {pn} 页为空,枚举结束(已扫 {page_no - 1} 页)")
|
||
page_no -= 1
|
||
break
|
||
|
||
for thread in objs:
|
||
seen += 1
|
||
record = thread_record(thread)
|
||
if not record["tid"] or record["tid"] in existing:
|
||
continue
|
||
existing.add(record["tid"])
|
||
fh.write(_dumps(record))
|
||
added += 1
|
||
fh.flush()
|
||
|
||
total = getattr(getattr(page, "page", None), "total_page", 0) or expected_pages
|
||
print(
|
||
f"\r 第 {page_no} 页 pn={pn}/{total or '?'} 本页 {len(objs)},"
|
||
f"新增 {added:,},累计 {len(existing):,}",
|
||
end="",
|
||
flush=True,
|
||
)
|
||
|
||
if pages and page_no >= pages:
|
||
break
|
||
if not getattr(getattr(page, "page", None), "has_more", False):
|
||
break
|
||
pn += 1
|
||
|
||
print()
|
||
if fid or fname:
|
||
cfg.fid = fid or cfg.fid
|
||
cfg.forum_name = fname or cfg.forum_name
|
||
cfg.save()
|
||
print(f"[i] 本次扫 {page_no} 页 / 看到 {seen:,} 个主题,新增 {added:,},清单共 {len(existing):,}")
|
||
return {
|
||
"pages": page_no,
|
||
"seen": seen,
|
||
"added": added,
|
||
"fid": cfg.fid,
|
||
"threads": len(existing),
|
||
}
|
||
|
||
|
||
async def _enum_page(client, cfg: Config, pn: int, expected_pages: int):
|
||
"""取一页主题;空结果按限流处理,退避重试。"""
|
||
tries = 0
|
||
while True:
|
||
await pace()
|
||
page = await client.get_threads(cfg.kw, pn=pn, rn=cfg.thread_rn)
|
||
if getattr(page, "objs", None):
|
||
return page
|
||
err = getattr(page, "err", None)
|
||
code = getattr(err, "code", None)
|
||
if code == 4:
|
||
raise CrawlError(f"贴吧「{cfg.kw}」不存在或无法访问(错误码 4)")
|
||
tries += 1
|
||
if tries >= cfg.tries:
|
||
return page
|
||
wait = cfg.backoff * tries
|
||
print(
|
||
f"\n [!] 枚举第 {pn} 页空结果({tries}/{cfg.tries}),{wait:.0f}s 后重试",
|
||
flush=True,
|
||
)
|
||
await asyncio.sleep(wait)
|
||
|
||
|
||
async def enum(cfg: Config, pages: int = 0) -> dict:
|
||
"""联网枚举主题。``pages=0`` 表示一直翻到没有下一页。"""
|
||
aiotieba = require_aiotieba()
|
||
bduss, stoken = read_auth(cfg)
|
||
async with new_client(aiotieba, cfg, bduss, stoken) as client:
|
||
return await enumerate_with_client(client, cfg, pages)
|
||
|
||
|
||
# ---------------------------------------------------------------- 抓帖
|
||
async def _fetch_comments(
|
||
client, cfg: Config, writer: JsonlWriter, tid: int, pid: int, stats: dict, lock: asyncio.Lock
|
||
) -> None:
|
||
"""楼中楼超过一次返回数量(默认一次带 50 条)时翻页补齐。
|
||
|
||
这一路能拿到每条的 agree/disagree;主页接口内联返回的那几十条是不带
|
||
这些字段的,所以「超量」的楼必须单独走这里。
|
||
"""
|
||
pn = 1
|
||
while True:
|
||
await pace()
|
||
result = await client.get_comments(tid, pid, pn=pn)
|
||
objs = getattr(result, "objs", None) or []
|
||
if not objs:
|
||
return
|
||
async with lock:
|
||
for comment in objs:
|
||
writer.comment(comment_record(tid, comment, pid))
|
||
writer.user(user_of(getattr(comment, "user", None)))
|
||
pn += 1
|
||
if not getattr(getattr(result, "page", None), "has_more", False):
|
||
return
|
||
if pn > cfg.comment_page_cap:
|
||
return
|
||
|
||
|
||
def _is_deleted(page: Any) -> bool:
|
||
"""帖子已删:aiotieba 把错误挂在 ``page.err`` 上(code=4),不抛异常。"""
|
||
err = getattr(page, "err", None)
|
||
if getattr(err, "code", None) == 4:
|
||
return True
|
||
# 少数版本把 err 做成 (code, msg) 元组
|
||
if isinstance(err, tuple) and err and err[0] == 4:
|
||
return True
|
||
return False
|
||
|
||
|
||
async def crawl_thread(
|
||
client,
|
||
cfg: Config,
|
||
writer: JsonlWriter,
|
||
row: dict,
|
||
stats: dict,
|
||
lock: asyncio.Lock,
|
||
) -> None:
|
||
"""抓一个主题的全部楼层。
|
||
|
||
成功才写 done.txt。任何「没抓全」的路径(空结果、超页数上限、异常)
|
||
都抛 :class:`CrawlError` 或原异常,留给 worker 记 err.jsonl 并跳过。
|
||
"""
|
||
tid = int(row["tid"])
|
||
pn = 1
|
||
tries = 0
|
||
while True:
|
||
if pn > cfg.page_cap:
|
||
raise CrawlError(
|
||
f"tid={tid} 超过页数上限 {cfg.page_cap}(config.json 的 page_cap),本轮跳过"
|
||
)
|
||
|
||
await pace()
|
||
page = await client.get_posts(
|
||
tid,
|
||
pn=pn,
|
||
rn=cfg.post_rn,
|
||
with_comments=True,
|
||
comment_rn=cfg.comment_rn,
|
||
)
|
||
posts = getattr(page, "objs", None) or []
|
||
|
||
if not posts:
|
||
if _is_deleted(page):
|
||
# 已删帖:没东西可抓,记一笔然后算完成,否则每轮都会重试
|
||
async with lock:
|
||
writer.err(
|
||
{
|
||
"tid": tid,
|
||
"err": "deleted: 帖子已被删除",
|
||
"code": 4,
|
||
"ts": int(time.time()),
|
||
}
|
||
)
|
||
writer.finish_thread(tid)
|
||
stats["deleted"] += 1
|
||
return
|
||
# 没有报错、服务端明确说没有下一页:就是真的翻到底了(末页可能不足 rn 楼)。
|
||
# 这类帖以前会被当成 429 一直重试,最后永远抓不完。
|
||
if (
|
||
getattr(page, "err", None) is None
|
||
and getattr(getattr(page, "page", None), "has_more", None) is False
|
||
):
|
||
break
|
||
tries += 1
|
||
if tries >= cfg.tries:
|
||
raise CrawlError(
|
||
f"tid={tid} 第 {pn} 页连续 {cfg.tries} 次空结果,跳过(不写 done)"
|
||
)
|
||
wait = cfg.backoff * tries
|
||
print(
|
||
f"\n [!] tid={tid} 第 {pn} 页空结果({tries}/{cfg.tries}),{wait:.0f}s 后重试",
|
||
flush=True,
|
||
)
|
||
await asyncio.sleep(wait)
|
||
continue
|
||
|
||
tries = 0
|
||
overflow: list[int] = []
|
||
async with lock:
|
||
for post in posts:
|
||
writer.post(post_record(tid, post))
|
||
writer.user(user_of(getattr(post, "user", None)))
|
||
|
||
for comment in getattr(post, "comments", None) or []:
|
||
writer.comment(comment_record(tid, comment, int(getattr(post, "pid", 0))))
|
||
writer.user(user_of(getattr(comment, "user", None)))
|
||
|
||
reply_num = _ts(getattr(post, "reply_num", 0))
|
||
inline = len(getattr(post, "comments", None) or [])
|
||
if reply_num > inline:
|
||
overflow.append(int(getattr(post, "pid", 0)))
|
||
stats["overflow"] += len(overflow)
|
||
|
||
# 补抓是「尽力而为」:失败只计数,不因此否定整帖(主页数据已经拿到了)
|
||
for pid in overflow:
|
||
try:
|
||
await _fetch_comments(client, cfg, writer, tid, pid, stats, lock)
|
||
except Exception:
|
||
async with lock:
|
||
stats["overflow_err"] += 1
|
||
|
||
pn += 1
|
||
if not getattr(getattr(page, "page", None), "has_more", False):
|
||
break
|
||
|
||
async with lock:
|
||
writer.finish_thread(tid)
|
||
stats["threads"] += 1
|
||
|
||
|
||
async def crawl(cfg: Config, threads: int = 0, workers: int | None = None) -> dict:
|
||
"""抓帖入口:读 threads.jsonl 与 done.txt,抓没抓过的。
|
||
|
||
``threads=0`` 表示全部未完成;``workers`` 只覆盖本次运行,不写回配置。
|
||
"""
|
||
aiotieba = require_aiotieba()
|
||
bduss, stoken = read_auth(cfg)
|
||
|
||
cfg.workers = int(workers or cfg.workers)
|
||
cfg.validate()
|
||
set_rate(cfg.rate)
|
||
|
||
done = read_tid_set(cfg.done_path)
|
||
rows: list[dict] = []
|
||
for row in load_threads(cfg.threads_path):
|
||
if row["tid"] in done:
|
||
continue
|
||
rows.append(row)
|
||
if threads and len(rows) >= threads:
|
||
break
|
||
|
||
print(f"[i] 已抓完 {len(done):,} 帖,本次待抓 {len(rows):,},并发 {cfg.workers},限速 {cfg.rate}/s")
|
||
if not rows:
|
||
print("[i] 没有待抓主题。清单抓完了,或者先跑 enum 生成 threads.jsonl")
|
||
return {
|
||
"threads": 0,
|
||
"deleted": 0,
|
||
"skipped": 0,
|
||
"errors": 0,
|
||
"posts": 0,
|
||
"comments": 0,
|
||
"users": 0,
|
||
"overflow": 0,
|
||
}
|
||
|
||
writer = JsonlWriter(cfg)
|
||
stats = {
|
||
"threads": 0,
|
||
"deleted": 0,
|
||
"skipped": 0,
|
||
"errors": 0,
|
||
"overflow": 0,
|
||
"overflow_err": 0,
|
||
}
|
||
lock = asyncio.Lock()
|
||
sem = asyncio.Semaphore(cfg.workers)
|
||
started = time.time()
|
||
|
||
async def worker(row: dict) -> None:
|
||
async with sem:
|
||
try:
|
||
await crawl_thread(client, cfg, writer, row, stats, lock)
|
||
except CrawlError as exc:
|
||
# 可跳过的失败:没写 done,下一轮还会再试
|
||
async with lock:
|
||
stats["skipped"] += 1
|
||
writer.err(
|
||
{
|
||
"tid": int(row["tid"]),
|
||
"err": str(exc),
|
||
"ts": int(time.time()),
|
||
}
|
||
)
|
||
if stats["skipped"] % 20 == 1:
|
||
print(f"\n [!] {exc}", flush=True)
|
||
except Exception as exc:
|
||
async with lock:
|
||
stats["errors"] += 1
|
||
writer.err(
|
||
{
|
||
"tid": int(row["tid"]),
|
||
"err": f"{type(exc).__name__}: {exc}",
|
||
"tb": traceback.format_exc()[-800:],
|
||
"ts": int(time.time()),
|
||
}
|
||
)
|
||
if stats["errors"] % 20 == 1:
|
||
print(
|
||
f"\n [!] tid={row['tid']} {type(exc).__name__}: {exc}",
|
||
flush=True,
|
||
)
|
||
finally:
|
||
async with lock:
|
||
finished = stats["threads"] + stats["deleted"] + stats["skipped"] + stats["errors"]
|
||
elapsed = max(time.time() - started, 1.0)
|
||
if finished % 50 == 0:
|
||
print(
|
||
f"\r {finished}/{len(rows)} 帖 楼 {writer.n_posts:,} "
|
||
f"楼中楼 {writer.n_comments:,} 失败 {stats['skipped'] + stats['errors']} "
|
||
f"{(finished / elapsed) * 60:.0f} 帖/分",
|
||
end="",
|
||
flush=True,
|
||
)
|
||
|
||
async with new_client(aiotieba, cfg, bduss, stoken) as client:
|
||
try:
|
||
forum = await client.get_forum(cfg.kw)
|
||
print(f"[i] {getattr(forum, 'fname', cfg.kw)} fid={getattr(forum, 'fid', 0)}")
|
||
except Exception as exc:
|
||
writer.close()
|
||
raise CrawlError(
|
||
f"登录或吧名有误:{type(exc).__name__}: {exc}"
|
||
) from exc
|
||
|
||
try:
|
||
await asyncio.gather(*(worker(row) for row in rows))
|
||
finally:
|
||
writer.close()
|
||
|
||
elapsed = time.time() - started
|
||
print(
|
||
f"\n[i] 本次完成 {stats['threads']:,} 帖(已删 {stats['deleted']} / 跳过 {stats['skipped']} / "
|
||
f"异常 {stats['errors']}),楼 {writer.n_posts:,} / 楼中楼 {writer.n_comments:,},"
|
||
f"用户 {writer.n_users:,},用时 {elapsed / 60:.1f} 分钟"
|
||
)
|
||
return {
|
||
**stats,
|
||
"posts": writer.n_posts,
|
||
"comments": writer.n_comments,
|
||
"users": writer.n_users,
|
||
}
|