Files
dsh 0824e0ef83 v1.2.0: 事件流对账、增量 update、ZIM 发布与表情渲染
从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
2026-11-18 11:41:25 +08:00

805 lines
29 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""aiotieba 采集器:枚举主题 + 逐帖抓楼层/楼中楼/用户资料。
只追加 JSONL,不直接写 SQLite。理由:
1. aiotieba 是 asyncio 并发,SQLite 同一时刻只允许一个写者。多个 worker 抢锁
会把「61 楼/秒」拖回个位数,还容易出现 ``database is locked``;
2. 进程被强杀时,JSONL 最多丢最后半行,而数据库事务会回滚掉一整批;
3. 抓取和导入解耦:导入逻辑改坏了可以重放 JSONL,不用重新联网;
4. 旧归档已经证明这条路可行(见旧备份 README 的 raw/ 与 raw2/)。
全局限速(:func:`pace`)是所有 worker 共享的令牌桶。aiotieba 遇到 429 时
**不会抛异常**,而是返回空结果并挂一个 ``err``,所以「空结果」必须当成
「可能被限流」处理:退避重试,连续多次空才跳过该帖。
断点续传靠 ``raw/done.txt``:一个 tid 抓完才写进去。已删帖(错误码 4)也写
done 并记 err,避免每轮重试;连续空结果、超过页数上限则**不写** done,下一轮
还会再试。
"""
from __future__ import annotations
import asyncio
import json
import math
import time
import traceback
from pathlib import Path
from typing import Any, Iterable
from .config import Config, read_tid_set
from .cookies import CookieError, parse_cookie_file, validate_cookies
class CrawlError(RuntimeError):
"""可以跳过当前帖、但不应该杀死整个任务的错误。"""
# ---------------------------------------------------------------- 限速
_rate = 1.3
_pace_lock: asyncio.Lock | None = None
_pace_loop: asyncio.AbstractEventLoop | None = None
_last_request = 0.0
def set_rate(rate: float) -> None:
"""设置全局限速(请求/秒)。在 asyncio.run 之前调用。"""
global _rate
if float(rate) <= 0:
raise ValueError("rate 必须大于 0")
_rate = float(rate)
async def pace() -> None:
"""全局令牌桶:所有 worker 共用一个节奏。
单个请求再快也没用,服务端看的是整体速率。用 ``time.monotonic`` 而不是
``time.time``:系统时间被同步盘/NTP 调整时不会突然睡很久。
Lock 按事件循环懒创建:测试里会连续跑多个 ``asyncio.run``,模块导入时
创建的 Lock 绑定第一个循环,第二个循环再用就会炸。
"""
global _pace_lock, _pace_loop, _last_request
loop = asyncio.get_running_loop()
if _pace_lock is None or _pace_loop is not loop:
_pace_lock = asyncio.Lock()
_pace_loop = loop
_last_request = 0.0
async with _pace_lock:
now = time.monotonic()
wait = _last_request + 1.0 / _rate - now
if wait > 0:
await asyncio.sleep(wait)
_last_request = time.monotonic()
# ---------------------------------------------------------------- JSON 化工具
def _jsonable(value: Any) -> Any:
"""把 protobuf 枚举/URL 容器之类转成 JSON 能存的东西。"""
if isinstance(value, (str, int, float, bool)) or value is None:
return value
return str(value)
def _ts(value: Any) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def _user_name(user: Any) -> str:
if user is None:
return ""
return str(getattr(user, "user_name", "") or getattr(user, "nick_name_new", "") or "")
def frags_of(contents: Any) -> list[dict]:
"""把 aiotieba 的 fragment 列表转成可长期保存的 JSON 结构。
为什么原样存 fragment 而不是只存纯文本:表情、@、链接、视频/语音的
元数据(时长、封面、尺寸)丢了就再也补不回来,而纯文本随时能从 fragment
重新生成。字段名尽量短(t/src/w/h),因为一份归档里可能有上百万条。
"""
out: list[dict] = []
for frag in getattr(contents, "objs", None) or []:
name = type(frag).__name__
if name == "FragText":
out.append({"t": "text", "text": _jsonable(frag.text)})
elif name == "FragEmoji":
out.append(
{"t": "emoji", "id": _jsonable(frag.id), "desc": _jsonable(frag.desc)}
)
elif name.startswith("FragImage"):
out.append(
{
"t": "img",
"src": _jsonable(frag.src),
"big": _jsonable(getattr(frag, "big_src", "")),
"origin": _jsonable(getattr(frag, "origin_src", "")),
"hash": _jsonable(getattr(frag, "hash", "")),
"w": _jsonable(getattr(frag, "show_width", 0)),
"h": _jsonable(getattr(frag, "show_height", 0)),
}
)
elif name == "FragAt":
out.append(
{
"t": "at",
"text": _jsonable(getattr(frag, "text", "")),
"uid": _jsonable(getattr(frag, "user_id", 0)),
}
)
elif name == "FragLink":
out.append(
{
"t": "link",
"text": _jsonable(getattr(frag, "text", "")),
"title": _jsonable(getattr(frag, "title", "")),
"url": _jsonable(getattr(frag, "raw_url", "")),
}
)
elif name.startswith("FragVideo"):
out.append(
{
"t": "video",
"src": _jsonable(getattr(frag, "src", "")),
"cover": _jsonable(getattr(frag, "cover_src", "")),
"duration": _jsonable(getattr(frag, "duration", 0)),
"w": _jsonable(getattr(frag, "width", 0)),
"h": _jsonable(getattr(frag, "height", 0)),
"view": _jsonable(getattr(frag, "view_num", 0)),
}
)
elif name == "FragVoice":
out.append(
{
"t": "voice",
"md5": _jsonable(getattr(frag, "md5", "")),
"duration": _jsonable(getattr(frag, "duration", 0)),
}
)
else:
# 未知类型别丢:至少留个名字,方便以后写迁移脚本
out.append({"t": name, "raw": _jsonable(frag)})
return out
def text_of(frags: Iterable[dict]) -> str:
"""从 fragment 还原纯文本(检索/正则解析回复对象都用它)。"""
parts: list[str] = []
for frag in frags:
kind = frag.get("t")
if kind == "text":
parts.append(frag.get("text") or "")
elif kind == "at":
parts.append(frag.get("text") or "")
elif kind == "link":
parts.append(frag.get("text") or frag.get("title") or "")
return "".join(parts).strip()
def user_of(user: Any) -> dict:
"""用户资料落一份独立记录;首页/楼层只带昵称,这里才是完整信息。"""
if user is None:
return {}
return {
"uid": _jsonable(getattr(user, "user_id", 0)),
"uname": _jsonable(getattr(user, "user_name", "")),
"nick": _jsonable(getattr(user, "nick_name_new", ""))
or _jsonable(getattr(user, "user_name", "")),
"portrait": _jsonable(getattr(user, "portrait", "")),
"level": _jsonable(getattr(user, "level", 0)),
"glevel": _jsonable(getattr(user, "glevel", 0)),
"gender": _jsonable(getattr(user, "gender", 0)),
"ip": _jsonable(getattr(user, "ip", "")),
"bawu": bool(getattr(user, "is_bawu", False)),
"vip": bool(getattr(user, "is_vip", False)),
"icons": [_jsonable(x) for x in (getattr(user, "icons", None) or [])],
}
def thread_record(thread: Any) -> dict:
"""主题列表 → threads.jsonl 的一行。"""
return {
"tid": _ts(getattr(thread, "tid", 0)),
"title": _jsonable(getattr(thread, "title", "")) or "",
"author": _user_name(getattr(thread, "user", None)),
"author_id": str(getattr(thread, "author_id", 0) or ""),
"reply_num": _ts(getattr(thread, "reply_num", 0)),
"is_good": 1 if getattr(thread, "is_good", False) else 0,
"is_top": 1 if getattr(thread, "is_top", False) else 0,
"fid": _ts(getattr(thread, "fid", 0)),
}
def post_record(tid: int, post: Any) -> dict:
"""一个楼层 → posts.jsonl 的一行(字段名与旧 raw2/posts2.jsonl 一致)。"""
frags = frags_of(getattr(post, "contents", None))
return {
"tid": tid,
"pid": _jsonable(getattr(post, "pid", 0)),
"floor": _jsonable(getattr(post, "floor", 0)),
"uid": _jsonable(getattr(post, "author_id", 0)),
"uname": _user_name(getattr(post, "user", None)),
"agree": _jsonable(getattr(post, "agree", 0)),
"disagree": _jsonable(getattr(post, "disagree", 0)),
"sign": _jsonable(getattr(post, "sign", "")),
"rn": _jsonable(getattr(post, "reply_num", 0)),
"ts": _ts(getattr(post, "create_time", 0)),
"lz": bool(getattr(post, "is_thread_author", False)),
"text": text_of(frags),
"frags": frags,
}
def comment_record(tid: int, comment: Any, parent_pid: int = 0) -> dict:
"""一条楼中楼 → comments.jsonl 的一行。
``ppid`` 是所在楼层的 pid;协议偶尔不给(0),就用父楼层补上。cid 存在
``pid`` 字段里,和旧 raw2/comments2.jsonl 保持一致。
"""
frags = frags_of(getattr(comment, "contents", None))
return {
"tid": tid,
"pid": _jsonable(getattr(comment, "pid", 0)),
"ppid": _jsonable(getattr(comment, "ppid", parent_pid) or parent_pid),
"floor": _jsonable(getattr(comment, "floor", 0)),
"uid": _jsonable(getattr(comment, "author_id", 0)),
"uname": _user_name(getattr(comment, "user", None)),
"rto": _jsonable(getattr(comment, "reply_to_id", 0)),
"agree": _jsonable(getattr(comment, "agree", 0)),
"disagree": _jsonable(getattr(comment, "disagree", 0)),
"ts": _ts(getattr(comment, "create_time", 0)),
"lz": bool(getattr(comment, "is_thread_author", False)),
"text": text_of(frags),
"frags": frags,
}
# ---------------------------------------------------------------- 落盘
def _dumps(record: dict) -> str:
return json.dumps(record, ensure_ascii=False) + "\n"
class JsonlWriter:
"""追加写 raw/*.jsonl。
用户资料先攒在内存:每层/每条楼中楼都带作者信息,如果每条都写一行,
users.jsonl 会膨胀到几百万行且大量重复。攒够 500 个不同用户、或距上次
落盘超过 60 秒,就批量写一次——被强杀最多丢这一分钟的新用户资料,
下次抓帖还会再遇到他们。
"""
def __init__(self, cfg: Config):
cfg.raw_dir.mkdir(parents=True, exist_ok=True)
self.cfg = cfg
self._posts = open(cfg.posts_path, "a", encoding="utf-8")
self._comments = open(cfg.comments_path, "a", encoding="utf-8")
self._users = open(cfg.users_path, "a", encoding="utf-8")
self._done = open(cfg.done_path, "a", encoding="utf-8")
self._err = open(cfg.err_path, "a", encoding="utf-8")
self.n_posts = 0
self.n_comments = 0
self.n_users = 0
self.users: dict[Any, dict] = {}
self._last_dump = time.monotonic()
self._since_flush = 0
# -- 楼层 / 楼中楼 -------------------------------------------------
def post(self, record: dict) -> None:
self._posts.write(_dumps(record))
self.n_posts += 1
self._tick()
def comment(self, record: dict) -> None:
self._comments.write(_dumps(record))
self.n_comments += 1
self._tick()
# -- 用户 ----------------------------------------------------------
def user(self, record: dict) -> None:
uid = record.get("uid")
if not uid or uid in self.users:
return
self.users[uid] = record
if (
len(self.users) >= self.cfg.user_dump_every
or time.monotonic() - self._last_dump >= self.cfg.user_dump_seconds
):
self.dump_users()
def dump_users(self) -> None:
if not self.users:
return
for record in self.users.values():
self._users.write(_dumps(record))
self.n_users += 1
self.users.clear()
self._users.flush()
self._last_dump = time.monotonic()
# -- 断点 / 错误 ----------------------------------------------------
def finish_thread(self, tid: int) -> None:
"""先 flush 数据再记 done。
反过来的话,进程在两者之间被强杀,下次会以为这帖抓完了,
实际 JSONL 缓冲区里还有几百楼没落盘。
"""
self.flush()
self._done.write(f"{int(tid)}\n")
self._done.flush()
def err(self, record: dict) -> None:
self._err.write(_dumps(record))
self._err.flush()
# -- 刷盘 / 关闭 ----------------------------------------------------
def _tick(self) -> None:
self._since_flush += 1
if self._since_flush >= self.cfg.flush_every:
self.flush()
def flush(self) -> None:
self._posts.flush()
self._comments.flush()
self._since_flush = 0
def close(self) -> None:
try:
self.dump_users()
finally:
for handle in (self._posts, self._comments, self._users, self._done, self._err):
try:
handle.close()
except Exception:
pass
# ---------------------------------------------------------------- 认证 / 客户端
def read_auth(cfg: Config) -> tuple[str, str]:
"""从 ``<data>/cookies.txt`` 读 BDUSS/STOKEN。
只认这个位置:命令行的 ``--data`` 已经把「用哪套 Cookie」表达清楚了,
再允许随便指路径,容易出现抓 A 吧用 B 号的情况。
"""
cookies = parse_cookie_file(cfg.cookies_path)
if not cookies:
raise CookieError(
f"没有可用的 Cookie:{cfg.cookies_path}\n"
f"把浏览器里的 Cookie 整条粘进这个文件,再跑 `python tieba.py check-cookie --data {cfg.data_dir}`"
)
warnings = validate_cookies(cookies)
for warning in warnings:
print(f"[!] {warning}")
return cookies["BDUSS"], cookies.get("STOKEN", "")
def require_aiotieba():
"""延迟导入:``--help`` / ``stats`` / ``import`` 不需要 aiotieba 也能跑。"""
try:
import aiotieba # noqa: F401
except ImportError as exc: # pragma: no cover - 只在没装依赖时触发
raise RuntimeError(
"采集需要 aiotieba:`pip install -r requirements.txt`"
) from exc
return aiotieba
def new_client(aiotieba, cfg: Config, bduss: str, stoken: str):
from aiotieba.config import TimeoutConfig
timeout = TimeoutConfig(
http_read=float(cfg.timeout), http_connect=min(float(cfg.timeout), 10.0)
)
return aiotieba.Client(BDUSS=bduss, STOKEN=stoken or "", timeout=timeout)
# ---------------------------------------------------------------- 枚举
def load_threads(path: Path) -> list[dict]:
"""读 threads.jsonl,坏行跳过;同一个 tid 只保留第一次出现。"""
if not path.exists():
raise FileNotFoundError(
f"缺少主题清单:{path}\n先跑 `python tieba.py enum --kw <吧名> --pages 1` 试水,"
f"或去掉 --pages 全量枚举"
)
rows: list[dict] = []
seen: set[int] = set()
bad = 0
with open(path, encoding="utf-8", errors="ignore") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
tid = int(row["tid"])
except (json.JSONDecodeError, KeyError, TypeError, ValueError):
bad += 1
continue
if tid in seen:
continue
seen.add(tid)
row["tid"] = tid
rows.append(row)
if bad:
print(f"[!] threads.jsonl 有 {bad} 行无法解析,已跳过")
return rows
async def enumerate_with_client(client, cfg: Config, pages: int = 0) -> dict:
"""枚举主题列表,追加写 threads.jsonl;返回统计。
每次都从第 1 页开始扫,靠 tid 去重。为什么不从「已有条数/30」续:
主题列表按最后回复时间排序,删帖/新回复都会让分页漂移,用页码当游标
会漏帖。重扫的代价只是多花十几分钟请求。
"""
set_rate(cfg.rate)
cfg.ensure_dirs()
existing: set[int] = set()
if cfg.threads_path.exists():
try:
existing = {int(row["tid"]) for row in load_threads(cfg.threads_path)}
except (FileNotFoundError, KeyError, TypeError, ValueError):
existing = set()
try:
forum = await client.get_forum(cfg.kw)
except Exception as exc:
raise CrawlError(
f"获取贴吧信息失败(吧名或 Cookie 可能不对):{type(exc).__name__}: {exc}"
) from exc
fid = _ts(getattr(forum, "fid", 0))
thread_num = _ts(getattr(forum, "thread_num", 0))
fname = str(getattr(forum, "fname", "") or cfg.kw)
print(f"[i] {fname} fid={fid} 主题={thread_num:,}")
expected_pages = (
math.ceil(thread_num / cfg.thread_rn) if thread_num and cfg.thread_rn else 0
)
added = seen = page_no = 0
pn = 1
with open(cfg.threads_path, "a", encoding="utf-8") as fh:
while True:
page_no += 1
page = await _enum_page(client, cfg, pn, expected_pages)
objs = getattr(page, "objs", None) or []
if not objs:
print(f"\n[i] 第 {pn} 页为空,枚举结束(已扫 {page_no - 1} 页)")
page_no -= 1
break
for thread in objs:
seen += 1
record = thread_record(thread)
if not record["tid"] or record["tid"] in existing:
continue
existing.add(record["tid"])
fh.write(_dumps(record))
added += 1
fh.flush()
total = getattr(getattr(page, "page", None), "total_page", 0) or expected_pages
print(
f"\r 第 {page_no} 页 pn={pn}/{total or '?'} 本页 {len(objs)},"
f"新增 {added:,},累计 {len(existing):,}",
end="",
flush=True,
)
if pages and page_no >= pages:
break
if not getattr(getattr(page, "page", None), "has_more", False):
break
pn += 1
print()
if fid or fname:
cfg.fid = fid or cfg.fid
cfg.forum_name = fname or cfg.forum_name
cfg.save()
print(f"[i] 本次扫 {page_no} 页 / 看到 {seen:,} 个主题,新增 {added:,},清单共 {len(existing):,}")
return {
"pages": page_no,
"seen": seen,
"added": added,
"fid": cfg.fid,
"threads": len(existing),
}
async def _enum_page(client, cfg: Config, pn: int, expected_pages: int):
"""取一页主题;空结果按限流处理,退避重试。"""
tries = 0
while True:
await pace()
page = await client.get_threads(cfg.kw, pn=pn, rn=cfg.thread_rn)
if getattr(page, "objs", None):
return page
err = getattr(page, "err", None)
code = getattr(err, "code", None)
if code == 4:
raise CrawlError(f"贴吧「{cfg.kw}」不存在或无法访问(错误码 4)")
tries += 1
if tries >= cfg.tries:
return page
wait = cfg.backoff * tries
print(
f"\n [!] 枚举第 {pn} 页空结果({tries}/{cfg.tries}),{wait:.0f}s 后重试",
flush=True,
)
await asyncio.sleep(wait)
async def enum(cfg: Config, pages: int = 0) -> dict:
"""联网枚举主题。``pages=0`` 表示一直翻到没有下一页。"""
aiotieba = require_aiotieba()
bduss, stoken = read_auth(cfg)
async with new_client(aiotieba, cfg, bduss, stoken) as client:
return await enumerate_with_client(client, cfg, pages)
# ---------------------------------------------------------------- 抓帖
async def _fetch_comments(
client, cfg: Config, writer: JsonlWriter, tid: int, pid: int, stats: dict, lock: asyncio.Lock
) -> None:
"""楼中楼超过一次返回数量(默认一次带 50 条)时翻页补齐。
这一路能拿到每条的 agree/disagree;主页接口内联返回的那几十条是不带
这些字段的,所以「超量」的楼必须单独走这里。
"""
pn = 1
while True:
await pace()
result = await client.get_comments(tid, pid, pn=pn)
objs = getattr(result, "objs", None) or []
if not objs:
return
async with lock:
for comment in objs:
writer.comment(comment_record(tid, comment, pid))
writer.user(user_of(getattr(comment, "user", None)))
pn += 1
if not getattr(getattr(result, "page", None), "has_more", False):
return
if pn > cfg.comment_page_cap:
return
def _is_deleted(page: Any) -> bool:
"""帖子已删:aiotieba 把错误挂在 ``page.err`` 上(code=4),不抛异常。"""
err = getattr(page, "err", None)
if getattr(err, "code", None) == 4:
return True
# 少数版本把 err 做成 (code, msg) 元组
if isinstance(err, tuple) and err and err[0] == 4:
return True
return False
async def crawl_thread(
client,
cfg: Config,
writer: JsonlWriter,
row: dict,
stats: dict,
lock: asyncio.Lock,
) -> None:
"""抓一个主题的全部楼层。
成功才写 done.txt。任何「没抓全」的路径(空结果、超页数上限、异常)
都抛 :class:`CrawlError` 或原异常,留给 worker 记 err.jsonl 并跳过。
"""
tid = int(row["tid"])
pn = 1
tries = 0
while True:
if pn > cfg.page_cap:
raise CrawlError(
f"tid={tid} 超过页数上限 {cfg.page_cap}(config.json 的 page_cap),本轮跳过"
)
await pace()
page = await client.get_posts(
tid,
pn=pn,
rn=cfg.post_rn,
with_comments=True,
comment_rn=cfg.comment_rn,
)
posts = getattr(page, "objs", None) or []
if not posts:
if _is_deleted(page):
# 已删帖:没东西可抓,记一笔然后算完成,否则每轮都会重试
async with lock:
writer.err(
{
"tid": tid,
"err": "deleted: 帖子已被删除",
"code": 4,
"ts": int(time.time()),
}
)
writer.finish_thread(tid)
stats["deleted"] += 1
return
# 没有报错、服务端明确说没有下一页:就是真的翻到底了(末页可能不足 rn 楼)。
# 这类帖以前会被当成 429 一直重试,最后永远抓不完。
if (
getattr(page, "err", None) is None
and getattr(getattr(page, "page", None), "has_more", None) is False
):
break
tries += 1
if tries >= cfg.tries:
raise CrawlError(
f"tid={tid} 第 {pn} 页连续 {cfg.tries} 次空结果,跳过(不写 done)"
)
wait = cfg.backoff * tries
print(
f"\n [!] tid={tid} 第 {pn} 页空结果({tries}/{cfg.tries}),{wait:.0f}s 后重试",
flush=True,
)
await asyncio.sleep(wait)
continue
tries = 0
overflow: list[int] = []
async with lock:
for post in posts:
writer.post(post_record(tid, post))
writer.user(user_of(getattr(post, "user", None)))
for comment in getattr(post, "comments", None) or []:
writer.comment(comment_record(tid, comment, int(getattr(post, "pid", 0))))
writer.user(user_of(getattr(comment, "user", None)))
reply_num = _ts(getattr(post, "reply_num", 0))
inline = len(getattr(post, "comments", None) or [])
if reply_num > inline:
overflow.append(int(getattr(post, "pid", 0)))
stats["overflow"] += len(overflow)
# 补抓是「尽力而为」:失败只计数,不因此否定整帖(主页数据已经拿到了)
for pid in overflow:
try:
await _fetch_comments(client, cfg, writer, tid, pid, stats, lock)
except Exception:
async with lock:
stats["overflow_err"] += 1
pn += 1
if not getattr(getattr(page, "page", None), "has_more", False):
break
async with lock:
writer.finish_thread(tid)
stats["threads"] += 1
async def crawl(cfg: Config, threads: int = 0, workers: int | None = None) -> dict:
"""抓帖入口:读 threads.jsonl 与 done.txt,抓没抓过的。
``threads=0`` 表示全部未完成;``workers`` 只覆盖本次运行,不写回配置。
"""
aiotieba = require_aiotieba()
bduss, stoken = read_auth(cfg)
cfg.workers = int(workers or cfg.workers)
cfg.validate()
set_rate(cfg.rate)
done = read_tid_set(cfg.done_path)
rows: list[dict] = []
for row in load_threads(cfg.threads_path):
if row["tid"] in done:
continue
rows.append(row)
if threads and len(rows) >= threads:
break
print(f"[i] 已抓完 {len(done):,} 帖,本次待抓 {len(rows):,},并发 {cfg.workers},限速 {cfg.rate}/s")
if not rows:
print("[i] 没有待抓主题。清单抓完了,或者先跑 enum 生成 threads.jsonl")
return {
"threads": 0,
"deleted": 0,
"skipped": 0,
"errors": 0,
"posts": 0,
"comments": 0,
"users": 0,
"overflow": 0,
}
writer = JsonlWriter(cfg)
stats = {
"threads": 0,
"deleted": 0,
"skipped": 0,
"errors": 0,
"overflow": 0,
"overflow_err": 0,
}
lock = asyncio.Lock()
sem = asyncio.Semaphore(cfg.workers)
started = time.time()
async def worker(row: dict) -> None:
async with sem:
try:
await crawl_thread(client, cfg, writer, row, stats, lock)
except CrawlError as exc:
# 可跳过的失败:没写 done,下一轮还会再试
async with lock:
stats["skipped"] += 1
writer.err(
{
"tid": int(row["tid"]),
"err": str(exc),
"ts": int(time.time()),
}
)
if stats["skipped"] % 20 == 1:
print(f"\n [!] {exc}", flush=True)
except Exception as exc:
async with lock:
stats["errors"] += 1
writer.err(
{
"tid": int(row["tid"]),
"err": f"{type(exc).__name__}: {exc}",
"tb": traceback.format_exc()[-800:],
"ts": int(time.time()),
}
)
if stats["errors"] % 20 == 1:
print(
f"\n [!] tid={row['tid']} {type(exc).__name__}: {exc}",
flush=True,
)
finally:
async with lock:
finished = stats["threads"] + stats["deleted"] + stats["skipped"] + stats["errors"]
elapsed = max(time.time() - started, 1.0)
if finished % 50 == 0:
print(
f"\r {finished}/{len(rows)} 帖 楼 {writer.n_posts:,} "
f"楼中楼 {writer.n_comments:,} 失败 {stats['skipped'] + stats['errors']} "
f"{(finished / elapsed) * 60:.0f} 帖/分",
end="",
flush=True,
)
async with new_client(aiotieba, cfg, bduss, stoken) as client:
try:
forum = await client.get_forum(cfg.kw)
print(f"[i] {getattr(forum, 'fname', cfg.kw)} fid={getattr(forum, 'fid', 0)}")
except Exception as exc:
writer.close()
raise CrawlError(
f"登录或吧名有误:{type(exc).__name__}: {exc}"
) from exc
try:
await asyncio.gather(*(worker(row) for row in rows))
finally:
writer.close()
elapsed = time.time() - started
print(
f"\n[i] 本次完成 {stats['threads']:,} 帖(已删 {stats['deleted']} / 跳过 {stats['skipped']} / "
f"异常 {stats['errors']}),楼 {writer.n_posts:,} / 楼中楼 {writer.n_comments:,},"
f"用户 {writer.n_users:,},用时 {elapsed / 60:.1f} 分钟"
)
return {
**stats,
"posts": writer.n_posts,
"comments": writer.n_comments,
"users": writer.n_users,
}