Files
dsh 85b8363fed fix(fetch-emoji): 个别表情 CDN 缺失不再以非零码退出
配合 shell set -e 时会中断后续 build-zim;现在只有全部失败才返回 1,缺失项回退文字标签。
2026-11-18 13:10:24 +08:00

1127 lines
44 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""命令行入口:``python tieba.py <子命令>``。
参数解析放这里,业务逻辑放各自模块,好处是每个模块都可以单独 import 做测试
(``tests/test_offline.py`` 就是这么做的)。
几个约定:
* 所有子命令都接受 ``--data``,默认 ``./data``;命令行给的值只覆盖本次运行,
不写回 ``config.json``(``init`` / ``enum`` 除外,它们本来就在维护配置)。
* 会联网的子命令:``check-cookie`` / ``enum`` / ``crawl`` / ``images``;
其余全部离线。
* 错误统一打到 stderr 并返回非零,不把 Python traceback 糊到用户脸上——
除非 ``--debug``(这里没加,需要时直接 import 模块跑)。
"""
from __future__ import annotations
import argparse
import asyncio
import contextlib
import hashlib
import json
import os
import shutil
import sqlite3
import sys
import tempfile
import time
import traceback
from pathlib import Path
from . import (__version__, crawler, db, events, export, images, importer, # noqa: F401
emoji, rebuild, replay, restore, update, users, verify, zim)
from .config import Config
from .cookies import (
CookieError,
masked_items,
parse_cookie_file,
parse_cookie_text,
save_cookie_json,
validate_cookies,
)
# ---------------------------------------------------------------- 输出
def setup_console() -> None:
"""Windows 控制台统一 UTF-8,避免中文和树形符号变乱码。"""
for stream in (sys.stdout, sys.stderr):
try:
stream.reconfigure(encoding="utf-8", errors="replace")
except Exception:
pass
def _print_stats(cfg: Config, stats: dict) -> None:
print("=" * 60)
print(f"数据目录 {cfg.data_dir}")
print(f"数据库 {cfg.db_path}")
print(
f"主题 {stats['threads']:>12,} 已抓完 {stats['threads_done']:,} "
f"异常 {stats['threads_err']:,}"
)
print(
f"楼层 {stats['posts']:>12,} 有小尾巴 {stats['posts_signed']:,} "
f"有赞 {stats['posts_agreed']:,}"
)
print(
f"楼中楼 {stats['comments']:>12,} 有回复对象 {stats['comments_targeted']:,}"
)
print(
f"图片 {stats['images']:>12,} 个 URL / {stats['image_files']:,} 个去重文件"
f" / {db.fmt_size(stats['image_bytes'])}"
)
print(f"用户 {stats['users']:>12,}")
print(
f"主题覆盖率 {stats['threads_with_posts']:,}/{stats['threads']:,} "
f"({db.coverage(stats):.1f}%)"
)
print("=" * 60)
def _layout_tree(cfg: Config) -> None:
"""打印数据目录结构,标出哪些已经生成。"""
items = [
("config.json", "参数(吧名/限速/并发)"),
("cookies.txt", "Cookie(下一步放这里)"),
("tieba.db", "SQLite 成品库"),
("threads.jsonl", "主题清单"),
("raw/posts.jsonl", "楼层(含 frags)"),
("raw/comments.jsonl", "楼中楼"),
("raw/users.jsonl", "用户资料"),
("raw/done.txt", "断点:已抓完的 tid"),
("raw/err.jsonl", "失败记录"),
("imgs/index.jsonl", "图片下载索引"),
("imgs/state.json", "图片下载计数"),
("imgs/ab/cd/<md5>.jpg", "按 md5 分桶的图片"),
]
print(f"\n数据目录:{cfg.data_dir.resolve()}")
for index, (rel, desc) in enumerate(items):
path = cfg.data_dir / rel.replace("<md5>", "")
if "<md5>" in rel:
exists = cfg.imgs_dir.exists() and any(cfg.imgs_dir.glob("[0-9a-f][0-9a-f]"))
else:
exists = path.exists()
branch = "└─" if index == len(items) - 1 else "├─"
mark = "[+]" if exists else "[ ]"
print(f" {branch} {mark} {rel:<26} {desc}")
# ---------------------------------------------------------------- 参数
def build_parser() -> argparse.ArgumentParser:
common = argparse.ArgumentParser(add_help=False)
common.add_argument(
"--data", default="./data", metavar="DIR", help="数据目录(默认 ./data)"
)
common.add_argument("-q", "--quiet", action="store_true", help="少打印")
parser = argparse.ArgumentParser(
prog="tieba.py",
description="贴吧归档工具:枚举主题 / 抓帖 / 导入 SQLite / 补图 / 统计。",
parents=[common],
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=(
"示例:\n"
" python tieba.py init --kw 东方年代记 --data data\n"
" python tieba.py check-cookie --data data\n"
" python tieba.py enum --pages 1 --data data\n"
" python tieba.py crawl --threads 10 --data data\n"
" python tieba.py import --data data\n"
" python tieba.py stats --data ..\\东方年代记吧_全量备份_2011-2026\\data\n"
),
)
parser.add_argument("--version", action="version", version=f"tieba-archiver {__version__}")
sub = parser.add_subparsers(dest="command", metavar="<子命令>")
p = sub.add_parser("init", parents=[common], help="建目录、写 config.json、建库")
p.add_argument("--kw", help="吧名,例如 东方年代记")
p.add_argument("--force", action="store_true", help="用默认值重写 config.json")
p.set_defaults(func=cmd_init)
p = sub.add_parser("check-cookie", parents=[common], help="解析并验证 Cookie 登录态")
p.add_argument(
"--cookies",
metavar="FILE",
help="从别处读 Cookie(仍会统一写回 <data>/cookies.txt)",
)
p.set_defaults(func=cmd_check_cookie)
p = sub.add_parser("enum", parents=[common], help="枚举主题,追加写 threads.jsonl")
p.add_argument("--pages", type=int, default=0, metavar="N", help="只扫 N 页(默认全部)")
p.set_defaults(func=cmd_enum)
p = sub.add_parser("crawl", parents=[common], help="抓主题楼层/楼中楼/用户资料")
p.add_argument("--threads", type=int, default=0, metavar="N", help="只抓 N 个未完成的主题")
p.add_argument("--workers", type=int, default=0, metavar="W", help="并发数(默认读配置)")
p.add_argument("--rate", type=float, default=0.0, metavar="R", help="全局限速:请求/秒")
p.set_defaults(func=cmd_crawl)
p = sub.add_parser("update", parents=[common], help="增量备份:新主题 + 老帖尾部新楼层")
p.add_argument("--pages", type=int, default=2, metavar="N", help="枚举至少翻 N 页(默认 2)")
p.add_argument("--threads", type=int, default=0, metavar="N", help="本次最多处理 N 个帖")
p.add_argument("--rate", type=float, default=0.0, metavar="R")
p.set_defaults(func=cmd_update)
p = sub.add_parser("verify", parents=[common], help="与服务器对账:发现删除/编辑")
p.add_argument("--threads", type=int, default=0, metavar="N", help="最多核对 N 个帖")
p.add_argument("--priority", choices=["recent", "oldest", "random", "unhashed"], default="recent",
help="先核对哪些帖(默认 recent:回复多的)")
p.add_argument("--tids", help="只核对这些 tid,逗号分隔")
p.add_argument("--rate", type=float, default=0.0, metavar="R", help="本次限速(请求/秒)")
p.set_defaults(func=cmd_verify)
p = sub.add_parser("as-of", parents=[common], help="看某个帖在指定时间点的样子(倒放事件)")
p.add_argument("--tid", type=int, required=True)
p.add_argument("--at", required=True, help="YYYY-MM-DD 或 'YYYY-MM-DD HH:MM:SS'")
p.add_argument("--limit", type=int, default=50, metavar="N", help="最多显示 N 楼")
p.add_argument("--json", action="store_true", help="输出完整 JSON")
p.set_defaults(func=cmd_asof)
p = sub.add_parser("events", parents=[common], help="查看事件流")
p.add_argument("--tid", type=int)
p.add_argument("--kind", help="只看某类事件,如 post_del")
p.add_argument("--limit", type=int, default=50, metavar="N")
p.set_defaults(func=cmd_events)
p = sub.add_parser("fetch-emoji", parents=[common], help="下载贴吧经典表情到 data/emoji")
p.add_argument("--force", action="store_true", help="重新下载已有的表情")
p.set_defaults(func=cmd_fetch_emoji)
p = sub.add_parser("build-zim", parents=[common], help="打包成 ZIM(给 Kiwix 用)")
p.add_argument("--out", help="输出 .zim 路径(默认 <data>/../<吧名>.zim)")
p.add_argument("--limit", type=int, default=0, metavar="N", help="只打包最新 N 个主题(试水用)")
p.add_argument("--no-images", action="store_true", help="不打包图片(体积小很多)")
p.add_argument("--title", help="ZIM 标题")
p.add_argument("--language", default="zho", help="语言代码,默认 zho")
p.add_argument("--max-bytes", metavar="SIZE",
help="按体积拆成多个 ZIM(如 3.5G);Windows 上单个 >4GiB 会写坏")
p.set_defaults(func=cmd_build_zim)
p = sub.add_parser("restore", parents=[common], help="从回收站恢复(也记事件)")
p.add_argument("--tid", type=int, help="整帖恢复")
p.add_argument("--pid", type=int, help="恢复单层")
p.add_argument("--cid", type=int, help="恢复单条楼中楼")
p.set_defaults(func=cmd_restore)
p = sub.add_parser("fill-users", parents=[common], help="补 user 表里缺的作者资料")
p.add_argument("--limit", type=int, default=0, metavar="N", help="本次最多补 N 个")
p.add_argument("--rate", type=float, default=0.0, metavar="R")
p.set_defaults(func=cmd_fill_users)
p = sub.add_parser("rebuild", parents=[common], help="检查事件流与物化表是否一致")
p.add_argument("--check", action="store_true", default=True, help="只检查(默认)")
p.set_defaults(func=cmd_rebuild)
p = sub.add_parser("import", parents=[common], help="JSONL → SQLite(幂等)")
p.set_defaults(func=cmd_import)
p = sub.add_parser("images", parents=[common], help="补图:按 posts.jsonl 下载缺失图片")
p.add_argument("--workers", type=int, default=images.DEFAULT_WORKERS, metavar="W")
p.add_argument("--limit", type=int, default=0, metavar="N", help="本次最多下 N 张")
p.set_defaults(func=cmd_images)
p = sub.add_parser("import-images", parents=[common], help="index.jsonl → image 表")
p.set_defaults(func=cmd_import_images)
p = sub.add_parser("stats", parents=[common], help="打印主题/楼层/楼中楼/图片/用户/覆盖率")
p.add_argument(
"--live",
action="store_true",
help="用普通只读连接(能看到未 checkpoint 的 WAL;默认 immutable 快照,不碰数据目录)",
)
p.set_defaults(func=cmd_stats)
p = sub.add_parser("export", parents=[common], help="从数据库导出 threads.jsonl")
p.set_defaults(func=cmd_export)
p = sub.add_parser("label", parents=[common], help="给存档贴标签(ARCHIVE.md/json + 库内 kv)")
p.add_argument("--label", help="标签文字,默认「<吧名>吧 全量备份」")
p.add_argument("--version", default="v1.0", help="版本号,默认 v1.0")
p.add_argument("--no-hash", action="store_true", help="跳过 sha256(大库更快)")
p.set_defaults(func=cmd_label)
p = sub.add_parser("selftest", parents=[common], help="离线自检(不联网)")
p.set_defaults(func=cmd_selftest)
return parser
# ---------------------------------------------------------------- 子命令
def cmd_init(args) -> int:
data_dir = Path(args.data)
config_exists = (data_dir / "config.json").exists()
if config_exists and not args.force:
# 保留用户已经调过的 rate/workers 等参数,只允许换吧名
cfg = Config.load(data_dir, kw=args.kw)
else:
kw = args.kw
if not kw and config_exists:
kw = Config.load(data_dir).kw
if not kw:
raise RuntimeError(
"首次初始化需要指定吧名,例如:\n"
" python tieba.py init --kw 东方年代记 --data data"
)
cfg = Config(kw=kw, data_dir=data_dir)
cfg.ensure_dirs()
cfg.save()
conn = db.connect(cfg)
try:
db.init_db(conn)
db.ensure_compat(conn)
finally:
conn.close()
print(f"[ok] 已初始化:{cfg.data_dir.resolve()}")
print(f"[i] 吧名 {cfg.kw},限速 {cfg.rate}/s,并发 {cfg.workers}")
_layout_tree(cfg)
print("\n下一步:")
print(f" 1. 把浏览器 Cookie 粘进 {cfg.cookies_path}")
print(f" 2. python tieba.py check-cookie --data {cfg.data_dir}")
print(f" 3. python tieba.py enum --pages 1 --data {cfg.data_dir}")
return 0
def cmd_check_cookie(args) -> int:
cfg = Config.load(args.data)
source = Path(args.cookies) if args.cookies else cfg.cookies_path
if not source.exists():
raise FileNotFoundError(
f"没有找到 Cookie 文件:{source}\n"
f"把浏览器 DevTools 里的 Cookie 整条粘进去,或参考 README 的 JSON 格式。"
)
cookies = parse_cookie_file(source)
if not cookies:
raise CookieError(
f"{source} 里没有解析出任何 Cookie。\n"
f"支持:整条 Cookie 请求头 / JSON 对象 / 'Copy as cURL' / Netscape cookies.txt"
)
warnings = validate_cookies(cookies)
print(f"[i] 来源:{source}")
print(f"[i] 解析到 {len(cookies)} 个键(值已打码):")
for key, masked in masked_items(cookies):
print(f" {key:<18} = {masked}")
for warning in warnings:
print(f"[!] {warning}")
target = save_cookie_json(cfg.cookies_path, cookies)
if source.resolve() != target.resolve():
print(f"[i] 已统一写成 JSON 副本:{target}")
else:
print(f"[i] 已统一写成 JSON:{target}")
return asyncio.run(_verify_login(cfg, cookies["BDUSS"], cookies.get("STOKEN", "")))
async def _verify_login(cfg: Config, bduss: str, stoken: str) -> int:
aiotieba = crawler.require_aiotieba()
from aiotieba.exception import TiebaServerError
try:
async with crawler.new_client(aiotieba, cfg, bduss, stoken) as client:
try:
user = await client.get_self_info()
except TiebaServerError as exc:
code = getattr(exc, "code", None)
if code == 1990006:
print(
"[X] 登录失效(错误码 1990006):BDUSS 已过期。\n"
" 重新登录百度后,从浏览器复制新的 Cookie 再跑一次。"
)
return 3
print(f"[X] 服务器返回错误 {code}: {getattr(exc, 'msg', exc)}")
return 4
except Exception as exc:
print(f"[X] 验证请求失败:{type(exc).__name__}: {exc}")
return 4
user_id = getattr(user, "user_id", 0)
uname = getattr(user, "user_name", "") or ""
nick = getattr(user, "nick_name_new", "") or ""
if not user_id:
print("[X] 服务器没有返回用户信息,Cookie 可能不完整")
return 3
print(f"[ok] 登录有效:user_id={user_id} user_name={uname} nick={nick}")
return 0
def cmd_enum(args) -> int:
if args.pages < 0:
raise ValueError("--pages 不能是负数")
cfg = Config.load(args.data)
result = asyncio.run(crawler.enum(cfg, args.pages))
print(f"[i] 主题清单:{cfg.threads_path}(共 {result['threads']:,} 个主题)")
print(f"[i] 下一步:python tieba.py crawl --threads 10 --data {cfg.data_dir}")
return 0
def cmd_crawl(args) -> int:
if args.threads < 0:
raise ValueError("--threads 不能是负数")
cfg = Config.load(args.data)
if args.workers:
cfg.workers = args.workers
if args.rate:
cfg.rate = args.rate
cfg.validate()
result = asyncio.run(crawler.crawl(cfg, args.threads, args.workers or None))
if result["threads"] or result["deleted"]:
print(f"[i] 下一步:python tieba.py import --data {cfg.data_dir}")
return 0
def cmd_import(args) -> int:
cfg = Config.load(args.data)
if not cfg.data_dir.exists():
raise FileNotFoundError(f"数据目录不存在:{cfg.data_dir}")
result = importer.import_all(cfg, quiet=args.quiet)
if not args.quiet:
_print_stats(cfg, result["stats"])
print(f"[i] 下一步:补图 `python tieba.py images --data {cfg.data_dir}`")
print(f" 或导出清单 `python tieba.py export --data {cfg.data_dir}`")
return 0
def cmd_images(args) -> int:
if args.workers < 1:
raise ValueError("--workers 至少为 1")
if args.limit < 0:
raise ValueError("--limit 不能是负数")
cfg = Config.load(args.data)
images.download_missing(cfg, args.workers, args.limit)
return 0
def cmd_import_images(args) -> int:
cfg = Config.load(args.data)
if not cfg.image_index_path.exists():
raise FileNotFoundError(
f"缺少 {cfg.image_index_path},先跑 `python tieba.py images --data {cfg.data_dir}`"
)
images.import_index(cfg, quiet=args.quiet)
return 0
def _sha256_file(path: Path, chunk: int = 1 << 20) -> str:
"""分块算 sha256:库有几百 MB,一次读完会占掉同样多的内存。"""
digest = hashlib.sha256()
with open(path, "rb") as fh:
while True:
block = fh.read(chunk)
if not block:
break
digest.update(block)
return digest.hexdigest()
def _count_images(path: Path) -> tuple[int, int]:
"""磁盘上的图片文件数与总字节。和 image 表的 md5 去重统计互相印证。"""
files = total = 0
if not path.exists():
return 0, 0
for p in path.rglob("*"):
if p.is_file() and p.suffix.lower() in {".jpg", ".png", ".gif", ".bmp", ".webp"}:
files += 1
total += p.stat().st_size
return files, total
def _latest_data_mtime(cfg) -> float:
"""数据文件最后一次被写入的时间。
数据库的 ``fetched_at`` 只在「插入新行」时写,后续 import 只更新字段不会刷新它;
所以再扫一遍 raw*/ 和图片索引的 mtime,两者取晚的,才是真正的「备份截止」。
故意不扫 ``tieba.db``:贴标签本身会写库,算进去会导致每次重贴时间都往后跳。
"""
newest = 0.0
paths = list(cfg.data_dir.glob("raw*/*")) + [cfg.image_index_path]
for path in paths:
try:
if path.is_file():
newest = max(newest, path.stat().st_mtime)
except OSError:
continue
return newest
def _render_label_md(manifest: dict) -> str:
"""给存档生成一张「箱子上贴的标签」:人眼可读,机器也能解析 JSON 那份。"""
s = manifest["stats"]
span = manifest["time_span"]
backup = manifest.get("backup", {})
lines = [
"# 存档标签",
"",
"| 项 | 值 |",
"|---|---|",
f"| 标签 | {manifest['label']} |",
f"| 版本 | {manifest['version']} |",
f"| 吧名 | {manifest['kw']}(fid {manifest['fid'] or '?'}) |",
f"| **备份截止** | **{backup.get('finished_at') or '?'}**(此后新发的帖不在本备份内) |",
f"| 内容截止 | {span['last']}(最后一帖的发布时间) |",
f"| 主题帖 | {s['threads']:,}(已抓完 {s['threads_done']:,}) |",
f"| 楼层 | {s['posts']:,} |",
f"| 楼中楼 | {s['comments']:,} |",
f"| 楼中楼回复对象 | {s.get('comments_targeted', 0):,} 条 |",
f"| 小尾巴 / 点赞 | {s.get('posts_signed', 0):,} 层 / {s.get('posts_agreed', 0):,} 层 |",
f"| 图片 | {s.get('images_done', 0):,} URL / {s.get('image_files', 0):,} 文件 "
f"/ {db.fmt_size(manifest['images']['bytes'])} |",
f"| 用户资料 | {s.get('users', 0):,} |",
f"| 时间跨度 | {span['first']} ~ {span['last']} |",
f"| tid 范围 | {manifest['tid_range']['min']} ~ {manifest['tid_range']['max']} |",
f"| 数据库 | {manifest['db']['file']} {db.fmt_size(manifest['db']['bytes'])} |",
f"| sha256 | {manifest['db']['sha256'] or '(未计算)'} |",
f"| 标签生成 | {manifest['created_at']} |",
f"| 工具 | {manifest['tool']} |",
"",
"> 机器可读版本见同目录 `ARCHIVE.json`;同一份标签也写进了库里的 `kv` 表。",
"",
]
return "\n".join(lines)
def cmd_label(args) -> int:
"""给当前数据目录贴一张标签:统计 + 时间跨度 + 库的 sha256,写 ARCHIVE.md/json。
标签是「这份存档是什么」的自证:搬走、刻盘、发给别人之后,靠它就能核对
内容有没有缺、有没有被改过。
"""
cfg = Config.load(args.data)
conn = db.connect(cfg, readonly=True, immutable=True)
try:
stats = db.stats(conn)
first, last = conn.execute("SELECT MIN(post_time), MAX(post_time) FROM post").fetchone()
tid_min, tid_max = conn.execute("SELECT MIN(tid), MAX(tid) FROM thread").fetchone()
fid = cfg.fid
if not fid:
# config.json 里没记 fid 时(比如直接对旧库贴标签),从 forum 表拿
row = conn.execute("SELECT fid FROM forum LIMIT 1").fetchone()
fid = row[0] if row else 0
# 「备份截止」= 库里最后一次写入(抓取/导入完成)的时间;
# 和「内容截止」(最后一帖的发布时间)是两回事,都要记。
last_write = max(
conn.execute("SELECT MAX(fetched_at) FROM post").fetchone()[0] or "",
conn.execute("SELECT MAX(fetched_at) FROM comment").fetchone()[0] or "",
) or ""
# 补用户、跑 verify 也会往库里写数据,它们的 updated_at/seen_at 也算“最后写入”
for table, column in (("user", "updated_at"), ("thread_seen", "seen_at")):
try:
value = conn.execute(f"SELECT MAX({column}) FROM {table}").fetchone()[0]
except sqlite3.Error:
value = None
last_write = max(last_write, value or "")
finally:
conn.close()
label = args.label or f"{cfg.kw}吧 全量备份"
version = args.version or "v1.0"
img_files, img_bytes = _count_images(cfg.imgs_dir)
# 备份截止 = max(库里最后一次写入, 原始数据文件的 mtime)
newest = _latest_data_mtime(cfg)
finished_at = max(
[last_write, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(newest)) if newest else ""]
) or ""
# 先把标签写进库、把 WAL 折叠回主库,再算 sha256:
# 这样哈希对应的是「最终落盘的库」,而且重复跑 label 不会自己改变自己的哈希。
wconn = db.connect(cfg)
try:
# 旧版本曾把 sha256/时间写进 kv;哈希一旦写进库就会改变库自己的哈希,
# 所以现在只留 label/version,旧键清掉,免得 SQL 查询看到过期值。
wconn.execute(
"DELETE FROM kv WHERE k IN ('archive_manifest_at', 'archive_db_sha256')"
)
for key, value in (
("archive_label", label),
("archive_version", version),
("archive_finished_at", finished_at),
("archive_content_cutoff", last or ""),
):
row = wconn.execute("SELECT v FROM kv WHERE k=?", (key,)).fetchone()
if not row or row[0] != value:
wconn.execute("INSERT OR REPLACE INTO kv(k,v) VALUES(?,?)", (key, value))
wconn.commit()
wconn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
finally:
wconn.close()
db_bytes = cfg.db_path.stat().st_size if cfg.db_path.exists() else 0
manifest = {
"label": label,
"version": version,
"kw": cfg.kw,
"fid": fid,
"created_at": time.strftime("%Y-%m-%d %H:%M:%S"),
"backup": {"finished_at": finished_at, "content_cutoff": last or ""},
"time_span": {"first": first or "", "last": last or ""},
"tid_range": {"min": tid_min, "max": tid_max},
"stats": stats,
"db": {"file": cfg.db_path.name, "bytes": db_bytes, "sha256": None},
"images": {
"files": img_files,
"bytes": img_bytes,
"urls": stats.get("images_done", 0),
},
"tool": f"tieba-archiver {__version__}",
}
if not args.no_hash:
if not args.quiet:
print(f"[i] 正在算 sha256({db.fmt_size(db_bytes)})…")
manifest["db"]["sha256"] = _sha256_file(cfg.db_path)
json_path = cfg.data_dir / "ARCHIVE.json"
md_path = cfg.data_dir / "ARCHIVE.md"
json_path.write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
md_path.write_text(_render_label_md(manifest), encoding="utf-8")
print(f"[ok] 标签:{label} {version}")
print(f" 主题 {stats['threads']:,} / 楼层 {stats['posts']:,} / 楼中楼 {stats['comments']:,}"
f" / 用户 {stats.get('users', 0):,}")
print(f" 清单:{json_path}")
print(f" 标签:{md_path}")
return 0
def cmd_fetch_emoji(args) -> int:
cfg = Config.load(args.data)
conn = db.connect(cfg)
try:
stats = emoji.fetch(conn, cfg, force=args.force)
finally:
conn.close()
print(f"[ok] 表情 {stats['total']} 个:下载 {stats['ok']},已有 {stats['skip']},失败 {len(stats['failed'])}")
if stats["failed"]:
print(" 失败(CDN 上没有,渲染时回退文字):", " ".join(stats["failed"][:20]))
# 部分表情缺失不影响构建,只有全都失败才算错
return 0 if stats["ok"] or stats["skip"] else 1
def cmd_build_zim(args) -> int:
cfg = Config.load(args.data)
out = Path(args.out) if args.out else (cfg.data_dir.parent / f"{cfg.kw}.zim")
if args.max_bytes:
parts = zim.build_zim_split(
cfg, out, zim.parse_bytes(args.max_bytes),
with_images=not args.no_images, title=args.title or "", language=args.language,
)
print()
total = sum(p["bytes"] for p in parts)
print(f"[ok] 共 {len(parts)} 个 ZIM,合计 {db.fmt_size(total)}")
for p in parts:
print(f" {Path(p['path']).name} {db.fmt_size(p['bytes'])} "
f"页面 {p['pages']:,} 图片 {p['images_added']:,}")
return 0
stats = zim.build_zim(
cfg, out, limit_threads=args.limit,
with_images=not args.no_images, title=args.title or "", language=args.language,
)
print()
print(f"[ok] {stats['path']} {db.fmt_size(stats['bytes'])}")
print(f" 页面 {stats['pages']:,},图片 {stats['images_added']:,}"
f"(引用 {stats['images']:,}),用时 {stats['seconds']/60:.1f} 分钟")
return 0
def cmd_restore(args) -> int:
if not (args.tid or args.pid or args.cid):
print("[X] 至少给一个 --tid / --pid / --cid")
return 2
cfg = Config.load(args.data)
stats = restore.restore(cfg, tid=args.tid, pid=args.pid, cid=args.cid)
print(f"[ok] 已恢复:主题 {stats['threads']},楼层 {stats['posts']},楼中楼 {stats['comments']}")
return 0
def cmd_fill_users(args) -> int:
cfg = Config.load(args.data)
stats = users.fill_missing(cfg, limit=args.limit, rate=args.rate or None)
print(f"\n[ok] 待补 {stats['total']},补上 {stats['filled']},失败 {stats['failed']}")
return 0
def cmd_rebuild(args) -> int:
cfg = Config.load(args.data)
stats = rebuild.check(cfg)
print(f"[i] 事件 {stats['events']:,} 条 {stats['by_kind']}")
print(f"[i] 回收站:楼层 {stats['recycled_posts']:,} / 楼中楼 {stats['recycled_comments']:,},"
f"整帖删 {stats['deleted_threads']:,}")
print(f"[i] 幂等重放:{'一致' if stats['idempotent'] else '不一致!'}")
print(f"[i] 回收站无事件:{stats['recycled_without_event']};"
f"post_add 指向不存在楼层:{stats['add_events_without_post']}")
ok = (stats["idempotent"] and not stats["recycled_without_event"]
and not stats["add_events_without_post"])
print("[ok] 一致" if ok else "[!] 有不一致项,见上")
return 0 if ok else 1
def cmd_update(args) -> int:
cfg = Config.load(args.data)
stats = update.update(cfg, pages=args.pages, threads=args.threads,
rate=args.rate or None)
print(f"[ok] 增量完成:枚举 {stats['pages']} 页,新主题 {stats['new_threads']},"
f"有更新主题 {stats['changed_threads']},新增楼层 {stats['added']},"
f"新增楼中楼 {stats.get('comments', 0)},已删 {stats['deleted']},"
f"失败 {stats['errors']}")
return 0
def cmd_verify(args) -> int:
cfg = Config.load(args.data)
tids = [int(x) for x in args.tids.split(",") if x.strip()] if args.tids else None
stats = verify.verify(cfg, threads=args.threads, priority=args.priority,
tids=tids, rate=args.rate or None)
print(
f"[ok] 核对完成:正常 {stats['ok']},新增 {stats['added']},编辑 {stats['edited']},"
f"删除 {stats['deleted']}(怀疑中 {stats['suspect']}),"
f"楼中楼删 {stats.get('comment_deleted', 0)}(怀疑中 {stats.get('comment_suspect', 0)}),"
f"整帖删 {stats['thread_deleted']}(怀疑中 {stats['thread_suspect']}),"
f"点赞采样 {stats['stat_samples']},失败 {stats['errors']}"
)
return 0
def cmd_asof(args) -> int:
cfg = Config.load(args.data)
conn = db.connect(cfg)
try:
at = replay.normalize_at(args.at)
rows = replay.as_of_thread(conn, int(args.tid), at)
finally:
conn.close()
if args.json:
print(json.dumps(rows, ensure_ascii=False, indent=2))
return 0
live = sum(1 for r in rows if r["status"] == "live")
print(f"[i] tid={args.tid} 在 {at} 时:{len(rows)} 楼(live {live} / 回收站 {len(rows) - live})")
for r in rows[: args.limit]:
flag = "" if r["status"] == "live" else " [回收站]"
print(f" {r['floor']:>5} {r['author']:<12} {r['post_time']} {r['content'][:48]}{flag}")
if len(rows) > args.limit:
print(f" … 还有 {len(rows) - args.limit} 楼,用 --limit 调整或 --json")
return 0
def cmd_events(args) -> int:
cfg = Config.load(args.data)
conn = db.connect(cfg, readonly=True, immutable=True)
try:
sql = "SELECT id, ts, kind, tid, pid, floor, evidence FROM event"
where, params = [], []
if args.tid:
where.append("tid=?")
params.append(int(args.tid))
if args.kind:
where.append("kind=?")
params.append(args.kind)
if where:
sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY id DESC LIMIT ?"
params.append(int(args.limit))
rows = conn.execute(sql, params).fetchall()
finally:
conn.close()
for row in rows:
print(f" #{row[0]:<5} {row[1]} {row[2]:<14} tid={row[3]} pid={row[4]} "
f"floor={row[5]} {row[6] or ''}")
print(f"[i] 共 {len(rows)} 条(最多显示 {args.limit} 条)")
return 0
def cmd_stats(args) -> int:
cfg = Config.load(args.data)
conn = db.connect(cfg, readonly=True, immutable=not args.live)
try:
stats = db.stats(conn)
finally:
conn.close()
_print_stats(cfg, stats)
if not args.live:
print("[i] 只读快照模式(immutable):不创建 -wal/-shm,也看不到未 checkpoint 的写入;")
print(" 要读正在导入的库,加 --live。")
return 0
def cmd_export(args) -> int:
cfg = Config.load(args.data)
count = export.export_threads(cfg, quiet=args.quiet)
if not args.quiet:
print(f"[i] 导出 {count:,} 个主题 → {cfg.threads_path}")
return 0
# ---------------------------------------------------------------- 自检
def _write_selftest_fixture(cfg: Config) -> None:
"""3 条楼层 + 2 条楼中楼的小样本,覆盖 importer 的全部关键路径。"""
cfg.ensure_dirs()
threads = {
"tid": 1001,
"fid": 2288117,
"title": "自检帖",
"author": "楼主甲",
"author_id": 11,
"reply_num": 2,
"is_good": 0,
"is_top": 0,
}
posts = [
{
"tid": 1001,
"pid": 5001,
"floor": 1,
"uid": 11,
"uname": "楼主甲",
"agree": 3,
"disagree": 0,
"sign": "自检小尾巴",
"rn": 1,
"ts": 1700000000,
"lz": True,
"text": "第一楼",
"frags": [{"t": "text", "text": "第一楼"}],
},
{
"tid": 1001,
"pid": 5002,
"floor": 2,
"uid": 22,
"uname": "路人乙",
"agree": 0,
"disagree": 0,
"sign": "",
"rn": 0,
"ts": 1700000100,
"lz": False,
"text": "带图",
"frags": [
{"t": "text", "text": "带图"},
{"t": "img", "src": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg"},
],
},
{
"tid": 1001,
"pid": 5003,
"floor": 3,
"uid": 11,
"uname": "楼主甲",
"agree": 0,
"disagree": 0,
"sign": "",
"rn": 1,
"ts": 1700000200,
"lz": True,
"text": "第三楼",
"frags": [{"t": "text", "text": "第三楼"}],
},
]
comments = [
{
"tid": 1001,
"pid": 9001,
"ppid": 5001,
"floor": 1,
"uid": 22,
"uname": "路人乙",
"rto": 11,
"agree": 1,
"disagree": 0,
"ts": 1700000300,
"lz": False,
"text": "直接回复楼主",
"frags": [{"t": "text", "text": "直接回复楼主"}],
},
{
"tid": 1001,
"pid": 9002,
"ppid": 5003,
"floor": 3,
"uid": 33,
"uname": "路人丙",
"rto": 0,
"agree": 0,
"disagree": 0,
"ts": 1700000400,
"lz": False,
"text": "回复 楼主甲 : 正文里带昵称",
"frags": [{"t": "text", "text": "回复 楼主甲 : 正文里带昵称"}],
},
]
users = [
{
"uid": 11,
"uname": "楼主甲",
"nick": "甲甲",
"portrait": "tb.1.aaa",
"level": 10,
"glevel": 5,
"gender": 1,
"ip": "上海",
"bawu": False,
"vip": True,
"icons": ["vip"],
},
{
"uid": 22,
"uname": "路人乙",
"nick": "乙乙",
"portrait": "",
"level": 3,
"glevel": 2,
"gender": 0,
"ip": "江苏",
"bawu": False,
"vip": False,
"icons": [],
},
]
files = {
cfg.threads_path: [threads],
cfg.posts_path: posts,
cfg.comments_path: comments,
cfg.users_path: users,
}
for path, records in files.items():
with open(path, "w", encoding="utf-8", newline="\n") as fh:
for record in records:
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
cfg.done_path.write_text("1001\n", encoding="utf-8")
with open(cfg.image_index_path, "w", encoding="utf-8", newline="\n") as fh:
fh.write(
json.dumps(
{
"md5": "0" * 32,
"url": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg",
"size": 3,
"path": "imgs/00/00/" + "0" * 32 + ".jpg",
},
ensure_ascii=False,
)
+ "\n"
)
@contextlib.contextmanager
def _selftest_tempdir():
"""为自检找一个可写的临时目录。
正常环境用系统 TEMP;某些 Windows 机器(组策略/沙箱)会把 TEMP 设成
不可写,这时退到当前工作目录。目录用 ``Path.mkdir`` 建:``mkdtemp``
在 Windows 上会带 mode=0700 的 ACL,受限令牌下反而写不进去。
都不行才报错。
"""
candidates = []
env = os.environ.get("TIEBA_TMPDIR")
if env:
candidates.append(Path(env))
candidates.append(Path(tempfile.gettempdir()))
candidates.append(Path.cwd())
path = None
last_exc: Exception | None = None
for root in candidates:
try:
root.mkdir(parents=True, exist_ok=True)
candidate = root / f"tieba-archiver-selftest-{os.getpid()}-{time.time_ns()}"
candidate.mkdir()
path = candidate
break
except OSError as exc:
last_exc = exc
if path is None:
raise RuntimeError(f"找不到可写的临时目录:{last_exc}")
try:
yield path
finally:
shutil.rmtree(path, ignore_errors=True)
def _run_selftest() -> int:
checks: list[bool] = []
def check(name, fn):
try:
fn()
print(f"[ok] {name}")
checks.append(True)
except Exception as exc:
print(f"[X] {name}: {type(exc).__name__}: {exc}")
traceback.print_exc()
checks.append(False)
def cookie_case():
header = (
'BAIDUID=AAA:FG=1; BDUSS=BBB; STOKEN=CCC; '
'RT="z=1&dm=baidu.com&si=x;co=y"; XFT=DDD'
)
jar = parse_cookie_text(header)
assert jar["BDUSS"] == "BBB", jar
assert len(jar) == 5, jar
assert ";" in jar["RT"] and "co=y" in jar["RT"], jar["RT"]
assert parse_cookie_text('{"BDUSS": "x", "STOKEN": "y"}')["STOKEN"] == "y"
assert (
parse_cookie_text("curl 'https://tieba.baidu.com' -H 'Cookie: BDUSS=x; STOKEN=y'")[
"BDUSS"
]
== "x"
)
assert parse_cookie_text("curl -b 'BDUSS=x; STOKEN=y' https://tieba.baidu.com")[
"STOKEN"
] == "y"
try:
validate_cookies({"STOKEN": "y"})
except CookieError:
pass
else:
raise AssertionError("缺 BDUSS 时应该报错")
def reply_case():
assert importer.parse_reply_name("回复 楼主甲 : 你好") == "楼主甲"
assert importer.parse_reply_name("回复楼主甲:你好") == "楼主甲"
assert importer.parse_reply_name("没有前缀") is None
# 超过 30 字的名字不当昵称(避免把长正文误认成回复对象)
long_name = "一二三四五六七八九十" * 3 + "一"
assert len(long_name) > 30
assert importer.parse_reply_name(f"回复 {long_name} : x") is None
check("Cookie 解析(引号内分号 / JSON / cURL / 缺 BDUSS)", cookie_case)
check("回复对象正则", reply_case)
with _selftest_tempdir() as tmp:
cfg = Config(kw="东方年代记", data_dir=Path(tmp), rate=1000.0)
cfg.ensure_dirs()
cfg.save()
def config_case():
loaded = Config.load(cfg.data_dir)
assert loaded.kw == cfg.kw
assert loaded.rate == 1000.0
assert loaded.db_path == Path(tmp) / "tieba.db"
def schema_case():
conn = db.connect(cfg)
try:
db.init_db(conn)
post_cols = db.table_columns(conn, "post")
comment_cols = db.table_columns(conn, "comment")
user_cols = db.table_columns(conn, "user")
assert {"pid", "tid", "agree", "disagree", "sign", "frags"} <= post_cols
assert {"cid", "rto", "reply_to_name", "reply_to_uid", "frags"} <= comment_cols
assert {"uid", "uname", "nick", "portrait", "bawu", "vip"} <= user_cols
finally:
conn.close()
check("config.json 往返", config_case)
check("schema 建库(v2 列齐全)", schema_case)
try:
_write_selftest_fixture(cfg)
first = importer.import_all(cfg, quiet=True)
second = importer.import_all(cfg, quiet=True)
except Exception as exc:
print(f"[X] 造 fixture / 导入失败:{type(exc).__name__}: {exc}")
traceback.print_exc()
return 1
def import_case():
assert first["posts"] == 3, first
assert first["comments"] == 2, first
assert first["users"] == 2, first
assert second["posts"] == 3 and second["comments"] == 2, second
assert second["stats"]["posts"] == 3
assert second["stats"]["comments"] == 2
assert second["stats"]["posts_signed"] == 1
assert second["stats"]["posts_agreed"] == 1
assert second["stats"]["comments_targeted"] == 2
assert second["stats"]["threads_done"] == 1
assert second["stats"]["threads_with_posts"] == 1
assert second["stats"]["users"] == 2
def reply_resolution_case():
conn = db.connect(cfg, readonly=True)
try:
rows = {
row["cid"]: (row["reply_to_name"], row["reply_to_uid"])
for row in conn.execute(
"SELECT cid, reply_to_name, reply_to_uid FROM comment"
)
}
finally:
conn.close()
assert rows[9001] == (None, 11), rows # rto 优先
assert rows[9002] == ("楼主甲", 11), rows # 正文昵称 → 本帖作者
def export_case():
count = export.export_threads(cfg, quiet=True)
assert count == 1
lines = cfg.threads_path.read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 1, lines
assert json.loads(lines[0])["tid"] == 1001
def image_case():
result = images.import_index(cfg, quiet=True)
assert result["filled"] == 1, result
assert result["stats"]["images"] >= 1
assert result["stats"]["image_files"] == 1
check("importer 小样本 + 幂等", import_case)
check("回复对象解析(rto 优先 / 正文昵称)", reply_resolution_case)
check("export threads.jsonl", export_case)
check("import-images(index.jsonl → image 表)", image_case)
def stats_case():
conn = db.connect(cfg, readonly=True, immutable=True)
try:
stats = db.stats(conn)
finally:
conn.close()
assert stats["threads"] == 1
assert stats["images"] == 1
check("stats 只读统计", stats_case)
passed = sum(checks)
print(f"\n[{'ok' if passed == len(checks) else 'X'}] 自检 {passed}/{len(checks)} 项通过(不联网)")
return 0 if passed == len(checks) else 1
def cmd_selftest(args) -> int:
print("[i] 离线自检:Cookie / 正则 / schema / importer / export / 图片入库")
return _run_selftest()
# ---------------------------------------------------------------- 入口
def main(argv: list[str] | None = None) -> int:
setup_console()
parser = build_parser()
args = parser.parse_args(argv)
if not getattr(args, "func", None):
parser.print_help()
return 1
try:
result = args.func(args)
return int(result or 0)
except KeyboardInterrupt:
print("\n[!] 已中断;done.txt 里已写的帖不会重抓,重跑同一条命令即可续传")
return 130
except CookieError as exc:
print(f"[X] {exc}", file=sys.stderr)
return 2
except FileNotFoundError as exc:
print(f"[X] {exc}", file=sys.stderr)
return 2
except (crawler.CrawlError, RuntimeError, ValueError, sqlite3.Error) as exc:
print(f"[X] {exc}", file=sys.stderr)
return 2
if __name__ == "__main__": # pragma: no cover
sys.exit(main())