1127 lines
44 KiB
Python
1127 lines
44 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""命令行入口:``python tieba.py <子命令>``。
|
||
|
||
参数解析放这里,业务逻辑放各自模块,好处是每个模块都可以单独 import 做测试
|
||
(``tests/test_offline.py`` 就是这么做的)。
|
||
|
||
几个约定:
|
||
|
||
* 所有子命令都接受 ``--data``,默认 ``./data``;命令行给的值只覆盖本次运行,
|
||
不写回 ``config.json``(``init`` / ``enum`` 除外,它们本来就在维护配置)。
|
||
* 会联网的子命令:``check-cookie`` / ``enum`` / ``crawl`` / ``images``;
|
||
其余全部离线。
|
||
* 错误统一打到 stderr 并返回非零,不把 Python traceback 糊到用户脸上——
|
||
除非 ``--debug``(这里没加,需要时直接 import 模块跑)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import contextlib
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import shutil
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import traceback
|
||
from pathlib import Path
|
||
|
||
from . import (__version__, crawler, db, events, export, images, importer, # noqa: F401
|
||
emoji, rebuild, replay, restore, update, users, verify, zim)
|
||
from .config import Config
|
||
from .cookies import (
|
||
CookieError,
|
||
masked_items,
|
||
parse_cookie_file,
|
||
parse_cookie_text,
|
||
save_cookie_json,
|
||
validate_cookies,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------- 输出
|
||
def setup_console() -> None:
|
||
"""Windows 控制台统一 UTF-8,避免中文和树形符号变乱码。"""
|
||
for stream in (sys.stdout, sys.stderr):
|
||
try:
|
||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _print_stats(cfg: Config, stats: dict) -> None:
|
||
print("=" * 60)
|
||
print(f"数据目录 {cfg.data_dir}")
|
||
print(f"数据库 {cfg.db_path}")
|
||
print(
|
||
f"主题 {stats['threads']:>12,} 已抓完 {stats['threads_done']:,} "
|
||
f"异常 {stats['threads_err']:,}"
|
||
)
|
||
print(
|
||
f"楼层 {stats['posts']:>12,} 有小尾巴 {stats['posts_signed']:,} "
|
||
f"有赞 {stats['posts_agreed']:,}"
|
||
)
|
||
print(
|
||
f"楼中楼 {stats['comments']:>12,} 有回复对象 {stats['comments_targeted']:,}"
|
||
)
|
||
print(
|
||
f"图片 {stats['images']:>12,} 个 URL / {stats['image_files']:,} 个去重文件"
|
||
f" / {db.fmt_size(stats['image_bytes'])}"
|
||
)
|
||
print(f"用户 {stats['users']:>12,}")
|
||
print(
|
||
f"主题覆盖率 {stats['threads_with_posts']:,}/{stats['threads']:,} "
|
||
f"({db.coverage(stats):.1f}%)"
|
||
)
|
||
print("=" * 60)
|
||
|
||
|
||
def _layout_tree(cfg: Config) -> None:
|
||
"""打印数据目录结构,标出哪些已经生成。"""
|
||
items = [
|
||
("config.json", "参数(吧名/限速/并发)"),
|
||
("cookies.txt", "Cookie(下一步放这里)"),
|
||
("tieba.db", "SQLite 成品库"),
|
||
("threads.jsonl", "主题清单"),
|
||
("raw/posts.jsonl", "楼层(含 frags)"),
|
||
("raw/comments.jsonl", "楼中楼"),
|
||
("raw/users.jsonl", "用户资料"),
|
||
("raw/done.txt", "断点:已抓完的 tid"),
|
||
("raw/err.jsonl", "失败记录"),
|
||
("imgs/index.jsonl", "图片下载索引"),
|
||
("imgs/state.json", "图片下载计数"),
|
||
("imgs/ab/cd/<md5>.jpg", "按 md5 分桶的图片"),
|
||
]
|
||
print(f"\n数据目录:{cfg.data_dir.resolve()}")
|
||
for index, (rel, desc) in enumerate(items):
|
||
path = cfg.data_dir / rel.replace("<md5>", "")
|
||
if "<md5>" in rel:
|
||
exists = cfg.imgs_dir.exists() and any(cfg.imgs_dir.glob("[0-9a-f][0-9a-f]"))
|
||
else:
|
||
exists = path.exists()
|
||
branch = "└─" if index == len(items) - 1 else "├─"
|
||
mark = "[+]" if exists else "[ ]"
|
||
print(f" {branch} {mark} {rel:<26} {desc}")
|
||
|
||
|
||
# ---------------------------------------------------------------- 参数
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
common = argparse.ArgumentParser(add_help=False)
|
||
common.add_argument(
|
||
"--data", default="./data", metavar="DIR", help="数据目录(默认 ./data)"
|
||
)
|
||
common.add_argument("-q", "--quiet", action="store_true", help="少打印")
|
||
|
||
parser = argparse.ArgumentParser(
|
||
prog="tieba.py",
|
||
description="贴吧归档工具:枚举主题 / 抓帖 / 导入 SQLite / 补图 / 统计。",
|
||
parents=[common],
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
epilog=(
|
||
"示例:\n"
|
||
" python tieba.py init --kw 东方年代记 --data data\n"
|
||
" python tieba.py check-cookie --data data\n"
|
||
" python tieba.py enum --pages 1 --data data\n"
|
||
" python tieba.py crawl --threads 10 --data data\n"
|
||
" python tieba.py import --data data\n"
|
||
" python tieba.py stats --data ..\\东方年代记吧_全量备份_2011-2026\\data\n"
|
||
),
|
||
)
|
||
parser.add_argument("--version", action="version", version=f"tieba-archiver {__version__}")
|
||
sub = parser.add_subparsers(dest="command", metavar="<子命令>")
|
||
|
||
p = sub.add_parser("init", parents=[common], help="建目录、写 config.json、建库")
|
||
p.add_argument("--kw", help="吧名,例如 东方年代记")
|
||
p.add_argument("--force", action="store_true", help="用默认值重写 config.json")
|
||
p.set_defaults(func=cmd_init)
|
||
|
||
p = sub.add_parser("check-cookie", parents=[common], help="解析并验证 Cookie 登录态")
|
||
p.add_argument(
|
||
"--cookies",
|
||
metavar="FILE",
|
||
help="从别处读 Cookie(仍会统一写回 <data>/cookies.txt)",
|
||
)
|
||
p.set_defaults(func=cmd_check_cookie)
|
||
|
||
p = sub.add_parser("enum", parents=[common], help="枚举主题,追加写 threads.jsonl")
|
||
p.add_argument("--pages", type=int, default=0, metavar="N", help="只扫 N 页(默认全部)")
|
||
p.set_defaults(func=cmd_enum)
|
||
|
||
p = sub.add_parser("crawl", parents=[common], help="抓主题楼层/楼中楼/用户资料")
|
||
p.add_argument("--threads", type=int, default=0, metavar="N", help="只抓 N 个未完成的主题")
|
||
p.add_argument("--workers", type=int, default=0, metavar="W", help="并发数(默认读配置)")
|
||
p.add_argument("--rate", type=float, default=0.0, metavar="R", help="全局限速:请求/秒")
|
||
p.set_defaults(func=cmd_crawl)
|
||
|
||
p = sub.add_parser("update", parents=[common], help="增量备份:新主题 + 老帖尾部新楼层")
|
||
p.add_argument("--pages", type=int, default=2, metavar="N", help="枚举至少翻 N 页(默认 2)")
|
||
p.add_argument("--threads", type=int, default=0, metavar="N", help="本次最多处理 N 个帖")
|
||
p.add_argument("--rate", type=float, default=0.0, metavar="R")
|
||
p.set_defaults(func=cmd_update)
|
||
|
||
p = sub.add_parser("verify", parents=[common], help="与服务器对账:发现删除/编辑")
|
||
p.add_argument("--threads", type=int, default=0, metavar="N", help="最多核对 N 个帖")
|
||
p.add_argument("--priority", choices=["recent", "oldest", "random", "unhashed"], default="recent",
|
||
help="先核对哪些帖(默认 recent:回复多的)")
|
||
p.add_argument("--tids", help="只核对这些 tid,逗号分隔")
|
||
p.add_argument("--rate", type=float, default=0.0, metavar="R", help="本次限速(请求/秒)")
|
||
p.set_defaults(func=cmd_verify)
|
||
|
||
p = sub.add_parser("as-of", parents=[common], help="看某个帖在指定时间点的样子(倒放事件)")
|
||
p.add_argument("--tid", type=int, required=True)
|
||
p.add_argument("--at", required=True, help="YYYY-MM-DD 或 'YYYY-MM-DD HH:MM:SS'")
|
||
p.add_argument("--limit", type=int, default=50, metavar="N", help="最多显示 N 楼")
|
||
p.add_argument("--json", action="store_true", help="输出完整 JSON")
|
||
p.set_defaults(func=cmd_asof)
|
||
|
||
p = sub.add_parser("events", parents=[common], help="查看事件流")
|
||
p.add_argument("--tid", type=int)
|
||
p.add_argument("--kind", help="只看某类事件,如 post_del")
|
||
p.add_argument("--limit", type=int, default=50, metavar="N")
|
||
p.set_defaults(func=cmd_events)
|
||
|
||
p = sub.add_parser("fetch-emoji", parents=[common], help="下载贴吧经典表情到 data/emoji")
|
||
p.add_argument("--force", action="store_true", help="重新下载已有的表情")
|
||
p.set_defaults(func=cmd_fetch_emoji)
|
||
|
||
p = sub.add_parser("build-zim", parents=[common], help="打包成 ZIM(给 Kiwix 用)")
|
||
p.add_argument("--out", help="输出 .zim 路径(默认 <data>/../<吧名>.zim)")
|
||
p.add_argument("--limit", type=int, default=0, metavar="N", help="只打包最新 N 个主题(试水用)")
|
||
p.add_argument("--no-images", action="store_true", help="不打包图片(体积小很多)")
|
||
p.add_argument("--title", help="ZIM 标题")
|
||
p.add_argument("--language", default="zho", help="语言代码,默认 zho")
|
||
p.add_argument("--max-bytes", metavar="SIZE",
|
||
help="按体积拆成多个 ZIM(如 3.5G);Windows 上单个 >4GiB 会写坏")
|
||
p.set_defaults(func=cmd_build_zim)
|
||
|
||
p = sub.add_parser("restore", parents=[common], help="从回收站恢复(也记事件)")
|
||
p.add_argument("--tid", type=int, help="整帖恢复")
|
||
p.add_argument("--pid", type=int, help="恢复单层")
|
||
p.add_argument("--cid", type=int, help="恢复单条楼中楼")
|
||
p.set_defaults(func=cmd_restore)
|
||
|
||
p = sub.add_parser("fill-users", parents=[common], help="补 user 表里缺的作者资料")
|
||
p.add_argument("--limit", type=int, default=0, metavar="N", help="本次最多补 N 个")
|
||
p.add_argument("--rate", type=float, default=0.0, metavar="R")
|
||
p.set_defaults(func=cmd_fill_users)
|
||
|
||
p = sub.add_parser("rebuild", parents=[common], help="检查事件流与物化表是否一致")
|
||
p.add_argument("--check", action="store_true", default=True, help="只检查(默认)")
|
||
p.set_defaults(func=cmd_rebuild)
|
||
|
||
p = sub.add_parser("import", parents=[common], help="JSONL → SQLite(幂等)")
|
||
p.set_defaults(func=cmd_import)
|
||
|
||
p = sub.add_parser("images", parents=[common], help="补图:按 posts.jsonl 下载缺失图片")
|
||
p.add_argument("--workers", type=int, default=images.DEFAULT_WORKERS, metavar="W")
|
||
p.add_argument("--limit", type=int, default=0, metavar="N", help="本次最多下 N 张")
|
||
p.set_defaults(func=cmd_images)
|
||
|
||
p = sub.add_parser("import-images", parents=[common], help="index.jsonl → image 表")
|
||
p.set_defaults(func=cmd_import_images)
|
||
|
||
p = sub.add_parser("stats", parents=[common], help="打印主题/楼层/楼中楼/图片/用户/覆盖率")
|
||
p.add_argument(
|
||
"--live",
|
||
action="store_true",
|
||
help="用普通只读连接(能看到未 checkpoint 的 WAL;默认 immutable 快照,不碰数据目录)",
|
||
)
|
||
p.set_defaults(func=cmd_stats)
|
||
|
||
p = sub.add_parser("export", parents=[common], help="从数据库导出 threads.jsonl")
|
||
p.set_defaults(func=cmd_export)
|
||
|
||
p = sub.add_parser("label", parents=[common], help="给存档贴标签(ARCHIVE.md/json + 库内 kv)")
|
||
p.add_argument("--label", help="标签文字,默认「<吧名>吧 全量备份」")
|
||
p.add_argument("--version", default="v1.0", help="版本号,默认 v1.0")
|
||
p.add_argument("--no-hash", action="store_true", help="跳过 sha256(大库更快)")
|
||
p.set_defaults(func=cmd_label)
|
||
|
||
p = sub.add_parser("selftest", parents=[common], help="离线自检(不联网)")
|
||
p.set_defaults(func=cmd_selftest)
|
||
|
||
return parser
|
||
|
||
|
||
# ---------------------------------------------------------------- 子命令
|
||
def cmd_init(args) -> int:
|
||
data_dir = Path(args.data)
|
||
config_exists = (data_dir / "config.json").exists()
|
||
|
||
if config_exists and not args.force:
|
||
# 保留用户已经调过的 rate/workers 等参数,只允许换吧名
|
||
cfg = Config.load(data_dir, kw=args.kw)
|
||
else:
|
||
kw = args.kw
|
||
if not kw and config_exists:
|
||
kw = Config.load(data_dir).kw
|
||
if not kw:
|
||
raise RuntimeError(
|
||
"首次初始化需要指定吧名,例如:\n"
|
||
" python tieba.py init --kw 东方年代记 --data data"
|
||
)
|
||
cfg = Config(kw=kw, data_dir=data_dir)
|
||
|
||
cfg.ensure_dirs()
|
||
cfg.save()
|
||
conn = db.connect(cfg)
|
||
try:
|
||
db.init_db(conn)
|
||
db.ensure_compat(conn)
|
||
finally:
|
||
conn.close()
|
||
|
||
print(f"[ok] 已初始化:{cfg.data_dir.resolve()}")
|
||
print(f"[i] 吧名 {cfg.kw},限速 {cfg.rate}/s,并发 {cfg.workers}")
|
||
_layout_tree(cfg)
|
||
print("\n下一步:")
|
||
print(f" 1. 把浏览器 Cookie 粘进 {cfg.cookies_path}")
|
||
print(f" 2. python tieba.py check-cookie --data {cfg.data_dir}")
|
||
print(f" 3. python tieba.py enum --pages 1 --data {cfg.data_dir}")
|
||
return 0
|
||
|
||
|
||
def cmd_check_cookie(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
source = Path(args.cookies) if args.cookies else cfg.cookies_path
|
||
if not source.exists():
|
||
raise FileNotFoundError(
|
||
f"没有找到 Cookie 文件:{source}\n"
|
||
f"把浏览器 DevTools 里的 Cookie 整条粘进去,或参考 README 的 JSON 格式。"
|
||
)
|
||
|
||
cookies = parse_cookie_file(source)
|
||
if not cookies:
|
||
raise CookieError(
|
||
f"{source} 里没有解析出任何 Cookie。\n"
|
||
f"支持:整条 Cookie 请求头 / JSON 对象 / 'Copy as cURL' / Netscape cookies.txt"
|
||
)
|
||
warnings = validate_cookies(cookies)
|
||
|
||
print(f"[i] 来源:{source}")
|
||
print(f"[i] 解析到 {len(cookies)} 个键(值已打码):")
|
||
for key, masked in masked_items(cookies):
|
||
print(f" {key:<18} = {masked}")
|
||
for warning in warnings:
|
||
print(f"[!] {warning}")
|
||
|
||
target = save_cookie_json(cfg.cookies_path, cookies)
|
||
if source.resolve() != target.resolve():
|
||
print(f"[i] 已统一写成 JSON 副本:{target}")
|
||
else:
|
||
print(f"[i] 已统一写成 JSON:{target}")
|
||
|
||
return asyncio.run(_verify_login(cfg, cookies["BDUSS"], cookies.get("STOKEN", "")))
|
||
|
||
|
||
async def _verify_login(cfg: Config, bduss: str, stoken: str) -> int:
|
||
aiotieba = crawler.require_aiotieba()
|
||
from aiotieba.exception import TiebaServerError
|
||
|
||
try:
|
||
async with crawler.new_client(aiotieba, cfg, bduss, stoken) as client:
|
||
try:
|
||
user = await client.get_self_info()
|
||
except TiebaServerError as exc:
|
||
code = getattr(exc, "code", None)
|
||
if code == 1990006:
|
||
print(
|
||
"[X] 登录失效(错误码 1990006):BDUSS 已过期。\n"
|
||
" 重新登录百度后,从浏览器复制新的 Cookie 再跑一次。"
|
||
)
|
||
return 3
|
||
print(f"[X] 服务器返回错误 {code}: {getattr(exc, 'msg', exc)}")
|
||
return 4
|
||
except Exception as exc:
|
||
print(f"[X] 验证请求失败:{type(exc).__name__}: {exc}")
|
||
return 4
|
||
|
||
user_id = getattr(user, "user_id", 0)
|
||
uname = getattr(user, "user_name", "") or ""
|
||
nick = getattr(user, "nick_name_new", "") or ""
|
||
if not user_id:
|
||
print("[X] 服务器没有返回用户信息,Cookie 可能不完整")
|
||
return 3
|
||
print(f"[ok] 登录有效:user_id={user_id} user_name={uname} nick={nick}")
|
||
return 0
|
||
|
||
|
||
def cmd_enum(args) -> int:
|
||
if args.pages < 0:
|
||
raise ValueError("--pages 不能是负数")
|
||
cfg = Config.load(args.data)
|
||
result = asyncio.run(crawler.enum(cfg, args.pages))
|
||
print(f"[i] 主题清单:{cfg.threads_path}(共 {result['threads']:,} 个主题)")
|
||
print(f"[i] 下一步:python tieba.py crawl --threads 10 --data {cfg.data_dir}")
|
||
return 0
|
||
|
||
|
||
def cmd_crawl(args) -> int:
|
||
if args.threads < 0:
|
||
raise ValueError("--threads 不能是负数")
|
||
cfg = Config.load(args.data)
|
||
if args.workers:
|
||
cfg.workers = args.workers
|
||
if args.rate:
|
||
cfg.rate = args.rate
|
||
cfg.validate()
|
||
result = asyncio.run(crawler.crawl(cfg, args.threads, args.workers or None))
|
||
if result["threads"] or result["deleted"]:
|
||
print(f"[i] 下一步:python tieba.py import --data {cfg.data_dir}")
|
||
return 0
|
||
|
||
|
||
def cmd_import(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
if not cfg.data_dir.exists():
|
||
raise FileNotFoundError(f"数据目录不存在:{cfg.data_dir}")
|
||
result = importer.import_all(cfg, quiet=args.quiet)
|
||
if not args.quiet:
|
||
_print_stats(cfg, result["stats"])
|
||
print(f"[i] 下一步:补图 `python tieba.py images --data {cfg.data_dir}`")
|
||
print(f" 或导出清单 `python tieba.py export --data {cfg.data_dir}`")
|
||
return 0
|
||
|
||
|
||
def cmd_images(args) -> int:
|
||
if args.workers < 1:
|
||
raise ValueError("--workers 至少为 1")
|
||
if args.limit < 0:
|
||
raise ValueError("--limit 不能是负数")
|
||
cfg = Config.load(args.data)
|
||
images.download_missing(cfg, args.workers, args.limit)
|
||
return 0
|
||
|
||
|
||
def cmd_import_images(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
if not cfg.image_index_path.exists():
|
||
raise FileNotFoundError(
|
||
f"缺少 {cfg.image_index_path},先跑 `python tieba.py images --data {cfg.data_dir}`"
|
||
)
|
||
images.import_index(cfg, quiet=args.quiet)
|
||
return 0
|
||
|
||
|
||
def _sha256_file(path: Path, chunk: int = 1 << 20) -> str:
|
||
"""分块算 sha256:库有几百 MB,一次读完会占掉同样多的内存。"""
|
||
digest = hashlib.sha256()
|
||
with open(path, "rb") as fh:
|
||
while True:
|
||
block = fh.read(chunk)
|
||
if not block:
|
||
break
|
||
digest.update(block)
|
||
return digest.hexdigest()
|
||
|
||
|
||
def _count_images(path: Path) -> tuple[int, int]:
|
||
"""磁盘上的图片文件数与总字节。和 image 表的 md5 去重统计互相印证。"""
|
||
files = total = 0
|
||
if not path.exists():
|
||
return 0, 0
|
||
for p in path.rglob("*"):
|
||
if p.is_file() and p.suffix.lower() in {".jpg", ".png", ".gif", ".bmp", ".webp"}:
|
||
files += 1
|
||
total += p.stat().st_size
|
||
return files, total
|
||
|
||
|
||
def _latest_data_mtime(cfg) -> float:
|
||
"""数据文件最后一次被写入的时间。
|
||
|
||
数据库的 ``fetched_at`` 只在「插入新行」时写,后续 import 只更新字段不会刷新它;
|
||
所以再扫一遍 raw*/ 和图片索引的 mtime,两者取晚的,才是真正的「备份截止」。
|
||
故意不扫 ``tieba.db``:贴标签本身会写库,算进去会导致每次重贴时间都往后跳。
|
||
"""
|
||
newest = 0.0
|
||
paths = list(cfg.data_dir.glob("raw*/*")) + [cfg.image_index_path]
|
||
for path in paths:
|
||
try:
|
||
if path.is_file():
|
||
newest = max(newest, path.stat().st_mtime)
|
||
except OSError:
|
||
continue
|
||
return newest
|
||
|
||
|
||
def _render_label_md(manifest: dict) -> str:
|
||
"""给存档生成一张「箱子上贴的标签」:人眼可读,机器也能解析 JSON 那份。"""
|
||
s = manifest["stats"]
|
||
span = manifest["time_span"]
|
||
backup = manifest.get("backup", {})
|
||
lines = [
|
||
"# 存档标签",
|
||
"",
|
||
"| 项 | 值 |",
|
||
"|---|---|",
|
||
f"| 标签 | {manifest['label']} |",
|
||
f"| 版本 | {manifest['version']} |",
|
||
f"| 吧名 | {manifest['kw']}(fid {manifest['fid'] or '?'}) |",
|
||
f"| **备份截止** | **{backup.get('finished_at') or '?'}**(此后新发的帖不在本备份内) |",
|
||
f"| 内容截止 | {span['last']}(最后一帖的发布时间) |",
|
||
f"| 主题帖 | {s['threads']:,}(已抓完 {s['threads_done']:,}) |",
|
||
f"| 楼层 | {s['posts']:,} |",
|
||
f"| 楼中楼 | {s['comments']:,} |",
|
||
f"| 楼中楼回复对象 | {s.get('comments_targeted', 0):,} 条 |",
|
||
f"| 小尾巴 / 点赞 | {s.get('posts_signed', 0):,} 层 / {s.get('posts_agreed', 0):,} 层 |",
|
||
f"| 图片 | {s.get('images_done', 0):,} URL / {s.get('image_files', 0):,} 文件 "
|
||
f"/ {db.fmt_size(manifest['images']['bytes'])} |",
|
||
f"| 用户资料 | {s.get('users', 0):,} |",
|
||
f"| 时间跨度 | {span['first']} ~ {span['last']} |",
|
||
f"| tid 范围 | {manifest['tid_range']['min']} ~ {manifest['tid_range']['max']} |",
|
||
f"| 数据库 | {manifest['db']['file']} {db.fmt_size(manifest['db']['bytes'])} |",
|
||
f"| sha256 | {manifest['db']['sha256'] or '(未计算)'} |",
|
||
f"| 标签生成 | {manifest['created_at']} |",
|
||
f"| 工具 | {manifest['tool']} |",
|
||
"",
|
||
"> 机器可读版本见同目录 `ARCHIVE.json`;同一份标签也写进了库里的 `kv` 表。",
|
||
"",
|
||
]
|
||
return "\n".join(lines)
|
||
|
||
|
||
def cmd_label(args) -> int:
|
||
"""给当前数据目录贴一张标签:统计 + 时间跨度 + 库的 sha256,写 ARCHIVE.md/json。
|
||
|
||
标签是「这份存档是什么」的自证:搬走、刻盘、发给别人之后,靠它就能核对
|
||
内容有没有缺、有没有被改过。
|
||
"""
|
||
cfg = Config.load(args.data)
|
||
conn = db.connect(cfg, readonly=True, immutable=True)
|
||
try:
|
||
stats = db.stats(conn)
|
||
first, last = conn.execute("SELECT MIN(post_time), MAX(post_time) FROM post").fetchone()
|
||
tid_min, tid_max = conn.execute("SELECT MIN(tid), MAX(tid) FROM thread").fetchone()
|
||
fid = cfg.fid
|
||
if not fid:
|
||
# config.json 里没记 fid 时(比如直接对旧库贴标签),从 forum 表拿
|
||
row = conn.execute("SELECT fid FROM forum LIMIT 1").fetchone()
|
||
fid = row[0] if row else 0
|
||
# 「备份截止」= 库里最后一次写入(抓取/导入完成)的时间;
|
||
# 和「内容截止」(最后一帖的发布时间)是两回事,都要记。
|
||
last_write = max(
|
||
conn.execute("SELECT MAX(fetched_at) FROM post").fetchone()[0] or "",
|
||
conn.execute("SELECT MAX(fetched_at) FROM comment").fetchone()[0] or "",
|
||
) or ""
|
||
# 补用户、跑 verify 也会往库里写数据,它们的 updated_at/seen_at 也算“最后写入”
|
||
for table, column in (("user", "updated_at"), ("thread_seen", "seen_at")):
|
||
try:
|
||
value = conn.execute(f"SELECT MAX({column}) FROM {table}").fetchone()[0]
|
||
except sqlite3.Error:
|
||
value = None
|
||
last_write = max(last_write, value or "")
|
||
finally:
|
||
conn.close()
|
||
|
||
label = args.label or f"{cfg.kw}吧 全量备份"
|
||
version = args.version or "v1.0"
|
||
img_files, img_bytes = _count_images(cfg.imgs_dir)
|
||
|
||
# 备份截止 = max(库里最后一次写入, 原始数据文件的 mtime)
|
||
newest = _latest_data_mtime(cfg)
|
||
finished_at = max(
|
||
[last_write, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(newest)) if newest else ""]
|
||
) or ""
|
||
|
||
# 先把标签写进库、把 WAL 折叠回主库,再算 sha256:
|
||
# 这样哈希对应的是「最终落盘的库」,而且重复跑 label 不会自己改变自己的哈希。
|
||
wconn = db.connect(cfg)
|
||
try:
|
||
# 旧版本曾把 sha256/时间写进 kv;哈希一旦写进库就会改变库自己的哈希,
|
||
# 所以现在只留 label/version,旧键清掉,免得 SQL 查询看到过期值。
|
||
wconn.execute(
|
||
"DELETE FROM kv WHERE k IN ('archive_manifest_at', 'archive_db_sha256')"
|
||
)
|
||
for key, value in (
|
||
("archive_label", label),
|
||
("archive_version", version),
|
||
("archive_finished_at", finished_at),
|
||
("archive_content_cutoff", last or ""),
|
||
):
|
||
row = wconn.execute("SELECT v FROM kv WHERE k=?", (key,)).fetchone()
|
||
if not row or row[0] != value:
|
||
wconn.execute("INSERT OR REPLACE INTO kv(k,v) VALUES(?,?)", (key, value))
|
||
wconn.commit()
|
||
wconn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||
finally:
|
||
wconn.close()
|
||
|
||
db_bytes = cfg.db_path.stat().st_size if cfg.db_path.exists() else 0
|
||
manifest = {
|
||
"label": label,
|
||
"version": version,
|
||
"kw": cfg.kw,
|
||
"fid": fid,
|
||
"created_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
"backup": {"finished_at": finished_at, "content_cutoff": last or ""},
|
||
"time_span": {"first": first or "", "last": last or ""},
|
||
"tid_range": {"min": tid_min, "max": tid_max},
|
||
"stats": stats,
|
||
"db": {"file": cfg.db_path.name, "bytes": db_bytes, "sha256": None},
|
||
"images": {
|
||
"files": img_files,
|
||
"bytes": img_bytes,
|
||
"urls": stats.get("images_done", 0),
|
||
},
|
||
"tool": f"tieba-archiver {__version__}",
|
||
}
|
||
if not args.no_hash:
|
||
if not args.quiet:
|
||
print(f"[i] 正在算 sha256({db.fmt_size(db_bytes)})…")
|
||
manifest["db"]["sha256"] = _sha256_file(cfg.db_path)
|
||
|
||
json_path = cfg.data_dir / "ARCHIVE.json"
|
||
md_path = cfg.data_dir / "ARCHIVE.md"
|
||
json_path.write_text(
|
||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
md_path.write_text(_render_label_md(manifest), encoding="utf-8")
|
||
|
||
print(f"[ok] 标签:{label} {version}")
|
||
print(f" 主题 {stats['threads']:,} / 楼层 {stats['posts']:,} / 楼中楼 {stats['comments']:,}"
|
||
f" / 用户 {stats.get('users', 0):,}")
|
||
print(f" 清单:{json_path}")
|
||
print(f" 标签:{md_path}")
|
||
return 0
|
||
|
||
|
||
def cmd_fetch_emoji(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
conn = db.connect(cfg)
|
||
try:
|
||
stats = emoji.fetch(conn, cfg, force=args.force)
|
||
finally:
|
||
conn.close()
|
||
print(f"[ok] 表情 {stats['total']} 个:下载 {stats['ok']},已有 {stats['skip']},失败 {len(stats['failed'])}")
|
||
if stats["failed"]:
|
||
print(" 失败(CDN 上没有,渲染时回退文字):", " ".join(stats["failed"][:20]))
|
||
# 部分表情缺失不影响构建,只有全都失败才算错
|
||
return 0 if stats["ok"] or stats["skip"] else 1
|
||
|
||
|
||
def cmd_build_zim(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
out = Path(args.out) if args.out else (cfg.data_dir.parent / f"{cfg.kw}.zim")
|
||
if args.max_bytes:
|
||
parts = zim.build_zim_split(
|
||
cfg, out, zim.parse_bytes(args.max_bytes),
|
||
with_images=not args.no_images, title=args.title or "", language=args.language,
|
||
)
|
||
print()
|
||
total = sum(p["bytes"] for p in parts)
|
||
print(f"[ok] 共 {len(parts)} 个 ZIM,合计 {db.fmt_size(total)}")
|
||
for p in parts:
|
||
print(f" {Path(p['path']).name} {db.fmt_size(p['bytes'])} "
|
||
f"页面 {p['pages']:,} 图片 {p['images_added']:,}")
|
||
return 0
|
||
stats = zim.build_zim(
|
||
cfg, out, limit_threads=args.limit,
|
||
with_images=not args.no_images, title=args.title or "", language=args.language,
|
||
)
|
||
print()
|
||
print(f"[ok] {stats['path']} {db.fmt_size(stats['bytes'])}")
|
||
print(f" 页面 {stats['pages']:,},图片 {stats['images_added']:,}"
|
||
f"(引用 {stats['images']:,}),用时 {stats['seconds']/60:.1f} 分钟")
|
||
return 0
|
||
|
||
|
||
def cmd_restore(args) -> int:
|
||
if not (args.tid or args.pid or args.cid):
|
||
print("[X] 至少给一个 --tid / --pid / --cid")
|
||
return 2
|
||
cfg = Config.load(args.data)
|
||
stats = restore.restore(cfg, tid=args.tid, pid=args.pid, cid=args.cid)
|
||
print(f"[ok] 已恢复:主题 {stats['threads']},楼层 {stats['posts']},楼中楼 {stats['comments']}")
|
||
return 0
|
||
|
||
|
||
def cmd_fill_users(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
stats = users.fill_missing(cfg, limit=args.limit, rate=args.rate or None)
|
||
print(f"\n[ok] 待补 {stats['total']},补上 {stats['filled']},失败 {stats['failed']}")
|
||
return 0
|
||
|
||
|
||
def cmd_rebuild(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
stats = rebuild.check(cfg)
|
||
print(f"[i] 事件 {stats['events']:,} 条 {stats['by_kind']}")
|
||
print(f"[i] 回收站:楼层 {stats['recycled_posts']:,} / 楼中楼 {stats['recycled_comments']:,},"
|
||
f"整帖删 {stats['deleted_threads']:,}")
|
||
print(f"[i] 幂等重放:{'一致' if stats['idempotent'] else '不一致!'}")
|
||
print(f"[i] 回收站无事件:{stats['recycled_without_event']};"
|
||
f"post_add 指向不存在楼层:{stats['add_events_without_post']}")
|
||
ok = (stats["idempotent"] and not stats["recycled_without_event"]
|
||
and not stats["add_events_without_post"])
|
||
print("[ok] 一致" if ok else "[!] 有不一致项,见上")
|
||
return 0 if ok else 1
|
||
|
||
|
||
def cmd_update(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
stats = update.update(cfg, pages=args.pages, threads=args.threads,
|
||
rate=args.rate or None)
|
||
print(f"[ok] 增量完成:枚举 {stats['pages']} 页,新主题 {stats['new_threads']},"
|
||
f"有更新主题 {stats['changed_threads']},新增楼层 {stats['added']},"
|
||
f"新增楼中楼 {stats.get('comments', 0)},已删 {stats['deleted']},"
|
||
f"失败 {stats['errors']}")
|
||
return 0
|
||
|
||
|
||
def cmd_verify(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
tids = [int(x) for x in args.tids.split(",") if x.strip()] if args.tids else None
|
||
stats = verify.verify(cfg, threads=args.threads, priority=args.priority,
|
||
tids=tids, rate=args.rate or None)
|
||
print(
|
||
f"[ok] 核对完成:正常 {stats['ok']},新增 {stats['added']},编辑 {stats['edited']},"
|
||
f"删除 {stats['deleted']}(怀疑中 {stats['suspect']}),"
|
||
f"楼中楼删 {stats.get('comment_deleted', 0)}(怀疑中 {stats.get('comment_suspect', 0)}),"
|
||
f"整帖删 {stats['thread_deleted']}(怀疑中 {stats['thread_suspect']}),"
|
||
f"点赞采样 {stats['stat_samples']},失败 {stats['errors']}"
|
||
)
|
||
return 0
|
||
|
||
|
||
def cmd_asof(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
conn = db.connect(cfg)
|
||
try:
|
||
at = replay.normalize_at(args.at)
|
||
rows = replay.as_of_thread(conn, int(args.tid), at)
|
||
finally:
|
||
conn.close()
|
||
if args.json:
|
||
print(json.dumps(rows, ensure_ascii=False, indent=2))
|
||
return 0
|
||
live = sum(1 for r in rows if r["status"] == "live")
|
||
print(f"[i] tid={args.tid} 在 {at} 时:{len(rows)} 楼(live {live} / 回收站 {len(rows) - live})")
|
||
for r in rows[: args.limit]:
|
||
flag = "" if r["status"] == "live" else " [回收站]"
|
||
print(f" {r['floor']:>5} {r['author']:<12} {r['post_time']} {r['content'][:48]}{flag}")
|
||
if len(rows) > args.limit:
|
||
print(f" … 还有 {len(rows) - args.limit} 楼,用 --limit 调整或 --json")
|
||
return 0
|
||
|
||
|
||
def cmd_events(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
conn = db.connect(cfg, readonly=True, immutable=True)
|
||
try:
|
||
sql = "SELECT id, ts, kind, tid, pid, floor, evidence FROM event"
|
||
where, params = [], []
|
||
if args.tid:
|
||
where.append("tid=?")
|
||
params.append(int(args.tid))
|
||
if args.kind:
|
||
where.append("kind=?")
|
||
params.append(args.kind)
|
||
if where:
|
||
sql += " WHERE " + " AND ".join(where)
|
||
sql += " ORDER BY id DESC LIMIT ?"
|
||
params.append(int(args.limit))
|
||
rows = conn.execute(sql, params).fetchall()
|
||
finally:
|
||
conn.close()
|
||
for row in rows:
|
||
print(f" #{row[0]:<5} {row[1]} {row[2]:<14} tid={row[3]} pid={row[4]} "
|
||
f"floor={row[5]} {row[6] or ''}")
|
||
print(f"[i] 共 {len(rows)} 条(最多显示 {args.limit} 条)")
|
||
return 0
|
||
|
||
|
||
def cmd_stats(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
conn = db.connect(cfg, readonly=True, immutable=not args.live)
|
||
try:
|
||
stats = db.stats(conn)
|
||
finally:
|
||
conn.close()
|
||
_print_stats(cfg, stats)
|
||
if not args.live:
|
||
print("[i] 只读快照模式(immutable):不创建 -wal/-shm,也看不到未 checkpoint 的写入;")
|
||
print(" 要读正在导入的库,加 --live。")
|
||
return 0
|
||
|
||
|
||
def cmd_export(args) -> int:
|
||
cfg = Config.load(args.data)
|
||
count = export.export_threads(cfg, quiet=args.quiet)
|
||
if not args.quiet:
|
||
print(f"[i] 导出 {count:,} 个主题 → {cfg.threads_path}")
|
||
return 0
|
||
|
||
|
||
# ---------------------------------------------------------------- 自检
|
||
def _write_selftest_fixture(cfg: Config) -> None:
|
||
"""3 条楼层 + 2 条楼中楼的小样本,覆盖 importer 的全部关键路径。"""
|
||
cfg.ensure_dirs()
|
||
threads = {
|
||
"tid": 1001,
|
||
"fid": 2288117,
|
||
"title": "自检帖",
|
||
"author": "楼主甲",
|
||
"author_id": 11,
|
||
"reply_num": 2,
|
||
"is_good": 0,
|
||
"is_top": 0,
|
||
}
|
||
posts = [
|
||
{
|
||
"tid": 1001,
|
||
"pid": 5001,
|
||
"floor": 1,
|
||
"uid": 11,
|
||
"uname": "楼主甲",
|
||
"agree": 3,
|
||
"disagree": 0,
|
||
"sign": "自检小尾巴",
|
||
"rn": 1,
|
||
"ts": 1700000000,
|
||
"lz": True,
|
||
"text": "第一楼",
|
||
"frags": [{"t": "text", "text": "第一楼"}],
|
||
},
|
||
{
|
||
"tid": 1001,
|
||
"pid": 5002,
|
||
"floor": 2,
|
||
"uid": 22,
|
||
"uname": "路人乙",
|
||
"agree": 0,
|
||
"disagree": 0,
|
||
"sign": "",
|
||
"rn": 0,
|
||
"ts": 1700000100,
|
||
"lz": False,
|
||
"text": "带图",
|
||
"frags": [
|
||
{"t": "text", "text": "带图"},
|
||
{"t": "img", "src": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg"},
|
||
],
|
||
},
|
||
{
|
||
"tid": 1001,
|
||
"pid": 5003,
|
||
"floor": 3,
|
||
"uid": 11,
|
||
"uname": "楼主甲",
|
||
"agree": 0,
|
||
"disagree": 0,
|
||
"sign": "",
|
||
"rn": 1,
|
||
"ts": 1700000200,
|
||
"lz": True,
|
||
"text": "第三楼",
|
||
"frags": [{"t": "text", "text": "第三楼"}],
|
||
},
|
||
]
|
||
comments = [
|
||
{
|
||
"tid": 1001,
|
||
"pid": 9001,
|
||
"ppid": 5001,
|
||
"floor": 1,
|
||
"uid": 22,
|
||
"uname": "路人乙",
|
||
"rto": 11,
|
||
"agree": 1,
|
||
"disagree": 0,
|
||
"ts": 1700000300,
|
||
"lz": False,
|
||
"text": "直接回复楼主",
|
||
"frags": [{"t": "text", "text": "直接回复楼主"}],
|
||
},
|
||
{
|
||
"tid": 1001,
|
||
"pid": 9002,
|
||
"ppid": 5003,
|
||
"floor": 3,
|
||
"uid": 33,
|
||
"uname": "路人丙",
|
||
"rto": 0,
|
||
"agree": 0,
|
||
"disagree": 0,
|
||
"ts": 1700000400,
|
||
"lz": False,
|
||
"text": "回复 楼主甲 : 正文里带昵称",
|
||
"frags": [{"t": "text", "text": "回复 楼主甲 : 正文里带昵称"}],
|
||
},
|
||
]
|
||
users = [
|
||
{
|
||
"uid": 11,
|
||
"uname": "楼主甲",
|
||
"nick": "甲甲",
|
||
"portrait": "tb.1.aaa",
|
||
"level": 10,
|
||
"glevel": 5,
|
||
"gender": 1,
|
||
"ip": "上海",
|
||
"bawu": False,
|
||
"vip": True,
|
||
"icons": ["vip"],
|
||
},
|
||
{
|
||
"uid": 22,
|
||
"uname": "路人乙",
|
||
"nick": "乙乙",
|
||
"portrait": "",
|
||
"level": 3,
|
||
"glevel": 2,
|
||
"gender": 0,
|
||
"ip": "江苏",
|
||
"bawu": False,
|
||
"vip": False,
|
||
"icons": [],
|
||
},
|
||
]
|
||
files = {
|
||
cfg.threads_path: [threads],
|
||
cfg.posts_path: posts,
|
||
cfg.comments_path: comments,
|
||
cfg.users_path: users,
|
||
}
|
||
for path, records in files.items():
|
||
with open(path, "w", encoding="utf-8", newline="\n") as fh:
|
||
for record in records:
|
||
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
cfg.done_path.write_text("1001\n", encoding="utf-8")
|
||
with open(cfg.image_index_path, "w", encoding="utf-8", newline="\n") as fh:
|
||
fh.write(
|
||
json.dumps(
|
||
{
|
||
"md5": "0" * 32,
|
||
"url": "https://tiebapic.baidu.com/forum/pic/item/abc.jpg",
|
||
"size": 3,
|
||
"path": "imgs/00/00/" + "0" * 32 + ".jpg",
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
+ "\n"
|
||
)
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def _selftest_tempdir():
|
||
"""为自检找一个可写的临时目录。
|
||
|
||
正常环境用系统 TEMP;某些 Windows 机器(组策略/沙箱)会把 TEMP 设成
|
||
不可写,这时退到当前工作目录。目录用 ``Path.mkdir`` 建:``mkdtemp``
|
||
在 Windows 上会带 mode=0700 的 ACL,受限令牌下反而写不进去。
|
||
|
||
都不行才报错。
|
||
"""
|
||
candidates = []
|
||
env = os.environ.get("TIEBA_TMPDIR")
|
||
if env:
|
||
candidates.append(Path(env))
|
||
candidates.append(Path(tempfile.gettempdir()))
|
||
candidates.append(Path.cwd())
|
||
|
||
path = None
|
||
last_exc: Exception | None = None
|
||
for root in candidates:
|
||
try:
|
||
root.mkdir(parents=True, exist_ok=True)
|
||
candidate = root / f"tieba-archiver-selftest-{os.getpid()}-{time.time_ns()}"
|
||
candidate.mkdir()
|
||
path = candidate
|
||
break
|
||
except OSError as exc:
|
||
last_exc = exc
|
||
if path is None:
|
||
raise RuntimeError(f"找不到可写的临时目录:{last_exc}")
|
||
try:
|
||
yield path
|
||
finally:
|
||
shutil.rmtree(path, ignore_errors=True)
|
||
|
||
|
||
def _run_selftest() -> int:
|
||
checks: list[bool] = []
|
||
|
||
def check(name, fn):
|
||
try:
|
||
fn()
|
||
print(f"[ok] {name}")
|
||
checks.append(True)
|
||
except Exception as exc:
|
||
print(f"[X] {name}: {type(exc).__name__}: {exc}")
|
||
traceback.print_exc()
|
||
checks.append(False)
|
||
|
||
def cookie_case():
|
||
header = (
|
||
'BAIDUID=AAA:FG=1; BDUSS=BBB; STOKEN=CCC; '
|
||
'RT="z=1&dm=baidu.com&si=x;co=y"; XFT=DDD'
|
||
)
|
||
jar = parse_cookie_text(header)
|
||
assert jar["BDUSS"] == "BBB", jar
|
||
assert len(jar) == 5, jar
|
||
assert ";" in jar["RT"] and "co=y" in jar["RT"], jar["RT"]
|
||
assert parse_cookie_text('{"BDUSS": "x", "STOKEN": "y"}')["STOKEN"] == "y"
|
||
assert (
|
||
parse_cookie_text("curl 'https://tieba.baidu.com' -H 'Cookie: BDUSS=x; STOKEN=y'")[
|
||
"BDUSS"
|
||
]
|
||
== "x"
|
||
)
|
||
assert parse_cookie_text("curl -b 'BDUSS=x; STOKEN=y' https://tieba.baidu.com")[
|
||
"STOKEN"
|
||
] == "y"
|
||
try:
|
||
validate_cookies({"STOKEN": "y"})
|
||
except CookieError:
|
||
pass
|
||
else:
|
||
raise AssertionError("缺 BDUSS 时应该报错")
|
||
|
||
def reply_case():
|
||
assert importer.parse_reply_name("回复 楼主甲 : 你好") == "楼主甲"
|
||
assert importer.parse_reply_name("回复楼主甲:你好") == "楼主甲"
|
||
assert importer.parse_reply_name("没有前缀") is None
|
||
# 超过 30 字的名字不当昵称(避免把长正文误认成回复对象)
|
||
long_name = "一二三四五六七八九十" * 3 + "一"
|
||
assert len(long_name) > 30
|
||
assert importer.parse_reply_name(f"回复 {long_name} : x") is None
|
||
|
||
check("Cookie 解析(引号内分号 / JSON / cURL / 缺 BDUSS)", cookie_case)
|
||
check("回复对象正则", reply_case)
|
||
|
||
with _selftest_tempdir() as tmp:
|
||
cfg = Config(kw="东方年代记", data_dir=Path(tmp), rate=1000.0)
|
||
cfg.ensure_dirs()
|
||
cfg.save()
|
||
|
||
def config_case():
|
||
loaded = Config.load(cfg.data_dir)
|
||
assert loaded.kw == cfg.kw
|
||
assert loaded.rate == 1000.0
|
||
assert loaded.db_path == Path(tmp) / "tieba.db"
|
||
|
||
def schema_case():
|
||
conn = db.connect(cfg)
|
||
try:
|
||
db.init_db(conn)
|
||
post_cols = db.table_columns(conn, "post")
|
||
comment_cols = db.table_columns(conn, "comment")
|
||
user_cols = db.table_columns(conn, "user")
|
||
assert {"pid", "tid", "agree", "disagree", "sign", "frags"} <= post_cols
|
||
assert {"cid", "rto", "reply_to_name", "reply_to_uid", "frags"} <= comment_cols
|
||
assert {"uid", "uname", "nick", "portrait", "bawu", "vip"} <= user_cols
|
||
finally:
|
||
conn.close()
|
||
|
||
check("config.json 往返", config_case)
|
||
check("schema 建库(v2 列齐全)", schema_case)
|
||
|
||
try:
|
||
_write_selftest_fixture(cfg)
|
||
first = importer.import_all(cfg, quiet=True)
|
||
second = importer.import_all(cfg, quiet=True)
|
||
except Exception as exc:
|
||
print(f"[X] 造 fixture / 导入失败:{type(exc).__name__}: {exc}")
|
||
traceback.print_exc()
|
||
return 1
|
||
|
||
def import_case():
|
||
assert first["posts"] == 3, first
|
||
assert first["comments"] == 2, first
|
||
assert first["users"] == 2, first
|
||
assert second["posts"] == 3 and second["comments"] == 2, second
|
||
assert second["stats"]["posts"] == 3
|
||
assert second["stats"]["comments"] == 2
|
||
assert second["stats"]["posts_signed"] == 1
|
||
assert second["stats"]["posts_agreed"] == 1
|
||
assert second["stats"]["comments_targeted"] == 2
|
||
assert second["stats"]["threads_done"] == 1
|
||
assert second["stats"]["threads_with_posts"] == 1
|
||
assert second["stats"]["users"] == 2
|
||
|
||
def reply_resolution_case():
|
||
conn = db.connect(cfg, readonly=True)
|
||
try:
|
||
rows = {
|
||
row["cid"]: (row["reply_to_name"], row["reply_to_uid"])
|
||
for row in conn.execute(
|
||
"SELECT cid, reply_to_name, reply_to_uid FROM comment"
|
||
)
|
||
}
|
||
finally:
|
||
conn.close()
|
||
assert rows[9001] == (None, 11), rows # rto 优先
|
||
assert rows[9002] == ("楼主甲", 11), rows # 正文昵称 → 本帖作者
|
||
|
||
def export_case():
|
||
count = export.export_threads(cfg, quiet=True)
|
||
assert count == 1
|
||
lines = cfg.threads_path.read_text(encoding="utf-8").strip().splitlines()
|
||
assert len(lines) == 1, lines
|
||
assert json.loads(lines[0])["tid"] == 1001
|
||
|
||
def image_case():
|
||
result = images.import_index(cfg, quiet=True)
|
||
assert result["filled"] == 1, result
|
||
assert result["stats"]["images"] >= 1
|
||
assert result["stats"]["image_files"] == 1
|
||
|
||
check("importer 小样本 + 幂等", import_case)
|
||
check("回复对象解析(rto 优先 / 正文昵称)", reply_resolution_case)
|
||
check("export threads.jsonl", export_case)
|
||
check("import-images(index.jsonl → image 表)", image_case)
|
||
|
||
def stats_case():
|
||
conn = db.connect(cfg, readonly=True, immutable=True)
|
||
try:
|
||
stats = db.stats(conn)
|
||
finally:
|
||
conn.close()
|
||
assert stats["threads"] == 1
|
||
assert stats["images"] == 1
|
||
|
||
check("stats 只读统计", stats_case)
|
||
|
||
passed = sum(checks)
|
||
print(f"\n[{'ok' if passed == len(checks) else 'X'}] 自检 {passed}/{len(checks)} 项通过(不联网)")
|
||
return 0 if passed == len(checks) else 1
|
||
|
||
|
||
def cmd_selftest(args) -> int:
|
||
print("[i] 离线自检:Cookie / 正则 / schema / importer / export / 图片入库")
|
||
return _run_selftest()
|
||
|
||
|
||
# ---------------------------------------------------------------- 入口
|
||
def main(argv: list[str] | None = None) -> int:
|
||
setup_console()
|
||
parser = build_parser()
|
||
args = parser.parse_args(argv)
|
||
if not getattr(args, "func", None):
|
||
parser.print_help()
|
||
return 1
|
||
|
||
try:
|
||
result = args.func(args)
|
||
return int(result or 0)
|
||
except KeyboardInterrupt:
|
||
print("\n[!] 已中断;done.txt 里已写的帖不会重抓,重跑同一条命令即可续传")
|
||
return 130
|
||
except CookieError as exc:
|
||
print(f"[X] {exc}", file=sys.stderr)
|
||
return 2
|
||
except FileNotFoundError as exc:
|
||
print(f"[X] {exc}", file=sys.stderr)
|
||
return 2
|
||
except (crawler.CrawlError, RuntimeError, ValueError, sqlite3.Error) as exc:
|
||
print(f"[X] {exc}", file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__": # pragma: no cover
|
||
sys.exit(main())
|