从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
241 lines
8.8 KiB
Python
241 lines
8.8 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""运行配置:一个数据目录对应一套采集参数。
|
||
|
||
为什么把参数收进 Config,而不是像旧脚本那样散在模块常量里:
|
||
|
||
* 一台机器上可能同时维护好几个吧的归档,每个 ``data/`` 一套参数,
|
||
用模块常量就做不到互不干扰;
|
||
* 命令行上的 ``--workers`` / ``--rate`` 只覆盖本次运行,不应该写回
|
||
``config.json``,否则一次试水会永久改变全量任务的行为;
|
||
* 测试可以构造一个指向临时目录的 Config,完全不碰真实数据。
|
||
|
||
``config.json`` 里只存参数,不存 ``data_dir``(数据目录由 ``--data`` 决定,
|
||
存进去反而会在搬目录后失效)。``fid`` / ``forum_name`` 是 enum 阶段自动
|
||
探测回填的归档元数据,方便 import 时补全 forum 表。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
from dataclasses import asdict, dataclass, fields
|
||
from pathlib import Path
|
||
|
||
DEFAULT_KW = "东方年代记"
|
||
CONFIG_NAME = "config.json"
|
||
DB_NAME = "tieba.db"
|
||
COOKIE_NAME = "cookies.txt"
|
||
|
||
# config.json 里的数字字段可能在手改后变成字符串,读回来时统一转一次。
|
||
_COERCE = {
|
||
"rate": float,
|
||
"workers": int,
|
||
"comment_rn": int,
|
||
"timeout": float,
|
||
"page_cap": int,
|
||
"post_rn": int,
|
||
"thread_rn": int,
|
||
"tries": int,
|
||
"backoff": float,
|
||
"comment_page_cap": int,
|
||
"user_dump_every": int,
|
||
"user_dump_seconds": float,
|
||
"flush_every": int,
|
||
"fid": int,
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class Config:
|
||
"""一套归档任务的参数。
|
||
|
||
必填两项:``kw``(吧名)和 ``data_dir``(数据目录)。
|
||
其余都有默认值,可以在 ``config.json`` 里覆盖。
|
||
"""
|
||
|
||
# ---- 必填 ----
|
||
kw: str
|
||
data_dir: Path
|
||
|
||
# ---- 采集节奏 ----
|
||
rate: float = 1.3 # 全局限速:每秒请求数(所有 worker 共享)
|
||
workers: int = 2 # 抓帖并发
|
||
comment_rn: int = 50 # 一次请求带多少楼中楼(aiotieba 上限 50)
|
||
timeout: float = 25.0 # 单请求超时(秒)
|
||
page_cap: int = 400 # 单帖最多翻多少页,防止遇到水楼跑飞
|
||
|
||
# ---- 内部参数:一般不用改 ----
|
||
post_rn: int = 30 # 楼层每页条数(aiotieba 上限 30)
|
||
thread_rn: int = 30 # 主题列表每页条数
|
||
tries: int = 3 # 同一页连续空结果几次后跳过该帖
|
||
backoff: float = 4.0 # 空结果退避基数(秒)
|
||
comment_page_cap: int = 200 # 楼中楼补抓最多翻多少页
|
||
user_dump_every: int = 500 # 用户资料攒够多少条落一次盘
|
||
user_dump_seconds: float = 60.0 # 或者最多隔多少秒落一次盘
|
||
flush_every: int = 300 # 楼层/楼中楼每写多少条 flush 一次
|
||
|
||
# ---- enum 自动回填 ----
|
||
fid: int = 0
|
||
forum_name: str = ""
|
||
|
||
def __post_init__(self) -> None:
|
||
self.data_dir = Path(self.data_dir)
|
||
self.kw = str(self.kw or "").strip()
|
||
self.validate()
|
||
|
||
# ------------------------------------------------------------ 校验
|
||
def validate(self) -> None:
|
||
"""尽早报错,别等抓到一半才发现 ``--rate 0`` 会除零。"""
|
||
if not self.kw:
|
||
raise ValueError("kw(吧名)不能为空")
|
||
if float(self.rate) <= 0:
|
||
raise ValueError("rate 必须大于 0(单位:请求/秒)")
|
||
if not (1 <= int(self.workers) <= 64):
|
||
raise ValueError("workers 必须在 1..64 之间")
|
||
if not (1 <= int(self.comment_rn) <= 50):
|
||
raise ValueError("comment_rn 必须在 1..50 之间(aiotieba 上限 50)")
|
||
if not (2 <= int(self.post_rn) <= 30):
|
||
raise ValueError("post_rn 必须在 2..30 之间(aiotieba 上限 30)")
|
||
if not (1 <= int(self.thread_rn) <= 100):
|
||
raise ValueError("thread_rn 必须在 1..100 之间")
|
||
if float(self.timeout) <= 0:
|
||
raise ValueError("timeout 必须大于 0")
|
||
if int(self.page_cap) < 1:
|
||
raise ValueError("page_cap 必须大于等于 1")
|
||
if int(self.tries) < 1:
|
||
raise ValueError("tries 必须大于等于 1")
|
||
if float(self.backoff) < 0:
|
||
raise ValueError("backoff 不能为负")
|
||
if int(self.comment_page_cap) < 1:
|
||
raise ValueError("comment_page_cap 必须大于等于 1")
|
||
if int(self.user_dump_every) < 1:
|
||
raise ValueError("user_dump_every 必须大于等于 1")
|
||
if float(self.user_dump_seconds) <= 0:
|
||
raise ValueError("user_dump_seconds 必须大于 0")
|
||
if int(self.flush_every) < 1:
|
||
raise ValueError("flush_every 必须大于等于 1")
|
||
|
||
# ------------------------------------------------------------ 路径
|
||
@property
|
||
def config_path(self) -> Path:
|
||
return self.data_dir / CONFIG_NAME
|
||
|
||
@property
|
||
def db_path(self) -> Path:
|
||
return self.data_dir / DB_NAME
|
||
|
||
@property
|
||
def cookies_path(self) -> Path:
|
||
return self.data_dir / COOKIE_NAME
|
||
|
||
@property
|
||
def threads_path(self) -> Path:
|
||
return self.data_dir / "threads.jsonl"
|
||
|
||
@property
|
||
def raw_dir(self) -> Path:
|
||
return self.data_dir / "raw"
|
||
|
||
@property
|
||
def posts_path(self) -> Path:
|
||
return self.raw_dir / "posts.jsonl"
|
||
|
||
@property
|
||
def comments_path(self) -> Path:
|
||
return self.raw_dir / "comments.jsonl"
|
||
|
||
@property
|
||
def users_path(self) -> Path:
|
||
return self.raw_dir / "users.jsonl"
|
||
|
||
@property
|
||
def done_path(self) -> Path:
|
||
return self.raw_dir / "done.txt"
|
||
|
||
@property
|
||
def err_path(self) -> Path:
|
||
return self.raw_dir / "err.jsonl"
|
||
|
||
@property
|
||
def imgs_dir(self) -> Path:
|
||
return self.data_dir / "imgs"
|
||
|
||
@property
|
||
def image_index_path(self) -> Path:
|
||
return self.imgs_dir / "index.jsonl"
|
||
|
||
@property
|
||
def image_state_path(self) -> Path:
|
||
return self.imgs_dir / "state.json"
|
||
|
||
def ensure_dirs(self) -> None:
|
||
"""建出工具要用的目录;只建目录,不写文件。"""
|
||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||
self.raw_dir.mkdir(parents=True, exist_ok=True)
|
||
self.imgs_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# ------------------------------------------------------------ 读写
|
||
def to_dict(self) -> dict:
|
||
data = asdict(self)
|
||
data.pop("data_dir", None) # 数据目录由 --data 决定,不写进配置
|
||
return data
|
||
|
||
def save(self) -> Path:
|
||
"""原子写 config.json,避免断电留下半个 JSON。"""
|
||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||
tmp = self.config_path.with_suffix(".json.tmp")
|
||
text = json.dumps(self.to_dict(), ensure_ascii=False, indent=2) + "\n"
|
||
with open(tmp, "w", encoding="utf-8", newline="\n") as fh:
|
||
fh.write(text)
|
||
fh.flush()
|
||
os.fsync(fh.fileno())
|
||
os.replace(tmp, self.config_path)
|
||
return self.config_path
|
||
|
||
@classmethod
|
||
def load(cls, data_dir: str | Path, **overrides) -> "Config":
|
||
"""读 ``<data>/config.json``,再用命令行参数覆盖。
|
||
|
||
配置不存在也能加载(stats / export 打开旧归档时就是这种情况),
|
||
这时用默认参数,只要不改文件就不会写入任何东西。
|
||
"""
|
||
data_dir = Path(data_dir)
|
||
path = data_dir / CONFIG_NAME
|
||
stored: dict = {}
|
||
if path.exists():
|
||
try:
|
||
raw = json.loads(path.read_text(encoding="utf-8") or "{}")
|
||
except json.JSONDecodeError as exc:
|
||
raise ValueError(f"{path} 不是合法 JSON:{exc}") from exc
|
||
if not isinstance(raw, dict):
|
||
raise ValueError(f"{path} 应该是一个 JSON 对象")
|
||
known = {f.name for f in fields(cls)} - {"data_dir"}
|
||
for key, value in raw.items():
|
||
if key not in known:
|
||
continue
|
||
try:
|
||
stored[key] = _COERCE[key](value) if key in _COERCE else value
|
||
except (TypeError, ValueError) as exc:
|
||
raise ValueError(f"config.json 的 {key}={value!r} 类型不对:{exc}") from exc
|
||
|
||
kw = overrides.pop("kw", None) or stored.pop("kw", None) or DEFAULT_KW
|
||
for key, value in overrides.items():
|
||
if value is not None:
|
||
stored[key] = value
|
||
return cls(kw=kw, data_dir=data_dir, **stored)
|
||
|
||
|
||
def read_tid_set(path: Path) -> set[int]:
|
||
"""读 ``done.txt`` 这类「一行一个 tid」的文件,坏行直接跳过。
|
||
|
||
断点续传优先:文件可能是崩溃时留下的,末尾半行不应该让整个任务起不来。
|
||
"""
|
||
tids: set[int] = set()
|
||
if not path.exists():
|
||
return tids
|
||
with open(path, encoding="utf-8", errors="ignore") as fh:
|
||
for line in fh:
|
||
line = line.strip()
|
||
if line.isdigit():
|
||
tids.add(int(line))
|
||
return tids
|