Files
dsh 0824e0ef83 v1.2.0: 事件流对账、增量 update、ZIM 发布与表情渲染
从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
2026-11-18 11:41:25 +08:00

328 lines
14 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""verify:把帖从头完整翻一遍,和库里对账,产生 add / del / edit 事件。
为什么要单独一个命令:
* ``update`` 只抓新内容,永远不会发现「这层没了」「这句被改了」;
* 贴吧没有删除/编辑通知,唯一办法是重新看一遍;
* 所以 verify 是事件流里 **唯一能产生删除/修改事件** 的来源。
安全规则(都很关键):
1. 只有整帖翻完(``has_more=False``、没有 429/异常)才做比对,翻一半断掉直接跳过;
2. 删除要**两次确认**:第一次只记 verify_suspect,第二次还缺才发 ``post_del``;
3. 整帖 4xx 也要两次确认才发 ``thread_deleted``,避免审核临时屏蔽造成误伤;
4. 点赞只记变化点,不算编辑(``content_hash`` 不含 agree/disagree)。
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
import time
from . import crawler, db, events
from .config import Config
def _imgs_json(rec: dict) -> str:
return json.dumps(
[f.get("src") for f in (rec.get("frags") or [])
if isinstance(f, dict) and f.get("t") == "img" and f.get("src")],
ensure_ascii=False,
)
def _hash_record(rec: dict) -> str:
return events.content_hash(rec.get("text") or "", _imgs_json(rec), rec.get("uname") or "")
def _row_from_record(rec: dict) -> dict:
ts = rec.get("ts") or 0
post_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) if ts else ""
imgs = _imgs_json(rec)
return {
"pid": int(rec["pid"]),
"tid": int(rec["tid"]),
"floor": rec.get("floor") or 0,
"author": rec.get("uname") or "",
"author_id": str(rec.get("uid") or ""),
"content": rec.get("text") or "",
"content_html": "",
"post_time": post_time,
"comment_num": rec.get("rn") or 0,
"imgs": imgs,
"is_lz": 1 if rec.get("lz") else 0,
"fetched_at": events.now(),
"agree": rec.get("agree") or 0,
"disagree": rec.get("disagree") or 0,
"sign": rec.get("sign") or "",
"frags": json.dumps(rec.get("frags") or [], ensure_ascii=False),
"status": "live",
"content_hash": _hash_record(rec),
}
def _emit_and_apply(conn: sqlite3.Connection, log: events.EventLog, ev: dict) -> None:
events.apply_event(conn, ev)
def _on_thread_deleted(conn, log, tid: int, stats: dict) -> str:
row = conn.execute("SELECT hits FROM thread_suspect WHERE tid=?", (tid,)).fetchone()
hits = (row[0] if row else 0) + 1
if hits >= 2:
ev = log.emit("thread_deleted", tid=tid, source="verify",
evidence=f"missing_hits={hits}")
_emit_and_apply(conn, log, ev)
conn.execute("DELETE FROM thread_suspect WHERE tid=?", (tid,))
stats["thread_deleted"] += 1
else:
conn.execute(
"INSERT INTO thread_suspect(tid, first_seen_at, last_seen_at, hits) VALUES(?,?,?,1) "
"ON CONFLICT(tid) DO UPDATE SET last_seen_at=excluded.last_seen_at, hits=hits+1",
(tid, events.now(), events.now()),
)
stats["thread_suspect"] += 1
return "deleted"
def _reconcile(conn: sqlite3.Connection, log: events.EventLog, tid: int,
observed: dict[int, dict], stats: dict,
observed_comments: dict[int, int] | None = None,
complete_pids: set[int] | None = None) -> None:
"""把「服务器这次看到的」和「库里 live 的」对齐。"""
rows = {}
for row in conn.execute(
"SELECT pid, floor, author, content, imgs, frags, agree, disagree, content_hash, status "
"FROM post WHERE tid=?", (tid,)
):
rows[int(row[0])] = {
"floor": row[1], "author": row[2], "content": row[3], "imgs": row[4],
"frags": row[5], "agree": row[6], "disagree": row[7],
"content_hash": row[8], "status": row[9],
}
live = {pid for pid, r in rows.items() if (r["status"] or "live") == "live"}
# ---- 新增 ----
for pid, rec in observed.items():
if pid in rows:
continue
row = _row_from_record(rec)
ev = log.emit("post_add", tid=tid, pid=pid, floor=row["floor"], payload=row,
source="verify")
_emit_and_apply(conn, log, ev)
stats["added"] += 1
# ---- 编辑 / 点赞采样 ----
now = events.now()
for pid, rec in observed.items():
row = rows.get(pid)
if not row:
continue
digest = _hash_record(rec)
if row["content_hash"] is None:
conn.execute("UPDATE post SET content_hash=? WHERE pid=?", (digest, pid))
elif row["content_hash"] != digest:
payload = {
"content": rec.get("text") or "",
"frags": json.dumps(rec.get("frags") or [], ensure_ascii=False),
"imgs": _imgs_json(rec),
"content_hash": digest,
# 旧值:as-of 倒放时要靠它撤销
"old_content": row["content"],
"old_frags": row["frags"],
"old_imgs": row["imgs"],
"old_content_hash": row["content_hash"],
}
ev = log.emit("post_edit", tid=tid, pid=pid, floor=rec.get("floor"),
payload=payload, source="verify")
_emit_and_apply(conn, log, ev)
stats["edited"] += 1
if (row["agree"] or 0) != (rec.get("agree") or 0) or \
(row["disagree"] or 0) != (rec.get("disagree") or 0):
conn.execute(
"INSERT OR IGNORE INTO post_stat_history(pid, ts, agree, disagree) VALUES(?,?,?,?)",
(pid, now, rec.get("agree") or 0, rec.get("disagree") or 0),
)
conn.execute("UPDATE post SET agree=?, disagree=? WHERE pid=?",
(rec.get("agree") or 0, rec.get("disagree") or 0, pid))
stats["stat_samples"] += 1
# ---- 删除(两次确认) ----
for pid in sorted(live - set(observed)):
row = conn.execute("SELECT hits FROM verify_suspect WHERE tid=? AND pid=?",
(tid, pid)).fetchone()
hits = (row[0] if row else 0) + 1
if hits >= 2:
ev = log.emit("post_del", tid=tid, pid=pid, floor=rows[pid]["floor"],
source="verify", evidence=f"missing_hits={hits}")
_emit_and_apply(conn, log, ev)
conn.execute("DELETE FROM verify_suspect WHERE tid=? AND pid=?", (tid, pid))
stats["deleted"] += 1
else:
conn.execute(
"INSERT INTO verify_suspect(tid, pid, first_seen_at, last_seen_at, hits) "
"VALUES(?,?,?,?,1) "
"ON CONFLICT(tid, pid) DO UPDATE SET last_seen_at=excluded.last_seen_at, hits=hits+1",
(tid, pid, now, now),
)
stats["suspect"] += 1
# 这次又看到的 pid,之前的怀疑作废
if observed:
conn.execute(
f"DELETE FROM verify_suspect WHERE tid=? AND pid IN ({','.join('?' for _ in observed)})",
(tid, *observed.keys()),
)
# ---- 快照与状态 ----
max_floor = max((int(rec.get("floor") or 0) for rec in observed.values()), default=0)
row = conn.execute("SELECT reply_num FROM thread WHERE tid=?", (tid,)).fetchone()
conn.execute(
"INSERT OR REPLACE INTO thread_seen(tid, reply_num, max_floor, seen_at) VALUES(?,?,?,?)",
(tid, row[0] if row else 0, max_floor, now),
)
conn.execute("DELETE FROM thread_suspect WHERE tid=?", (tid,))
conn.execute("UPDATE thread SET updated_at=? WHERE tid=?", (now, tid))
# ---- 楼中楼删除:只比对「完整重取」的楼层 ----
if observed_comments is not None and complete_pids:
marks = ",".join("?" for _ in complete_pids)
seen = set(observed_comments)
rows = conn.execute(
f"SELECT cid, pid FROM comment WHERE tid=? AND status='live' AND pid IN ({marks})",
(tid, *complete_pids),
).fetchall()
for cid, pid in rows:
if int(cid) in seen:
continue
row = conn.execute("SELECT hits FROM comment_suspect WHERE cid=?", (cid,)).fetchone()
hits = (row[0] if row else 0) + 1
if hits >= 2:
ev = log.emit("comment_del", tid=tid, pid=int(pid), cid=int(cid),
source="verify", evidence=f"missing_hits={hits}")
_emit_and_apply(conn, log, ev)
conn.execute("DELETE FROM comment_suspect WHERE cid=?", (cid,))
stats["comment_deleted"] = stats.get("comment_deleted", 0) + 1
else:
conn.execute(
"INSERT INTO comment_suspect(cid, first_seen_at, last_seen_at, hits) "
"VALUES(?,?,?,1) ON CONFLICT(cid) DO UPDATE SET "
"last_seen_at=excluded.last_seen_at, hits=hits+1",
(cid, now, now),
)
stats["comment_suspect"] = stats.get("comment_suspect", 0) + 1
for cid in seen:
conn.execute("DELETE FROM comment_suspect WHERE cid=?", (cid,))
async def verify_thread(client, cfg: Config, conn: sqlite3.Connection,
log: events.EventLog, tid: int, stats: dict) -> str:
observed: dict[int, dict] = {}
observed_comments: dict[int, int] = {}
complete_pids: set[int] = set()
pn = 1
while True:
page = None
posts = []
for attempt in range(max(1, cfg.tries)):
await crawler.pace()
page = await client.get_posts(
tid, pn=pn, rn=cfg.post_rn, with_comments=True, comment_rn=cfg.comment_rn
)
err = getattr(page, "err", None)
if getattr(err, "code", None) == 4 or (isinstance(err, tuple) and err and err[0] == 4):
return _on_thread_deleted(conn, log, tid, stats)
posts = getattr(page, "objs", None) or []
if posts:
break
more = getattr(getattr(page, "page", None), "has_more", None)
if getattr(page, "err", None) is None and more is False:
break # 正常翻到底
if attempt + 1 < max(1, cfg.tries):
await asyncio.sleep(cfg.backoff * (attempt + 1))
if not posts:
more = getattr(getattr(page, "page", None), "has_more", None)
if getattr(page, "err", None) is None and more is False:
break
raise RuntimeError(f"第 {pn} 页连续空结果(可能是 429),本轮跳过")
for post in posts:
rec = crawler.post_record(tid, post)
observed[int(rec["pid"])] = rec
inline = getattr(post, "comments", None) or []
if int(getattr(post, "reply_num", 0) or 0) <= len(inline):
complete_pids.add(int(post.pid))
for comment in inline:
observed_comments[int(comment.pid)] = int(post.pid)
pn += 1
if not getattr(getattr(page, "page", None), "has_more", False):
break
if pn > cfg.page_cap:
raise RuntimeError(f"超过页数上限 {cfg.page_cap}")
_reconcile(conn, log, tid, observed, stats, observed_comments, complete_pids)
return "ok"
async def _verify_loop(cfg, todo, conn, log, stats, bduss, stoken) -> None:
aiotieba = crawler.require_aiotieba()
async with aiotieba.Client(BDUSS=bduss, STOKEN=stoken) as client:
for i, tid in enumerate(todo, 1):
try:
await verify_thread(client, cfg, conn, log, tid, stats)
stats["ok"] += 1
except Exception as exc:
stats["errors"] += 1
print(f" [!] tid={tid} {type(exc).__name__}: {exc}", flush=True)
conn.commit()
if i % 20 == 0:
print(f"\r {i}/{len(todo)} 新增{stats['added']} 编辑{stats['edited']} "
f"删除{stats['deleted']} 怀疑{stats['suspect']} "
f"整帖删{stats['thread_deleted']} 失败{stats['errors']}", end="", flush=True)
def verify(cfg: Config, threads: int = 0, priority: str = "recent",
tids: list[int] | None = None, rate: float | None = None) -> dict:
bduss, stoken = crawler.read_auth(cfg)
if rate:
cfg.rate = float(rate)
cfg.validate()
crawler.set_rate(cfg.rate)
conn = db.connect(cfg)
db.init_db(conn)
db.ensure_compat(conn)
if tids:
todo = [int(t) for t in tids]
else:
order = {
"recent": "ORDER BY reply_num DESC",
"oldest": "ORDER BY tid ASC",
"random": "ORDER BY RANDOM()",
}.get(priority)
if order:
sql = f"SELECT tid FROM thread WHERE deleted_at IS NULL {order}"
elif priority == "unhashed":
# 只挑还没算过 content_hash 的帖:核对覆盖率能均匀推进,不重复干活
sql = ("SELECT tid FROM thread WHERE deleted_at IS NULL "
"AND tid IN (SELECT tid FROM post WHERE content_hash IS NULL) "
"ORDER BY reply_num ASC")
else:
sql = "SELECT tid FROM thread WHERE deleted_at IS NULL ORDER BY RANDOM()"
if threads:
sql += f" LIMIT {int(threads)}"
todo = [int(r[0]) for r in conn.execute(sql)]
print(f"[i] 待核对 {len(todo):,} 帖,优先级 {priority},限速 {cfg.rate}/s")
batch = events.start_batch(conn, "verify", note=f"threads={len(todo)} priority={priority}")
log = events.EventLog(conn, cfg, batch, "verify")
stats = {"ok": 0, "added": 0, "edited": 0, "deleted": 0, "suspect": 0,
"thread_deleted": 0, "thread_suspect": 0, "stat_samples": 0, "errors": 0}
asyncio.run(_verify_loop(cfg, todo, conn, log, stats, bduss, stoken))
log.close()
events.finish_batch(conn, batch)
conn.commit()
print()
return stats