从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
328 lines
14 KiB
Python
328 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""verify:把帖从头完整翻一遍,和库里对账,产生 add / del / edit 事件。
|
||
|
||
为什么要单独一个命令:
|
||
* ``update`` 只抓新内容,永远不会发现「这层没了」「这句被改了」;
|
||
* 贴吧没有删除/编辑通知,唯一办法是重新看一遍;
|
||
* 所以 verify 是事件流里 **唯一能产生删除/修改事件** 的来源。
|
||
|
||
安全规则(都很关键):
|
||
1. 只有整帖翻完(``has_more=False``、没有 429/异常)才做比对,翻一半断掉直接跳过;
|
||
2. 删除要**两次确认**:第一次只记 verify_suspect,第二次还缺才发 ``post_del``;
|
||
3. 整帖 4xx 也要两次确认才发 ``thread_deleted``,避免审核临时屏蔽造成误伤;
|
||
4. 点赞只记变化点,不算编辑(``content_hash`` 不含 agree/disagree)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import sqlite3
|
||
import time
|
||
|
||
from . import crawler, db, events
|
||
from .config import Config
|
||
|
||
|
||
def _imgs_json(rec: dict) -> str:
|
||
return json.dumps(
|
||
[f.get("src") for f in (rec.get("frags") or [])
|
||
if isinstance(f, dict) and f.get("t") == "img" and f.get("src")],
|
||
ensure_ascii=False,
|
||
)
|
||
|
||
|
||
def _hash_record(rec: dict) -> str:
|
||
return events.content_hash(rec.get("text") or "", _imgs_json(rec), rec.get("uname") or "")
|
||
|
||
|
||
def _row_from_record(rec: dict) -> dict:
|
||
ts = rec.get("ts") or 0
|
||
post_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) if ts else ""
|
||
imgs = _imgs_json(rec)
|
||
return {
|
||
"pid": int(rec["pid"]),
|
||
"tid": int(rec["tid"]),
|
||
"floor": rec.get("floor") or 0,
|
||
"author": rec.get("uname") or "",
|
||
"author_id": str(rec.get("uid") or ""),
|
||
"content": rec.get("text") or "",
|
||
"content_html": "",
|
||
"post_time": post_time,
|
||
"comment_num": rec.get("rn") or 0,
|
||
"imgs": imgs,
|
||
"is_lz": 1 if rec.get("lz") else 0,
|
||
"fetched_at": events.now(),
|
||
"agree": rec.get("agree") or 0,
|
||
"disagree": rec.get("disagree") or 0,
|
||
"sign": rec.get("sign") or "",
|
||
"frags": json.dumps(rec.get("frags") or [], ensure_ascii=False),
|
||
"status": "live",
|
||
"content_hash": _hash_record(rec),
|
||
}
|
||
|
||
|
||
def _emit_and_apply(conn: sqlite3.Connection, log: events.EventLog, ev: dict) -> None:
|
||
events.apply_event(conn, ev)
|
||
|
||
|
||
def _on_thread_deleted(conn, log, tid: int, stats: dict) -> str:
|
||
row = conn.execute("SELECT hits FROM thread_suspect WHERE tid=?", (tid,)).fetchone()
|
||
hits = (row[0] if row else 0) + 1
|
||
if hits >= 2:
|
||
ev = log.emit("thread_deleted", tid=tid, source="verify",
|
||
evidence=f"missing_hits={hits}")
|
||
_emit_and_apply(conn, log, ev)
|
||
conn.execute("DELETE FROM thread_suspect WHERE tid=?", (tid,))
|
||
stats["thread_deleted"] += 1
|
||
else:
|
||
conn.execute(
|
||
"INSERT INTO thread_suspect(tid, first_seen_at, last_seen_at, hits) VALUES(?,?,?,1) "
|
||
"ON CONFLICT(tid) DO UPDATE SET last_seen_at=excluded.last_seen_at, hits=hits+1",
|
||
(tid, events.now(), events.now()),
|
||
)
|
||
stats["thread_suspect"] += 1
|
||
return "deleted"
|
||
|
||
|
||
def _reconcile(conn: sqlite3.Connection, log: events.EventLog, tid: int,
|
||
observed: dict[int, dict], stats: dict,
|
||
observed_comments: dict[int, int] | None = None,
|
||
complete_pids: set[int] | None = None) -> None:
|
||
"""把「服务器这次看到的」和「库里 live 的」对齐。"""
|
||
rows = {}
|
||
for row in conn.execute(
|
||
"SELECT pid, floor, author, content, imgs, frags, agree, disagree, content_hash, status "
|
||
"FROM post WHERE tid=?", (tid,)
|
||
):
|
||
rows[int(row[0])] = {
|
||
"floor": row[1], "author": row[2], "content": row[3], "imgs": row[4],
|
||
"frags": row[5], "agree": row[6], "disagree": row[7],
|
||
"content_hash": row[8], "status": row[9],
|
||
}
|
||
live = {pid for pid, r in rows.items() if (r["status"] or "live") == "live"}
|
||
|
||
# ---- 新增 ----
|
||
for pid, rec in observed.items():
|
||
if pid in rows:
|
||
continue
|
||
row = _row_from_record(rec)
|
||
ev = log.emit("post_add", tid=tid, pid=pid, floor=row["floor"], payload=row,
|
||
source="verify")
|
||
_emit_and_apply(conn, log, ev)
|
||
stats["added"] += 1
|
||
|
||
# ---- 编辑 / 点赞采样 ----
|
||
now = events.now()
|
||
for pid, rec in observed.items():
|
||
row = rows.get(pid)
|
||
if not row:
|
||
continue
|
||
digest = _hash_record(rec)
|
||
if row["content_hash"] is None:
|
||
conn.execute("UPDATE post SET content_hash=? WHERE pid=?", (digest, pid))
|
||
elif row["content_hash"] != digest:
|
||
payload = {
|
||
"content": rec.get("text") or "",
|
||
"frags": json.dumps(rec.get("frags") or [], ensure_ascii=False),
|
||
"imgs": _imgs_json(rec),
|
||
"content_hash": digest,
|
||
# 旧值:as-of 倒放时要靠它撤销
|
||
"old_content": row["content"],
|
||
"old_frags": row["frags"],
|
||
"old_imgs": row["imgs"],
|
||
"old_content_hash": row["content_hash"],
|
||
}
|
||
ev = log.emit("post_edit", tid=tid, pid=pid, floor=rec.get("floor"),
|
||
payload=payload, source="verify")
|
||
_emit_and_apply(conn, log, ev)
|
||
stats["edited"] += 1
|
||
if (row["agree"] or 0) != (rec.get("agree") or 0) or \
|
||
(row["disagree"] or 0) != (rec.get("disagree") or 0):
|
||
conn.execute(
|
||
"INSERT OR IGNORE INTO post_stat_history(pid, ts, agree, disagree) VALUES(?,?,?,?)",
|
||
(pid, now, rec.get("agree") or 0, rec.get("disagree") or 0),
|
||
)
|
||
conn.execute("UPDATE post SET agree=?, disagree=? WHERE pid=?",
|
||
(rec.get("agree") or 0, rec.get("disagree") or 0, pid))
|
||
stats["stat_samples"] += 1
|
||
|
||
# ---- 删除(两次确认) ----
|
||
for pid in sorted(live - set(observed)):
|
||
row = conn.execute("SELECT hits FROM verify_suspect WHERE tid=? AND pid=?",
|
||
(tid, pid)).fetchone()
|
||
hits = (row[0] if row else 0) + 1
|
||
if hits >= 2:
|
||
ev = log.emit("post_del", tid=tid, pid=pid, floor=rows[pid]["floor"],
|
||
source="verify", evidence=f"missing_hits={hits}")
|
||
_emit_and_apply(conn, log, ev)
|
||
conn.execute("DELETE FROM verify_suspect WHERE tid=? AND pid=?", (tid, pid))
|
||
stats["deleted"] += 1
|
||
else:
|
||
conn.execute(
|
||
"INSERT INTO verify_suspect(tid, pid, first_seen_at, last_seen_at, hits) "
|
||
"VALUES(?,?,?,?,1) "
|
||
"ON CONFLICT(tid, pid) DO UPDATE SET last_seen_at=excluded.last_seen_at, hits=hits+1",
|
||
(tid, pid, now, now),
|
||
)
|
||
stats["suspect"] += 1
|
||
|
||
# 这次又看到的 pid,之前的怀疑作废
|
||
if observed:
|
||
conn.execute(
|
||
f"DELETE FROM verify_suspect WHERE tid=? AND pid IN ({','.join('?' for _ in observed)})",
|
||
(tid, *observed.keys()),
|
||
)
|
||
|
||
# ---- 快照与状态 ----
|
||
max_floor = max((int(rec.get("floor") or 0) for rec in observed.values()), default=0)
|
||
row = conn.execute("SELECT reply_num FROM thread WHERE tid=?", (tid,)).fetchone()
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO thread_seen(tid, reply_num, max_floor, seen_at) VALUES(?,?,?,?)",
|
||
(tid, row[0] if row else 0, max_floor, now),
|
||
)
|
||
conn.execute("DELETE FROM thread_suspect WHERE tid=?", (tid,))
|
||
conn.execute("UPDATE thread SET updated_at=? WHERE tid=?", (now, tid))
|
||
|
||
# ---- 楼中楼删除:只比对「完整重取」的楼层 ----
|
||
if observed_comments is not None and complete_pids:
|
||
marks = ",".join("?" for _ in complete_pids)
|
||
seen = set(observed_comments)
|
||
rows = conn.execute(
|
||
f"SELECT cid, pid FROM comment WHERE tid=? AND status='live' AND pid IN ({marks})",
|
||
(tid, *complete_pids),
|
||
).fetchall()
|
||
for cid, pid in rows:
|
||
if int(cid) in seen:
|
||
continue
|
||
row = conn.execute("SELECT hits FROM comment_suspect WHERE cid=?", (cid,)).fetchone()
|
||
hits = (row[0] if row else 0) + 1
|
||
if hits >= 2:
|
||
ev = log.emit("comment_del", tid=tid, pid=int(pid), cid=int(cid),
|
||
source="verify", evidence=f"missing_hits={hits}")
|
||
_emit_and_apply(conn, log, ev)
|
||
conn.execute("DELETE FROM comment_suspect WHERE cid=?", (cid,))
|
||
stats["comment_deleted"] = stats.get("comment_deleted", 0) + 1
|
||
else:
|
||
conn.execute(
|
||
"INSERT INTO comment_suspect(cid, first_seen_at, last_seen_at, hits) "
|
||
"VALUES(?,?,?,1) ON CONFLICT(cid) DO UPDATE SET "
|
||
"last_seen_at=excluded.last_seen_at, hits=hits+1",
|
||
(cid, now, now),
|
||
)
|
||
stats["comment_suspect"] = stats.get("comment_suspect", 0) + 1
|
||
for cid in seen:
|
||
conn.execute("DELETE FROM comment_suspect WHERE cid=?", (cid,))
|
||
|
||
|
||
async def verify_thread(client, cfg: Config, conn: sqlite3.Connection,
|
||
log: events.EventLog, tid: int, stats: dict) -> str:
|
||
observed: dict[int, dict] = {}
|
||
observed_comments: dict[int, int] = {}
|
||
complete_pids: set[int] = set()
|
||
pn = 1
|
||
while True:
|
||
page = None
|
||
posts = []
|
||
for attempt in range(max(1, cfg.tries)):
|
||
await crawler.pace()
|
||
page = await client.get_posts(
|
||
tid, pn=pn, rn=cfg.post_rn, with_comments=True, comment_rn=cfg.comment_rn
|
||
)
|
||
err = getattr(page, "err", None)
|
||
if getattr(err, "code", None) == 4 or (isinstance(err, tuple) and err and err[0] == 4):
|
||
return _on_thread_deleted(conn, log, tid, stats)
|
||
posts = getattr(page, "objs", None) or []
|
||
if posts:
|
||
break
|
||
more = getattr(getattr(page, "page", None), "has_more", None)
|
||
if getattr(page, "err", None) is None and more is False:
|
||
break # 正常翻到底
|
||
if attempt + 1 < max(1, cfg.tries):
|
||
await asyncio.sleep(cfg.backoff * (attempt + 1))
|
||
if not posts:
|
||
more = getattr(getattr(page, "page", None), "has_more", None)
|
||
if getattr(page, "err", None) is None and more is False:
|
||
break
|
||
raise RuntimeError(f"第 {pn} 页连续空结果(可能是 429),本轮跳过")
|
||
for post in posts:
|
||
rec = crawler.post_record(tid, post)
|
||
observed[int(rec["pid"])] = rec
|
||
inline = getattr(post, "comments", None) or []
|
||
if int(getattr(post, "reply_num", 0) or 0) <= len(inline):
|
||
complete_pids.add(int(post.pid))
|
||
for comment in inline:
|
||
observed_comments[int(comment.pid)] = int(post.pid)
|
||
pn += 1
|
||
if not getattr(getattr(page, "page", None), "has_more", False):
|
||
break
|
||
if pn > cfg.page_cap:
|
||
raise RuntimeError(f"超过页数上限 {cfg.page_cap}")
|
||
|
||
_reconcile(conn, log, tid, observed, stats, observed_comments, complete_pids)
|
||
return "ok"
|
||
|
||
|
||
async def _verify_loop(cfg, todo, conn, log, stats, bduss, stoken) -> None:
|
||
aiotieba = crawler.require_aiotieba()
|
||
async with aiotieba.Client(BDUSS=bduss, STOKEN=stoken) as client:
|
||
for i, tid in enumerate(todo, 1):
|
||
try:
|
||
await verify_thread(client, cfg, conn, log, tid, stats)
|
||
stats["ok"] += 1
|
||
except Exception as exc:
|
||
stats["errors"] += 1
|
||
print(f" [!] tid={tid} {type(exc).__name__}: {exc}", flush=True)
|
||
conn.commit()
|
||
if i % 20 == 0:
|
||
print(f"\r {i}/{len(todo)} 新增{stats['added']} 编辑{stats['edited']} "
|
||
f"删除{stats['deleted']} 怀疑{stats['suspect']} "
|
||
f"整帖删{stats['thread_deleted']} 失败{stats['errors']}", end="", flush=True)
|
||
|
||
|
||
def verify(cfg: Config, threads: int = 0, priority: str = "recent",
|
||
tids: list[int] | None = None, rate: float | None = None) -> dict:
|
||
bduss, stoken = crawler.read_auth(cfg)
|
||
if rate:
|
||
cfg.rate = float(rate)
|
||
cfg.validate()
|
||
crawler.set_rate(cfg.rate)
|
||
|
||
conn = db.connect(cfg)
|
||
db.init_db(conn)
|
||
db.ensure_compat(conn)
|
||
|
||
if tids:
|
||
todo = [int(t) for t in tids]
|
||
else:
|
||
order = {
|
||
"recent": "ORDER BY reply_num DESC",
|
||
"oldest": "ORDER BY tid ASC",
|
||
"random": "ORDER BY RANDOM()",
|
||
}.get(priority)
|
||
if order:
|
||
sql = f"SELECT tid FROM thread WHERE deleted_at IS NULL {order}"
|
||
elif priority == "unhashed":
|
||
# 只挑还没算过 content_hash 的帖:核对覆盖率能均匀推进,不重复干活
|
||
sql = ("SELECT tid FROM thread WHERE deleted_at IS NULL "
|
||
"AND tid IN (SELECT tid FROM post WHERE content_hash IS NULL) "
|
||
"ORDER BY reply_num ASC")
|
||
else:
|
||
sql = "SELECT tid FROM thread WHERE deleted_at IS NULL ORDER BY RANDOM()"
|
||
if threads:
|
||
sql += f" LIMIT {int(threads)}"
|
||
todo = [int(r[0]) for r in conn.execute(sql)]
|
||
|
||
print(f"[i] 待核对 {len(todo):,} 帖,优先级 {priority},限速 {cfg.rate}/s")
|
||
batch = events.start_batch(conn, "verify", note=f"threads={len(todo)} priority={priority}")
|
||
log = events.EventLog(conn, cfg, batch, "verify")
|
||
stats = {"ok": 0, "added": 0, "edited": 0, "deleted": 0, "suspect": 0,
|
||
"thread_deleted": 0, "thread_suspect": 0, "stat_samples": 0, "errors": 0}
|
||
|
||
asyncio.run(_verify_loop(cfg, todo, conn, log, stats, bduss, stoken))
|
||
|
||
log.close()
|
||
events.finish_batch(conn, batch)
|
||
conn.commit()
|
||
print()
|
||
return stats
|