Files
dsh 0824e0ef83 v1.2.0: 事件流对账、增量 update、ZIM 发布与表情渲染
从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
2026-11-18 11:41:25 +08:00

233 lines
8.7 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""update:增量备份。
和 verify 的分工:
* update 只**加**:新主题、老主题尾部的新楼层;从不判断“某层没了”;
* verify 才做删除/编辑对账(要完整翻帖,成本高)。
流程:
1. 增量枚举主题列表,遇到「整页没有新 tid」就停(列表按时间倒序,新帖在前);
2. 新主题完整抓;
3. 已知主题里 `thread.reply_num > thread_seen.reply_num` 的,从「已知最大楼层所在页」回退一页翻到末尾;
4. 新楼层发 `post_add` 事件并物化,图片 URL 进 image 表。
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
from . import crawler, db, events
from .config import Config
from .verify import _imgs_json
def _ensure_thread(conn: sqlite3.Connection, rec: dict, status: str = "new") -> None:
conn.execute(
"""INSERT INTO thread(tid, fid, title, author, author_id, reply_num,
is_good, is_top, status, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)
ON CONFLICT(tid) DO UPDATE SET
title=excluded.title, author=excluded.author, author_id=excluded.author_id,
reply_num=excluded.reply_num, is_good=excluded.is_good,
is_top=excluded.is_top, updated_at=excluded.updated_at""",
(rec["tid"], rec.get("fid") or 0, rec.get("title") or "", rec.get("author") or "",
str(rec.get("author_id") or ""), rec.get("reply_num") or 0,
rec.get("is_good") or 0, rec.get("is_top") or 0, status, events.now()),
)
def _insert_post(conn, log, rec, stats) -> bool:
pid = int(rec["pid"])
if conn.execute("SELECT 1 FROM post WHERE pid=?", (pid,)).fetchone():
return False
from .verify import _row_from_record
row = _row_from_record(rec)
ev = log.emit("post_add", tid=int(rec["tid"]), pid=pid, floor=row["floor"],
payload=row, source="update")
events.apply_event(conn, ev)
for url in json.loads(_imgs_json(rec)):
conn.execute(
"INSERT OR IGNORE INTO image(url, tid, pid, status) VALUES(?,?,?,'new')",
(url, rec["tid"], pid),
)
stats["added"] += 1
return True
def _insert_comment(conn, log, rec, stats) -> bool:
"""新楼中楼也走事件流(comment_add),保持「事件是唯一真相」。"""
from .importer import parse_reply_name
cid = int(rec["pid"])
if conn.execute("SELECT 1 FROM comment WHERE cid=?", (cid,)).fetchone():
return False
row = {
"cid": cid,
"pid": int(rec.get("ppid") or 0),
"tid": int(rec["tid"]),
"author": rec.get("uname") or "",
"author_id": str(rec.get("uid") or ""),
"content": rec.get("text") or "",
"post_time": _ts_str(rec.get("ts")),
"fetched_at": events.now(),
"floor": rec.get("floor") or 0,
"agree": rec.get("agree") or 0,
"disagree": rec.get("disagree") or 0,
"rto": rec.get("rto") or 0,
"reply_to_name": parse_reply_name(rec.get("text") or ""),
"frags": json.dumps(rec.get("frags") or [], ensure_ascii=False),
"status": "live",
}
ev = log.emit("comment_add", tid=row["tid"], pid=row["pid"], cid=cid,
floor=row["floor"], payload=row, source="update")
events.apply_event(conn, ev)
stats["comments"] = stats.get("comments", 0) + 1
return True
def _ts_str(ts) -> str:
import time
try:
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(ts)))
except (TypeError, ValueError, OSError):
return ""
async def _crawl(client, cfg: Config, conn, log, tid: int, start_page: int,
stats: dict) -> None:
pn = max(1, start_page)
max_floor = 0
while True:
await crawler.pace()
page = await client.get_posts(
tid, pn=pn, rn=cfg.post_rn, with_comments=True, comment_rn=cfg.comment_rn
)
err = getattr(page, "err", None)
if getattr(err, "code", None) == 4:
stats["deleted"] += 1
return
posts = getattr(page, "objs", None) or []
if not posts:
if getattr(page, "err", None) is None and \
getattr(getattr(page, "page", None), "has_more", None) is False:
break
raise RuntimeError(f"第 {pn} 页空结果(可能 429)")
for post in posts:
rec = crawler.post_record(tid, post)
_insert_post(conn, log, rec, stats)
max_floor = max(max_floor, int(rec.get("floor") or 0))
for comment in (getattr(post, "comments", None) or []):
_insert_comment(conn, log,
crawler.comment_record(tid, comment, int(post.pid)), stats)
pn += 1
if not getattr(getattr(page, "page", None), "has_more", False):
break
if pn > cfg.page_cap:
raise RuntimeError(f"超过页数上限 {cfg.page_cap}")
row = conn.execute("SELECT reply_num FROM thread WHERE tid=?", (tid,)).fetchone()
conn.execute(
"INSERT OR REPLACE INTO thread_seen(tid, reply_num, max_floor, seen_at) VALUES(?,?,?,?)",
(tid, row[0] if row else 0, max_floor, events.now()),
)
conn.execute("UPDATE thread SET status='done', updated_at=? WHERE tid=?",
(events.now(), tid))
async def _run(cfg: Config, pages: int, threads: int, client, conn, log, stats) -> None:
known = {int(r[0]) for r in conn.execute("SELECT tid FROM thread")}
new_tids: list[int] = []
page_no = 0
pn = 1
while True:
await crawler.pace()
page = await client.get_threads(cfg.kw, pn=pn, rn=cfg.thread_rn)
objs = getattr(page, "objs", None) or []
if not objs:
break
page_no += 1
new_on_page = 0
for thread in objs:
rec = crawler.thread_record(thread)
tid = int(rec["tid"])
if not tid:
continue
if tid not in known:
known.add(tid)
new_tids.append(tid)
new_on_page += 1
_ensure_thread(conn, rec, status="new")
with open(cfg.threads_path, "a", encoding="utf-8") as fh:
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
else:
conn.execute(
"UPDATE thread SET reply_num=?, is_good=?, is_top=?, updated_at=? WHERE tid=?",
(rec.get("reply_num") or 0, rec.get("is_good") or 0,
rec.get("is_top") or 0, events.now(), tid),
)
conn.commit()
stats["pages"] += 1
if new_on_page == 0 and page_no >= max(1, pages):
break
if not getattr(getattr(page, "page", None), "has_more", False):
break
pn += 1
stats["new_threads"] = len(new_tids)
todo_new = new_tids[: threads or None]
for tid in todo_new:
await _crawl(client, cfg, conn, log, tid, 1, stats)
conn.commit()
changed = [
(int(r[0]), int(r[1] or 0))
for r in conn.execute(
"SELECT t.tid, s.max_floor FROM thread t JOIN thread_seen s ON s.tid=t.tid "
"WHERE t.reply_num > s.reply_num AND t.deleted_at IS NULL "
"AND t.tid NOT IN (SELECT tid FROM thread WHERE status='new')"
)
]
stats["changed_threads"] = len(changed)
for tid, max_floor in changed[: threads or None]:
start = max(1, max_floor // cfg.post_rn - 1)
await _crawl(client, cfg, conn, log, tid, start, stats)
conn.commit()
def update(cfg: Config, pages: int = 2, threads: int = 0, rate: float | None = None,
client=None) -> dict:
"""增量备份入口。传 ``client`` 时跳过认证(测试用)。"""
if client is None:
aiotieba = crawler.require_aiotieba()
bduss, stoken = crawler.read_auth(cfg)
client = aiotieba.Client(BDUSS=bduss, STOKEN=stoken)
if rate:
cfg.rate = float(rate)
cfg.validate()
crawler.set_rate(cfg.rate)
conn = db.connect(cfg)
db.init_db(conn)
db.ensure_compat(conn)
batch = events.start_batch(conn, "update", note=f"pages={pages} threads={threads}")
log = events.EventLog(conn, cfg, batch, "update")
stats = {"pages": 0, "new_threads": 0, "changed_threads": 0, "added": 0,
"deleted": 0, "errors": 0}
async def _main():
if hasattr(client, "__aenter__"):
async with client as c:
await _run(cfg, pages, threads, c, conn, log, stats)
else:
await _run(cfg, pages, threads, client, conn, log, stats)
try:
asyncio.run(_main())
finally:
log.close()
events.finish_batch(conn, batch)
conn.commit()
return stats