从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
233 lines
8.7 KiB
Python
233 lines
8.7 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""update:增量备份。
|
||
|
||
和 verify 的分工:
|
||
* update 只**加**:新主题、老主题尾部的新楼层;从不判断“某层没了”;
|
||
* verify 才做删除/编辑对账(要完整翻帖,成本高)。
|
||
|
||
流程:
|
||
1. 增量枚举主题列表,遇到「整页没有新 tid」就停(列表按时间倒序,新帖在前);
|
||
2. 新主题完整抓;
|
||
3. 已知主题里 `thread.reply_num > thread_seen.reply_num` 的,从「已知最大楼层所在页」回退一页翻到末尾;
|
||
4. 新楼层发 `post_add` 事件并物化,图片 URL 进 image 表。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import sqlite3
|
||
|
||
from . import crawler, db, events
|
||
from .config import Config
|
||
from .verify import _imgs_json
|
||
|
||
|
||
def _ensure_thread(conn: sqlite3.Connection, rec: dict, status: str = "new") -> None:
|
||
conn.execute(
|
||
"""INSERT INTO thread(tid, fid, title, author, author_id, reply_num,
|
||
is_good, is_top, status, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)
|
||
ON CONFLICT(tid) DO UPDATE SET
|
||
title=excluded.title, author=excluded.author, author_id=excluded.author_id,
|
||
reply_num=excluded.reply_num, is_good=excluded.is_good,
|
||
is_top=excluded.is_top, updated_at=excluded.updated_at""",
|
||
(rec["tid"], rec.get("fid") or 0, rec.get("title") or "", rec.get("author") or "",
|
||
str(rec.get("author_id") or ""), rec.get("reply_num") or 0,
|
||
rec.get("is_good") or 0, rec.get("is_top") or 0, status, events.now()),
|
||
)
|
||
|
||
|
||
def _insert_post(conn, log, rec, stats) -> bool:
|
||
pid = int(rec["pid"])
|
||
if conn.execute("SELECT 1 FROM post WHERE pid=?", (pid,)).fetchone():
|
||
return False
|
||
from .verify import _row_from_record
|
||
|
||
row = _row_from_record(rec)
|
||
ev = log.emit("post_add", tid=int(rec["tid"]), pid=pid, floor=row["floor"],
|
||
payload=row, source="update")
|
||
events.apply_event(conn, ev)
|
||
for url in json.loads(_imgs_json(rec)):
|
||
conn.execute(
|
||
"INSERT OR IGNORE INTO image(url, tid, pid, status) VALUES(?,?,?,'new')",
|
||
(url, rec["tid"], pid),
|
||
)
|
||
stats["added"] += 1
|
||
return True
|
||
|
||
|
||
def _insert_comment(conn, log, rec, stats) -> bool:
|
||
"""新楼中楼也走事件流(comment_add),保持「事件是唯一真相」。"""
|
||
from .importer import parse_reply_name
|
||
|
||
cid = int(rec["pid"])
|
||
if conn.execute("SELECT 1 FROM comment WHERE cid=?", (cid,)).fetchone():
|
||
return False
|
||
row = {
|
||
"cid": cid,
|
||
"pid": int(rec.get("ppid") or 0),
|
||
"tid": int(rec["tid"]),
|
||
"author": rec.get("uname") or "",
|
||
"author_id": str(rec.get("uid") or ""),
|
||
"content": rec.get("text") or "",
|
||
"post_time": _ts_str(rec.get("ts")),
|
||
"fetched_at": events.now(),
|
||
"floor": rec.get("floor") or 0,
|
||
"agree": rec.get("agree") or 0,
|
||
"disagree": rec.get("disagree") or 0,
|
||
"rto": rec.get("rto") or 0,
|
||
"reply_to_name": parse_reply_name(rec.get("text") or ""),
|
||
"frags": json.dumps(rec.get("frags") or [], ensure_ascii=False),
|
||
"status": "live",
|
||
}
|
||
ev = log.emit("comment_add", tid=row["tid"], pid=row["pid"], cid=cid,
|
||
floor=row["floor"], payload=row, source="update")
|
||
events.apply_event(conn, ev)
|
||
stats["comments"] = stats.get("comments", 0) + 1
|
||
return True
|
||
|
||
|
||
def _ts_str(ts) -> str:
|
||
import time
|
||
|
||
try:
|
||
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(ts)))
|
||
except (TypeError, ValueError, OSError):
|
||
return ""
|
||
|
||
|
||
async def _crawl(client, cfg: Config, conn, log, tid: int, start_page: int,
|
||
stats: dict) -> None:
|
||
pn = max(1, start_page)
|
||
max_floor = 0
|
||
while True:
|
||
await crawler.pace()
|
||
page = await client.get_posts(
|
||
tid, pn=pn, rn=cfg.post_rn, with_comments=True, comment_rn=cfg.comment_rn
|
||
)
|
||
err = getattr(page, "err", None)
|
||
if getattr(err, "code", None) == 4:
|
||
stats["deleted"] += 1
|
||
return
|
||
posts = getattr(page, "objs", None) or []
|
||
if not posts:
|
||
if getattr(page, "err", None) is None and \
|
||
getattr(getattr(page, "page", None), "has_more", None) is False:
|
||
break
|
||
raise RuntimeError(f"第 {pn} 页空结果(可能 429)")
|
||
for post in posts:
|
||
rec = crawler.post_record(tid, post)
|
||
_insert_post(conn, log, rec, stats)
|
||
max_floor = max(max_floor, int(rec.get("floor") or 0))
|
||
for comment in (getattr(post, "comments", None) or []):
|
||
_insert_comment(conn, log,
|
||
crawler.comment_record(tid, comment, int(post.pid)), stats)
|
||
pn += 1
|
||
if not getattr(getattr(page, "page", None), "has_more", False):
|
||
break
|
||
if pn > cfg.page_cap:
|
||
raise RuntimeError(f"超过页数上限 {cfg.page_cap}")
|
||
row = conn.execute("SELECT reply_num FROM thread WHERE tid=?", (tid,)).fetchone()
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO thread_seen(tid, reply_num, max_floor, seen_at) VALUES(?,?,?,?)",
|
||
(tid, row[0] if row else 0, max_floor, events.now()),
|
||
)
|
||
conn.execute("UPDATE thread SET status='done', updated_at=? WHERE tid=?",
|
||
(events.now(), tid))
|
||
|
||
|
||
async def _run(cfg: Config, pages: int, threads: int, client, conn, log, stats) -> None:
|
||
known = {int(r[0]) for r in conn.execute("SELECT tid FROM thread")}
|
||
new_tids: list[int] = []
|
||
page_no = 0
|
||
pn = 1
|
||
while True:
|
||
await crawler.pace()
|
||
page = await client.get_threads(cfg.kw, pn=pn, rn=cfg.thread_rn)
|
||
objs = getattr(page, "objs", None) or []
|
||
if not objs:
|
||
break
|
||
page_no += 1
|
||
new_on_page = 0
|
||
for thread in objs:
|
||
rec = crawler.thread_record(thread)
|
||
tid = int(rec["tid"])
|
||
if not tid:
|
||
continue
|
||
if tid not in known:
|
||
known.add(tid)
|
||
new_tids.append(tid)
|
||
new_on_page += 1
|
||
_ensure_thread(conn, rec, status="new")
|
||
with open(cfg.threads_path, "a", encoding="utf-8") as fh:
|
||
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
else:
|
||
conn.execute(
|
||
"UPDATE thread SET reply_num=?, is_good=?, is_top=?, updated_at=? WHERE tid=?",
|
||
(rec.get("reply_num") or 0, rec.get("is_good") or 0,
|
||
rec.get("is_top") or 0, events.now(), tid),
|
||
)
|
||
conn.commit()
|
||
stats["pages"] += 1
|
||
if new_on_page == 0 and page_no >= max(1, pages):
|
||
break
|
||
if not getattr(getattr(page, "page", None), "has_more", False):
|
||
break
|
||
pn += 1
|
||
|
||
stats["new_threads"] = len(new_tids)
|
||
todo_new = new_tids[: threads or None]
|
||
for tid in todo_new:
|
||
await _crawl(client, cfg, conn, log, tid, 1, stats)
|
||
conn.commit()
|
||
|
||
changed = [
|
||
(int(r[0]), int(r[1] or 0))
|
||
for r in conn.execute(
|
||
"SELECT t.tid, s.max_floor FROM thread t JOIN thread_seen s ON s.tid=t.tid "
|
||
"WHERE t.reply_num > s.reply_num AND t.deleted_at IS NULL "
|
||
"AND t.tid NOT IN (SELECT tid FROM thread WHERE status='new')"
|
||
)
|
||
]
|
||
stats["changed_threads"] = len(changed)
|
||
for tid, max_floor in changed[: threads or None]:
|
||
start = max(1, max_floor // cfg.post_rn - 1)
|
||
await _crawl(client, cfg, conn, log, tid, start, stats)
|
||
conn.commit()
|
||
|
||
|
||
def update(cfg: Config, pages: int = 2, threads: int = 0, rate: float | None = None,
|
||
client=None) -> dict:
|
||
"""增量备份入口。传 ``client`` 时跳过认证(测试用)。"""
|
||
if client is None:
|
||
aiotieba = crawler.require_aiotieba()
|
||
bduss, stoken = crawler.read_auth(cfg)
|
||
client = aiotieba.Client(BDUSS=bduss, STOKEN=stoken)
|
||
|
||
if rate:
|
||
cfg.rate = float(rate)
|
||
cfg.validate()
|
||
crawler.set_rate(cfg.rate)
|
||
|
||
conn = db.connect(cfg)
|
||
db.init_db(conn)
|
||
db.ensure_compat(conn)
|
||
batch = events.start_batch(conn, "update", note=f"pages={pages} threads={threads}")
|
||
log = events.EventLog(conn, cfg, batch, "update")
|
||
stats = {"pages": 0, "new_threads": 0, "changed_threads": 0, "added": 0,
|
||
"deleted": 0, "errors": 0}
|
||
|
||
async def _main():
|
||
if hasattr(client, "__aenter__"):
|
||
async with client as c:
|
||
await _run(cfg, pages, threads, c, conn, log, stats)
|
||
else:
|
||
await _run(cfg, pages, threads, client, conn, log, stats)
|
||
|
||
try:
|
||
asyncio.run(_main())
|
||
finally:
|
||
log.close()
|
||
events.finish_batch(conn, batch)
|
||
conn.commit()
|
||
return stats
|