Files
dsh 0824e0ef83 v1.2.0: 事件流对账、增量 update、ZIM 发布与表情渲染
从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
2026-11-18 11:41:25 +08:00

135 lines
4.6 KiB
Python

# -*- coding: utf-8 -*-
"""补用户资料:正文里出现过、但 user 表里没有的作者。
为什么会有缺口:早期版本的用户资料只存在内存里、进程被强杀就丢了(真实丢过一批)。
``get_user_info`` 支持按 user_id 查(不只昵称/portrait),所以可以逐个人补,
不用重抓整帖。
每次运行都会重新算「正文里的 author_id − user 表」,天然可断点续传;
失败的下一轮还会被挑出来。
"""
from __future__ import annotations
import asyncio
import sqlite3
from . import crawler, db, events
from .config import Config
def _int(value, default: int = 0) -> int:
"""protobuf 枚举(Gender 之类)转 int;异常值退化为默认值。"""
if isinstance(value, bool):
return int(value)
if isinstance(value, int):
return value
try:
return int(value) # IntEnum
except (TypeError, ValueError):
return int(getattr(value, "value", default) or default)
def _user_from_info(info) -> dict:
return {
"uid": _int(getattr(info, "user_id", 0)),
"uname": str(getattr(info, "user_name", "") or ""),
"nick": str(getattr(info, "nick_name_new", "") or getattr(info, "user_name", "") or ""),
"portrait": str(getattr(info, "portrait", "") or ""),
"level": _int(getattr(info, "level", 0)),
"glevel": _int(getattr(info, "glevel", 0)),
"gender": _int(getattr(info, "gender", 0)),
"ip": str(getattr(info, "ip", "") or ""),
"bawu": bool(getattr(info, "is_bawu", False)),
"vip": bool(getattr(info, "is_vip", False)),
"icons": [str(x) for x in (getattr(info, "icons", None) or [])],
}
def missing_uids(conn: sqlite3.Connection) -> list[tuple[int, str]]:
rows = conn.execute(
"""
SELECT CAST(author_id AS INTEGER) AS uid, MAX(author) AS uname
FROM (
SELECT author_id, author FROM post
UNION ALL
SELECT author_id, author FROM comment
)
WHERE author_id != '' AND author_id != '0'
AND CAST(author_id AS INTEGER) NOT IN (SELECT uid FROM user)
GROUP BY uid
ORDER BY uid
"""
).fetchall()
return [(int(r[0]), r[1] or "") for r in rows]
def _save_user(conn: sqlite3.Connection, u: dict) -> None:
conn.execute(
"INSERT OR REPLACE INTO user(uid, uname, nick, portrait, level, glevel, gender, "
"ip, bawu, vip, icons, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
(int(u["uid"]), u.get("uname") or "", u.get("nick") or "", u.get("portrait") or "",
u.get("level") or 0, u.get("glevel") or 0, u.get("gender") or 0, u.get("ip") or "",
1 if u.get("bawu") else 0, 1 if u.get("vip") else 0,
",".join(u.get("icons") or []), events.now()),
)
async def _run(cfg: Config, rows: list[tuple[int, str]], client, stats: dict) -> None:
conn = db.connect(cfg)
try:
for i, (uid, uname) in enumerate(rows, 1):
await crawler.pace()
try:
info = await client.get_user_info(uid)
except Exception as exc: # 429/用户注销等:跳过,下一轮再试
stats["failed"] += 1
if i % 50 == 0:
print(f"\r {i}/{len(rows)} 补 {stats['filled']} 失败 {stats['failed']}",
end="", flush=True)
continue
u = _user_from_info(info)
if not u.get("uid"):
stats["failed"] += 1
continue
_save_user(conn, u)
stats["filled"] += 1
if i % 25 == 0:
conn.commit()
print(f"\r {i}/{len(rows)} 补 {stats['filled']} 失败 {stats['failed']}",
end="", flush=True)
conn.commit()
finally:
conn.close()
def fill_missing(cfg: Config, limit: int = 0, rate: float | None = None,
client=None) -> dict:
if client is None:
aiotieba = crawler.require_aiotieba()
bduss, stoken = crawler.read_auth(cfg)
client = aiotieba.Client(BDUSS=bduss, STOKEN=stoken)
if rate:
cfg.rate = float(rate)
cfg.validate()
crawler.set_rate(cfg.rate)
conn = db.connect(cfg)
db.init_db(conn)
db.ensure_compat(conn)
rows = missing_uids(conn)
conn.close()
if limit:
rows = rows[:limit]
stats = {"total": len(rows), "filled": 0, "failed": 0}
async def _main():
if hasattr(client, "__aenter__"):
async with client as c:
await _run(cfg, rows, c, stats)
else:
await _run(cfg, rows, client, stats)
asyncio.run(_main())
return stats