从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
135 lines
4.6 KiB
Python
135 lines
4.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""补用户资料:正文里出现过、但 user 表里没有的作者。
|
|
|
|
为什么会有缺口:早期版本的用户资料只存在内存里、进程被强杀就丢了(真实丢过一批)。
|
|
``get_user_info`` 支持按 user_id 查(不只昵称/portrait),所以可以逐个人补,
|
|
不用重抓整帖。
|
|
|
|
每次运行都会重新算「正文里的 author_id − user 表」,天然可断点续传;
|
|
失败的下一轮还会被挑出来。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sqlite3
|
|
|
|
from . import crawler, db, events
|
|
from .config import Config
|
|
|
|
|
|
def _int(value, default: int = 0) -> int:
|
|
"""protobuf 枚举(Gender 之类)转 int;异常值退化为默认值。"""
|
|
if isinstance(value, bool):
|
|
return int(value)
|
|
if isinstance(value, int):
|
|
return value
|
|
try:
|
|
return int(value) # IntEnum
|
|
except (TypeError, ValueError):
|
|
return int(getattr(value, "value", default) or default)
|
|
|
|
|
|
def _user_from_info(info) -> dict:
|
|
return {
|
|
"uid": _int(getattr(info, "user_id", 0)),
|
|
"uname": str(getattr(info, "user_name", "") or ""),
|
|
"nick": str(getattr(info, "nick_name_new", "") or getattr(info, "user_name", "") or ""),
|
|
"portrait": str(getattr(info, "portrait", "") or ""),
|
|
"level": _int(getattr(info, "level", 0)),
|
|
"glevel": _int(getattr(info, "glevel", 0)),
|
|
"gender": _int(getattr(info, "gender", 0)),
|
|
"ip": str(getattr(info, "ip", "") or ""),
|
|
"bawu": bool(getattr(info, "is_bawu", False)),
|
|
"vip": bool(getattr(info, "is_vip", False)),
|
|
"icons": [str(x) for x in (getattr(info, "icons", None) or [])],
|
|
}
|
|
|
|
|
|
def missing_uids(conn: sqlite3.Connection) -> list[tuple[int, str]]:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT CAST(author_id AS INTEGER) AS uid, MAX(author) AS uname
|
|
FROM (
|
|
SELECT author_id, author FROM post
|
|
UNION ALL
|
|
SELECT author_id, author FROM comment
|
|
)
|
|
WHERE author_id != '' AND author_id != '0'
|
|
AND CAST(author_id AS INTEGER) NOT IN (SELECT uid FROM user)
|
|
GROUP BY uid
|
|
ORDER BY uid
|
|
"""
|
|
).fetchall()
|
|
return [(int(r[0]), r[1] or "") for r in rows]
|
|
|
|
|
|
def _save_user(conn: sqlite3.Connection, u: dict) -> None:
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO user(uid, uname, nick, portrait, level, glevel, gender, "
|
|
"ip, bawu, vip, icons, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
(int(u["uid"]), u.get("uname") or "", u.get("nick") or "", u.get("portrait") or "",
|
|
u.get("level") or 0, u.get("glevel") or 0, u.get("gender") or 0, u.get("ip") or "",
|
|
1 if u.get("bawu") else 0, 1 if u.get("vip") else 0,
|
|
",".join(u.get("icons") or []), events.now()),
|
|
)
|
|
|
|
|
|
async def _run(cfg: Config, rows: list[tuple[int, str]], client, stats: dict) -> None:
|
|
conn = db.connect(cfg)
|
|
try:
|
|
for i, (uid, uname) in enumerate(rows, 1):
|
|
await crawler.pace()
|
|
try:
|
|
info = await client.get_user_info(uid)
|
|
except Exception as exc: # 429/用户注销等:跳过,下一轮再试
|
|
stats["failed"] += 1
|
|
if i % 50 == 0:
|
|
print(f"\r {i}/{len(rows)} 补 {stats['filled']} 失败 {stats['failed']}",
|
|
end="", flush=True)
|
|
continue
|
|
u = _user_from_info(info)
|
|
if not u.get("uid"):
|
|
stats["failed"] += 1
|
|
continue
|
|
_save_user(conn, u)
|
|
stats["filled"] += 1
|
|
if i % 25 == 0:
|
|
conn.commit()
|
|
print(f"\r {i}/{len(rows)} 补 {stats['filled']} 失败 {stats['failed']}",
|
|
end="", flush=True)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def fill_missing(cfg: Config, limit: int = 0, rate: float | None = None,
|
|
client=None) -> dict:
|
|
if client is None:
|
|
aiotieba = crawler.require_aiotieba()
|
|
bduss, stoken = crawler.read_auth(cfg)
|
|
client = aiotieba.Client(BDUSS=bduss, STOKEN=stoken)
|
|
if rate:
|
|
cfg.rate = float(rate)
|
|
cfg.validate()
|
|
crawler.set_rate(cfg.rate)
|
|
|
|
conn = db.connect(cfg)
|
|
db.init_db(conn)
|
|
db.ensure_compat(conn)
|
|
rows = missing_uids(conn)
|
|
conn.close()
|
|
if limit:
|
|
rows = rows[:limit]
|
|
|
|
stats = {"total": len(rows), "filled": 0, "failed": 0}
|
|
|
|
async def _main():
|
|
if hasattr(client, "__aenter__"):
|
|
async with client as c:
|
|
await _run(cfg, rows, c, stats)
|
|
else:
|
|
await _run(cfg, rows, client, stats)
|
|
|
|
asyncio.run(_main())
|
|
return stats
|