从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
65 lines
2.3 KiB
Python
65 lines
2.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""从 SQLite 导出主题清单到 ``threads.jsonl``。
|
|
|
|
为什么需要导出:抓取阶段完全不碰数据库(见 crawler.py 的理由),只认一份
|
|
JSONL 清单。枚举时写清单,导入后如果又补抓/修数据,库里的 thread 表才是
|
|
权威,这时用 export 把清单从库里重新生成,保证两边一致。
|
|
|
|
只读打开数据库;写文件用「临时文件 + 原子替换」,中途失败不会把原清单
|
|
截断成半截。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from . import db
|
|
from .config import Config
|
|
|
|
|
|
def export_threads(cfg: Config, out_path: str | Path | None = None, *, quiet: bool = False) -> int:
|
|
"""把 ``thread`` 表导出成 JSONL,返回导出的主题数。"""
|
|
conn = db.connect(cfg, readonly=True)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT tid, fid, title, author, author_id, reply_num, is_good, is_top "
|
|
"FROM thread ORDER BY tid"
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
|
|
if not rows:
|
|
raise RuntimeError(
|
|
f"{cfg.db_path} 里没有主题记录;先跑 enum + import,或检查 --data 指向对不对"
|
|
)
|
|
|
|
out = Path(out_path) if out_path else cfg.threads_path
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = out.with_suffix(out.suffix + ".tmp")
|
|
total_reply = 0
|
|
with open(tmp, "w", encoding="utf-8", newline="\n") as fh:
|
|
for row in rows:
|
|
reply_num = int(row["reply_num"] or 0)
|
|
total_reply += reply_num
|
|
fh.write(
|
|
json.dumps(
|
|
{
|
|
"tid": row["tid"],
|
|
"fid": row["fid"] or 0,
|
|
"title": row["title"] or "",
|
|
"author": row["author"] or "",
|
|
"author_id": row["author_id"] or "",
|
|
"reply_num": reply_num,
|
|
"is_good": row["is_good"] or 0,
|
|
"is_top": row["is_top"] or 0,
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
+ "\n"
|
|
)
|
|
os.replace(tmp, out)
|
|
if not quiet:
|
|
print(f"[i] 主题回复总数 {total_reply:,}(含楼中楼)")
|
|
return len(rows)
|