Files
dsh 0824e0ef83 v1.2.0: 事件流对账、增量 update、ZIM 发布与表情渲染
从东方年代记吧一次性备份脚本整理成的贴吧存档工具:crawl/update/verify/as-of/restore/rebuild/images/build-zim,SQLite 事件流 + 回收站 + 中文全文索引 ZIM。
2026-11-18 11:41:25 +08:00

65 lines
2.3 KiB
Python

# -*- coding: utf-8 -*-
"""从 SQLite 导出主题清单到 ``threads.jsonl``。
为什么需要导出:抓取阶段完全不碰数据库(见 crawler.py 的理由),只认一份
JSONL 清单。枚举时写清单,导入后如果又补抓/修数据,库里的 thread 表才是
权威,这时用 export 把清单从库里重新生成,保证两边一致。
只读打开数据库;写文件用「临时文件 + 原子替换」,中途失败不会把原清单
截断成半截。
"""
from __future__ import annotations
import json
import os
from pathlib import Path
from . import db
from .config import Config
def export_threads(cfg: Config, out_path: str | Path | None = None, *, quiet: bool = False) -> int:
"""把 ``thread`` 表导出成 JSONL,返回导出的主题数。"""
conn = db.connect(cfg, readonly=True)
try:
rows = conn.execute(
"SELECT tid, fid, title, author, author_id, reply_num, is_good, is_top "
"FROM thread ORDER BY tid"
).fetchall()
finally:
conn.close()
if not rows:
raise RuntimeError(
f"{cfg.db_path} 里没有主题记录;先跑 enum + import,或检查 --data 指向对不对"
)
out = Path(out_path) if out_path else cfg.threads_path
out.parent.mkdir(parents=True, exist_ok=True)
tmp = out.with_suffix(out.suffix + ".tmp")
total_reply = 0
with open(tmp, "w", encoding="utf-8", newline="\n") as fh:
for row in rows:
reply_num = int(row["reply_num"] or 0)
total_reply += reply_num
fh.write(
json.dumps(
{
"tid": row["tid"],
"fid": row["fid"] or 0,
"title": row["title"] or "",
"author": row["author"] or "",
"author_id": row["author_id"] or "",
"reply_num": reply_num,
"is_good": row["is_good"] or 0,
"is_top": row["is_top"] or 0,
},
ensure_ascii=False,
)
+ "\n"
)
os.replace(tmp, out)
if not quiet:
print(f"[i] 主题回复总数 {total_reply:,}(含楼中楼)")
return len(rows)