From 53f4d00d2e4f1d738473011f2cde851b8e897f63 Mon Sep 17 00:00:00 2001 From: dsh Date: Sat, 21 Nov 2026 15:06:34 +0800 Subject: [PATCH] =?UTF-8?q?docs:=20=E5=AE=8C=E6=95=B4=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=A8=8B=EF=BC=88workflow.md=EF=BC=89=E4=B8=8E?= =?UTF-8?q?=E6=89=93=E5=8C=85=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 docs/workflow.md:新吧备份、增量、ZIM 构建、Kiwix 发布、常见坑;examples/ 收录 adnmb 与小说打包脚本。 --- README.md | 4 ++ docs/workflow.md | 156 ++++++++++++++++++++++++++++++++++++++++ examples/adnmb_zim.py | 163 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 323 insertions(+) create mode 100644 docs/workflow.md create mode 100644 examples/adnmb_zim.py diff --git a/README.md b/README.md index 0046b34..2baada6 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,10 @@ data/ └─ export/ ``` +## 完整流程 + +从零备份一个吧 → 入库补图 → 构建 ZIM → 发布到 Kiwix → 增量更新,见 [docs/workflow.md](docs/workflow.md)。 + ## 表情 渲染机制、已知缺失(`image_emoticon51–59`)和抢救记录见 [docs/emoji.md](docs/emoji.md)。 diff --git a/docs/workflow.md b/docs/workflow.md new file mode 100644 index 0000000..61c8988 --- /dev/null +++ b/docs/workflow.md @@ -0,0 +1,156 @@ +# 操作流程(存档用) + +从零备份一个贴吧、打包成 ZIM、发布到 Kiwix 的完整流程。按顺序照抄即可。 + +约定: + +- 本地工具目录:`C:\Users\peng0\DSH\tieba-archiver` +- 本地 Python:用归档自带的 venv(已装 aiotieba / requests / libzim), + 下文记作 `$py`: + `C:\Users\peng0\DSH\东方年代记吧_全量备份_2011-2026\.venv\Scripts\python.exe` +- 服务器:`peng@erin`,工具在 `/home/peng/tieba/tieba-archiver`, + venv 在 `/home/peng/tieba/venv`(已装 libzim、requests、aiotieba) +- Kiwix(podman 容器):ZIM 放 `/home/peng/containers/kiwix/`, + 服务 `systemctl --user restart kiwix`,端口 **8095** + +--- + +## 1. 新建一个吧的备份 + +```powershell +# 1) 建目录、放 cookie +mkdir C:\Users\peng0\DSH\某某吧_备份\data +copy C:\Users\peng0\DSH\东方年代记吧_全量备份_2011-2026\data\cookies.txt C:\Users\peng0\DSH\某某吧_备份\data\ + +# 2) 初始化(会自动解析 fid) +& $py tieba.py init --data C:\Users\peng0\DSH\某某吧_备份\data --kw 某某 + +# 3) 枚举主题列表(--pages 3 可以先试水) +& $py tieba.py enum --data C:\Users\peng0\DSH\某某吧_备份\data + +# 4) 抓楼层(后台跑,rate 1.3~1.5) +& $py tieba.py crawl --data C:\Users\peng0\DSH\某某吧_备份\data --rate 1.5 +``` + +**建议直接在 erin 上抓**(网络快、数据不用回传): + +```bash +mkdir -p /home/peng/某某吧/data +scp data/cookies.txt peng@erin:/home/peng/某某吧/data/ +cd /home/peng/tieba/tieba-archiver +/home/peng/tieba/venv/bin/python tieba.py init --data /home/peng/某某吧/data --kw 某某 +nohup /home/peng/tieba/venv/bin/python tieba.py enum --data /home/peng/某某吧/data > enum.log 2>&1 & +nohup /home/peng/tieba/venv/bin/python tieba.py crawl --data /home/peng/某某吧/data --rate 1.5 > crawl.log 2>&1 & +tail -f crawl.log # 每 50 帖一行,失败数为 0 就是正常 +``` + +## 2. 入库、补图、打标签、构建 + +```bash +cd /home/peng/tieba/tieba-archiver +PY=/home/peng/tieba/venv/bin/python +DATA=/home/peng/某某吧/data + +$PY tieba.py import --data $DATA # raw/*.jsonl → tieba.db +$PY tieba.py fetch-emoji --data $DATA # 下载经典表情到 data/emoji/ +$PY tieba.py images --data $DATA --workers 4 +$PY tieba.py import-images --data $DATA +$PY tieba.py label --data $DATA --label "某某吧 全量备份" --version v1.0 +$PY tieba.py stats --data $DATA --live # 注意 --live,否则看不到 WAL 里的新行 +$PY tieba.py build-zim --data $DATA --out /home/peng/某某吧/某某吧.zim +``` + +然后发布(见第 4 节)。 + +## 3. 日常增量 + +```bash +$PY tieba.py update --data $DATA # 抓新帖、新楼层、新楼中楼(事件流) +$PY tieba.py verify --data $DATA --threads 200 --priority unhashed --rate 1.0 +$PY tieba.py build-zim --data $DATA --out /home/peng/某某吧/某某吧.zim +``` + +- `verify` 会顺带补 `content_hash`、采样点赞; +- 删除/编辑要第二轮才确认(两次确认机制); +- 全量核对约 1 帖/秒,2011–2026 那个吧要 8–10 小时,建议夜里跑。 + +## 4. 发布到 Kiwix + +```bash +# 文件名就是 Kiwix 里的 URL 名:/content/<文件名>/index.html +scp 某某吧.zim peng@erin:/home/peng/containers/kiwix/某某吧.zim +ssh peng@erin "systemctl --user restart kiwix" +curl -s -o /dev/null -w '%{http_code}\n' \ + http://127.0.0.1:8095/content/某某吧/index.html # 期望 200 +``` + +- 书名 = **ZIM 文件名**(不是元数据里的 Title); +- 页内 CSS 文件名带内容哈希(`style-.css`),改样式重建不会吃缓存; +- 客户端仍显示旧内容时:强刷 / 重启 Kiwix 客户端,或干脆换文件名(新 URL); +- 重启 Kiwix 会重扫整个库(含 28 GB 维基),慢是正常的。 + +## 5. 已有文本/帖子打包成 ZIM(Gemini、A岛那种) + +流程:先把内容整理成 markdown/HTML → 用 `libzim` 写 ZIM(**必须在 WSL/Linux 里写**)。 + +参考脚本:`examples/adnmb_zim.py`(markdown 串 → ZIM)、`examples/novel_nyasama_zim.py`(抓取结果 → 小说 ZIM)。 + +```python +from libzim.writer import Creator, Item, StringProvider, Hint + +class HtmlItem(Item): + def __init__(self, path, title, content): + super().__init__(); self.path, self.title, self.content = path, title, content + def get_path(self): return self.path + def get_title(self): return self.title + def get_mimetype(self): return "text/css" if self.path.endswith(".css") else "text/html" + def get_contentprovider(self): return StringProvider(self.content) + def get_hints(self): return {Hint.FRONT_ARTICLE: True} + +with Creator("out.zim").config_indexing(True, "zho") as c: + c.set_mainpath("index.html") + c.add_item(HtmlItem("style.css", "样式", CSS)) + c.add_item(HtmlItem("index.html", "标题", HTML)) + for k, v in {"Name": "adnmb-32444047", "Title": "标题", "Language": "zho", + "Creator": "作者", "Publisher": "来源"}.items(): + c.add_metadata(k, v) +``` + +命令行(WSL): + +```powershell +wsl -e /opt/zimvenv/bin/python /mnt/c/Users/peng0/DSH/_adnmb_build.py +``` + +命名规则:**文件名带来源和串号/标识**,如 `adnmb-32444047.zim`、`gemini.zim`。 + +## 6. 已知的坑 + +| 坑 | 现象 | 处理 | +|---|---|---| +| Windows libzim >2 GiB | ZIM header 写成 `0xFFFF…`,打不开 | 单文件构建放 WSL/Linux;或 `--max-bytes 1.8G` 分卷 | +| WSL 写 `/mnt/c` | 构建卡在 `Adding checksum` | ZIM 写到 `/root/` 再 `cp` 到 C 盘 | +| 构建机缺 requests | `images` 直接退出 | `pip install requests` | +| `fetch-emoji` 个别 404 | 旧脚本 exit 1 中断流水线 | 已修(v1.2.0 后只警告) | +| `stats` 显示图片 0 | 开了 immutable 快照 | 加 `--live` | +| Kiwix 客户端缓存 | 改了样式/表情仍显示旧的 | CSS 版本化已解决样式;页面换文件名最稳 | +| 表情缺失 | `[孤独寂寞冷]` 文字标签 | `image_emoticon51–59` 全网无存,见 `docs/emoji.md` | + +## 7. 服务器现状 + +| 路径 | 内容 | +|---|---| +| `/home/peng/tieba/data` | 东方年代记吧数据(15 GB,含 imgs) | +| `/home/peng/tieba/tieba-archiver` | 工具代码 | +| `/home/peng/tieba/venv` | Python 环境(libzim / requests / aiotieba) | +| `/home/peng/yeshenxue` | 东方夜神雪吧数据 + 工具副本 + ZIM | +| `/home/peng/containers/kiwix` | Kiwix 库:`touhou-nendaiki.zim`、`yeshenxue.zim`、`gemini.zim`、`adnmb-32444047.zim` | + +## 8. 速查 + +```text +tieba.py 命令:init enum crawl update verify import images import-images + fetch-emoji label stats export build-zim as-of events restore rebuild +关键参数:--data 数据目录 --rate 限速 --threads N --priority unhashed + --out ZIM 输出 --limit 只打 N 帖 --no-images 纯文本 +``` diff --git a/examples/adnmb_zim.py b/examples/adnmb_zim.py new file mode 100644 index 0000000..206a3db --- /dev/null +++ b/examples/adnmb_zim.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +"""把 A岛串(提取的 markdown)打包成 ZIM,风格对齐《私立幻想学院 Gemini》那本。""" +import html +import pathlib +import re + +import sys + +SRC = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "source.md") +OUT = pathlib.Path(sys.argv[2] if len(sys.argv) > 2 else "adnmb.zim") + +CSS = """ +:root{--bg:#f4f1ea;--card:#fffdf8;--text:#2b2622;--muted:#8a8177;--link:#8a5a2b;--border:#e6ded0} +@media (prefers-color-scheme:dark){:root{--bg:#15130f;--card:#1e1b16;--text:#e8e2d6;--muted:#9a9184;--link:#d2a679;--border:#2e2820}} +*{box-sizing:border-box} +body{margin:0;background:var(--bg);color:var(--text); + font:16px/1.95 "Songti SC","Noto Serif CJK SC",Georgia,"Microsoft YaHei",serif} +a{color:var(--link);text-decoration:none}a:hover{text-decoration:underline} +.topbar{position:sticky;top:0;background:var(--card);border-bottom:1px solid var(--border)} +.topbar .wrap{max-width:760px;margin:0 auto;padding:10px 18px;display:flex;gap:16px;align-items:baseline} +.brand{font-weight:700;color:var(--text)} +.wrap{max-width:760px;margin:0 auto;padding:22px 18px 60px} +h1{font-size:23px;line-height:1.5;margin:0 0 6px} +h2{font-size:17px;margin:30px 0 10px;color:var(--muted);font-weight:600} +.meta{color:var(--muted);font-size:13px;margin-bottom:6px} +.floor{padding:14px 0;border-top:1px solid var(--border)} +.floor:first-of-type{border-top:0} +.fh{margin-bottom:6px;font-size:13px;color:var(--muted)} +.fh .num{display:inline-block;min-width:32px;padding:1px 7px;margin-right:6px; + background:var(--card);border:1px solid var(--border);border-radius:999px;text-align:center} +.fh .au{color:var(--text);font-weight:600;margin-right:8px} +p{margin:0 0 .9em;text-indent:2em} +.quote{display:inline-block;color:var(--muted);font-size:13px;background:var(--card); + border:1px solid var(--border);border-radius:6px;padding:0 8px;margin-bottom:6px} +""" + +FLOOR_RE = re.compile(r"^### (\d+)F\s*|\s*(.+?)\s*|\s*(?:「(.+?)」\s*|\s*)?(\d{4}/\d+/\d+\s+\d+:\d+)\s*|\s*No\.(\d+)") +CMT_RE = re.compile(r"^### (\S+)\s*|\s*(\d{4}/\d+/\d+\s+\d+:\d+)\s*|\s*No\.(\d+)") + + +def body_html(lines: list[str]) -> str: + text = "\n".join(lines).strip() + blocks = [b.strip() for b in re.split(r"\n\s*\n", text) if b.strip()] + out = [] + for block in blocks: + if block.startswith("`>>No.") and block.endswith("`"): + ref = html.escape(block.strip("`")) + out.append(f'
{ref}
') + continue + block = html.escape(block) + block = re.sub(r"`(>>No\.\d+)`", r'\1', block) + out.append(f"

{block.replace(chr(10), '
')}

") + return "".join(out) + + +def main() -> None: + lines = SRC.read_text(encoding="utf-8").splitlines() + title = "A岛 小说串 No.32444047" + floors, comments = [], [] + current = None + in_comments = False + for line in lines: + if line.startswith("## 评论区"): + in_comments = True + current = None + continue + m = FLOOR_RE.match(line) if not in_comments else None + if m: + current = {"kind": "floor", "floor": m.group(1), "author": m.group(2).strip(), + "tag": (m.group(3) or "").strip(), "date": m.group(4), "no": m.group(5), + "body": []} + floors.append(current) + continue + c = CMT_RE.match(line) if in_comments else None + if c: + current = {"kind": "cmt", "floor": "", "author": c.group(1), "tag": "", + "date": c.group(2), "no": c.group(3), "body": []} + comments.append(current) + continue + if current is not None and not line.startswith("#"): + current["body"].append(line) + + parts = [ + '
', + "

A岛 · 小说串 No.32444047

", + '
adnmb.com | 共 %d 楼 · %d 条评论 | 2020/11/26 – 2020/11/30
' + % (len(floors), len(comments)), + '
搬运:oUcVKhv | 原作者:Das6fY5(备胎岛串 No.1767815)
', + "
", + ] + for f in floors: + tag = f'「{html.escape(f["tag"])}」' if f["tag"] else "" + parts.append('
') + parts.append( + f'
{f["floor"]}F' + f'{html.escape(f["author"])}' + f'{tag} {html.escape(f["date"])} | No.{f["no"]}
' + ) + parts.append(body_html(f["body"])) + parts.append("
") + if comments: + parts.append("

评论区

") + for c in comments: + parts.append('
') + parts.append( + f'
{html.escape(c["author"])}' + f'{html.escape(c["date"])} | No.{c["no"]}
' + ) + parts.append(body_html(c["body"])) + parts.append("
") + + page = ( + '' + '' + "A岛 · 小说串 No.32444047" + '' + '
A岛 · No.32444047' + 'adnmb.com
' + '
' + "".join(parts) + "
" + ) + + from libzim.writer import Creator, Hint, Item, StringProvider + + class HtmlItem(Item): + def __init__(self, path, title, content): + super().__init__() + self.path, self.title, self.content = path, title, content + + def get_path(self): + return self.path + + def get_title(self): + return self.title + + def get_mimetype(self): + return "text/css" if self.path.endswith(".css") else "text/html" + + def get_contentprovider(self): + return StringProvider(self.content) + + def get_hints(self): + return {Hint.FRONT_ARTICLE: True} + + OUT.parent.mkdir(parents=True, exist_ok=True) + with Creator(str(OUT)).config_indexing(True, "zho") as c: + c.set_mainpath("index.html") + c.add_item(HtmlItem("style.css", "样式", CSS)) + c.add_item(HtmlItem("index.html", "A岛 小说串 No.32444047", page)) + for k, v in { + "Name": "adnmb-32444047", + "Title": "A岛 · 小说串 No.32444047", + "Language": "zho", + "Creator": "A岛 oUcVKhv(转载)", + "Publisher": "adnmb.com", + "Date": "2020-11-30", + "Description": "%d 楼 · %d 条评论,原文串 No.32444047" % (len(floors), len(comments)), + }.items(): + c.add_metadata(k, v) + print("写出", OUT, OUT.stat().st_size / 1024, "KB", "楼层", len(floors), "评论", len(comments)) + print("用法: python adnmb_zim.py <提取的.md> <输出.zim>") + + +main()