commit 45249b59f9af73896ce3a5f8669c67a6eda5ba6f Author: StarCells Date: Sat Nov 14 13:48:24 2026 +0800 光谱工具集:采集、回放、寻峰、波长标定 - spectro_core.py: 共享核心(加载/标定/基线/寻峰/高斯拟合/双峰联合拟合/CSV-JSON) - peak_finder.py: 寻峰与峰信息导出,--pick 交互选峰(含 RAW 预览) - wavelength_calibration.py: 改用 core 寻峰,汞灯 577/579 双峰可分 - wavelength_player.py / hq_average.py: 复用 core 的标定与解码加载 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1e86fda --- /dev/null +++ b/.gitattributes @@ -0,0 +1,14 @@ +# 行尾统一:仓库内文本一律 LF(Windows 检出仍可用) +* text=auto +*.py text eol=lf +*.md text eol=lf +*.txt text eol=lf +*.csv text eol=lf +*.json text eol=lf +*.ps1 text eol=crlf + +# 二进制,不做行尾转换 +*.png binary +*.jpg binary +*.npy binary +*.bin binary diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ef11760 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# ---- 采集数据:体积大,不入库 ------------------------------------------- +recordings/ +*.bin +*.jsonl + +# ---- 运行产物 ----------------------------------------------------------- +peak_exports/ +*.npy +*.log + +# ---- Python ------------------------------------------------------------- +__pycache__/ +*.py[cod] +*.so +.venv/ +venv/ +env/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# ---- 编辑器 / 系统 ------------------------------------------------------ +.vscode/ +.idea/ +.DS_Store +Thumbs.db +desktop.ini + +# ---- 临时文件 ----------------------------------------------------------- +*.tmp +*.bak +*.orig +*.rej diff --git a/PEAK_FINDER.md b/PEAK_FINDER.md new file mode 100644 index 0000000..a076050 --- /dev/null +++ b/PEAK_FINDER.md @@ -0,0 +1,283 @@ +# 寻峰与峰信息导出(peak_finder.py) + +`peak_finder.py` 在 `hq_record.py` 录制的 RAW 数据里找谱线峰:用 `hq_spectrum.py` +的解码和降维方式得到 R/G/B 三条谱,扣除局部基线后寻峰、拟合、给出峰形和**像素坐标**, +最后把**选中的峰**导出为图片、CSV、JSON 和原始像素块。 + +**本程序不做标定。** 它测的是传感器列坐标(`sensor_x_px`);只有当你在命令行里显式给出 +`--calibration <某个 calibration.json>` 时,才会按该文件里的系数把列坐标换算成波长 +(`wavelength_nm = polyval(coefficients_descending, sensor_x_px)`)。不给就只输出像素, +波长列留空。这会避免把某一次实验的标定悄悄套到别的录制上——标定是否适用由你判断, +依据(系数、确认范围、参考录制)会原样写进 `peaks.json`。 + +支持的数据是 `hq_record.py` 录制的 RAW(`BGGR_PISP_COMP1`、2028×1520、uint8 容器)。 +逐帧元数据里的黑电平(`SensorBlackLevels`,本数据集为 4096)会被自动使用。 + +依赖:Python 3 + NumPy + Matplotlib,**不需要 SciPy**。脚本放在 `hq_spectrum.py` +同目录即可(本目录里那份 `hq_spectrum.py` 是 2026-11-14 版的副本,方便单独运行; +你自己的 `hq_spectrum.py` 更新后记得同步过来,或用 `--module-dir` 指过去)。 + +兼容性:与 2026-10-24 版和 2026-11-14 版 `hq_spectrum.py` 都测过。新版把 PC1B 解码器 +改成了按需处理 qmode 3 + 直接写入 uint16(实测三份录制逐像素完全一致),并重写了 +`make_preview`——后者现在要求**整数** Bayer 帧,本程序已改为把原始 uint16 帧(多帧 +平均时是四舍五入后的均值)交给它。 + +## 快速开始 + +在 `peak_finder.py` 所在目录运行。先设两个路径变量(换成你自己的),下面直接引用: + +```powershell +$rec = "C:\Users\peng0\Dev\Light\recordings\hq_20261101_114417" # 录制目录 +$cal = "C:\Users\peng0\Dev\Light\Programs\wavelength_calibration\calibration.json" + +# 1. 先只看找到了哪些峰(不导出图片,速度快) +python peak_finder.py --input $rec --select none + +# 2. 导出全部峰(每峰一张峰形图 + CSV + 原始像素块) +python peak_finder.py --input $rec --select all + +# 3. 只导出 540–560 nm 的峰和最强的两条(按波长选就必须给 --calibration) +python peak_finder.py --input "C:\Users\peng0\Dev\Light\recordings\hq_20261110_230441" ` + --calibration $cal --select nm:540-560,top:2 + +# 4. 按编号导出(编号以上一次输出的 P001… 为准) +python peak_finder.py --input $rec --select id:2,id:4-5 + +# 5. 在窗口里点选:左边全谱,右边跟着鼠标显示该峰的 RAW 像素块和放大谱形 +# 悬停预览、点击切换选/不选,a=全选,n=全不选,e=导出,q=取消 +python peak_finder.py --input $rec --pick + +# 6. 平均多帧降噪;只取 ROI 中的一段行,减小谱线倾斜造成的展宽 +python peak_finder.py --input "C:\Users\peng0\Dev\Light\recordings\hq_20261111_092116" ` + --frame 5 --average 8 --rows 600,760 + +# 7. 顺带对照汞灯谱线表(需要波长,所以也给标定) +python peak_finder.py --input $rec --calibration $cal --match line_tables\mercury_hg1.csv +``` + +只有第 3 条和第 7 条给了 `--calibration`;其余只输出像素坐标,要波长自己补上。 +给了 `--calibration` 时终端会打印: + +``` + id nm x_px ch SNR height FWHM px FWHM nm slit y flags +* P001 435.6458 302.489 RB 517 2983 18.08 5.151 805.571 extrapolated, Hg I -0.19 nm +* P002 545.8237 689.157 RGB 2925 13545 10.35 2.949 763.262 saturated, Hg I -0.25 nm + P003 579.5502 807.519 RGB 939 4346 14.74 4.2 771.807 Hg I +0.48 nm +``` + +(这三条相对 NIST 汞灯 435.8328 / 546.0735 / 579.0663 nm 的偏差为 −0.19 / −0.25 / +0.48 nm, +属于该标定本身约 0.7 nm RMS 的水平。) + +## 交互选峰窗口(`--pick`) + +``` +┌───────────────────────────────┬──────────────────────┐ +│ │ RAW 预览(整幅、与 │ +│ 全谱(选中/未选中/聚焦) │ 播放器同款)+ROI 框 │ +│ │ +峰值所在列 │ +│ ├──────────────────────┤ +│ │ 放大谱形 + 拟合曲线 │ +│ │ + 半高宽标记 │ +└───────────────────────────────┴──────────────────────┘ +``` + +鼠标移到某条谱线上,右边两块立刻切换成那个峰。右上就是 `hq_spectrum` 播放器里那张 +auto-stretched RAW 预览(缩放到 ROI 区域):橙色框是 ROI,青色竖线是当前峰的传感器列, +青圈是它在缝方向的行位置;右下是放大的净信号和拟合曲线。 + +点一下切换选中(红色为选中、灰色为未选中、线更粗的是当前聚焦)。`a` 全选、`n` 全不选、 +`e` 导出并关闭、`q` 取消。见 `pick_window.png`。 + +窗口用固定的三块布局(不依赖 constrained_layout),可以随便拉伸;如果窗口比屏幕还大, +程序会先按屏幕尺寸缩一档,避免开出来就有一半在屏幕外。 + +> 导出的单峰 PNG 里那小块仍然是**未去马赛克的 Bayer 像素块**(`inferno` 色阶), +> 用来看像素级的线形和倾斜;想看"谱线在相机画面里的位置",看这个选峰窗口或 +> `peaks_overview.png`。 + +## 参数 + +### 输入 + +| 参数 | 默认 | 说明 | +| --- | --- | --- | +| `--input` | 必填 | 录制目录(含 `meta.json`、`frames.bin`) | +| `--frame` | `0` | 帧号(0 起) | +| `--average` | `1` | 从 `--frame` 起平均多少帧;>1 时导出的像素块是均值(float32) | +| `--roi` | `auto` | `auto` 自动检测光谱带;`x0,y0,x1,y1` 手动;`calibration` 用标定 JSON 里保存的 ROI | +| `--rows` | 无 | 在 ROI 内再限制行范围 `y0,y1`(全传感器行号),可减小谱线倾斜展宽 | +| `--reduce` | `mean` | 沿缝方向的行合并方式 `mean`/`median` | +| `--black` / `--full-scale` | 自动 | 覆盖黑电平/满量程(解码后 ADU) | +| `--calibration` | 无 | 要换算波长时显式给出的标定 JSON;不给则只输出像素坐标(`wavelength_nm` 留空) | +| `--wavelength-offset` | `0` | 给所给标定加一个固定波长偏移(nm),仅用于已确认的系统偏移;本身就是标定假设,慎用 | +| `--module-dir` | 脚本目录 | `hq_spectrum.py` 所在目录 | +| `--quiet` | 关 | 不打印过程信息 | + +### 寻峰 + +| 参数 | 默认 | 说明 | +| --- | --- | --- | +| `--baseline-window` | `100` | 形态学开运算基线的窗口(传感器像素);`0` 表示不减基线 | +| `--peak-distance` | `4` | 同通道两峰最小间距(传感器像素,2 px = 一个采样点),再近就只留高的;调大用于合并"勉强分开"的混合峰 | +| `--peak-height` | `0.02` | 最小峰高,占该通道最大净信号的比值 | +| `--peak-prominence` | `0.01` | 最小地形突出度,占该通道最大净信号的比值 | +| `--min-snr` | `15` | 最小峰高,单位是该通道噪声估计 | +| `--max-peaks` | `40` | 每通道最多峰数 | +| `--group-tol` | `4` | 跨 R/G/B 合并成同一条谱线的距离(像素) | +| `--fwhm-span` | `60` | 半高宽搜索的最大半程(像素);超出记为无半高宽 | +| `--fit-window` | `0` | 拟合窗半宽(像素);0 自动按半高宽定 | +| `--shape` | `gauss` | 线型模型 `gauss`/`lorentz` | + +### 选择与导出 + +| 参数 | 默认 | 说明 | +| --- | --- | --- | +| `--select` | `all` | `all`、`none`、`3`(编号)、`3-7`(编号段)、`nm:540-560`(波长段)、`546.1`(波长±典型线距的一半)、`top:5`(SNR 前 5),可用逗号组合取并集 | +| `--pick` | 关 | 开窗口点选:左边全谱,右边实时显示鼠标所指峰的 RAW 像素块与放大谱形;悬停预览、点击切换选中 | +| `--output` | `peak_exports/<录制名>_f<帧号>` | 输出目录;单帧是 `_f0003`,`--average 8` 时是 `_f0003-0010_avg8` | +| `--patch-half-width` | `24` | 导出原始像素块的横向半宽(像素) | +| `--patch-format` | `npy` | 像素块格式 `npy`/`csv`/`both`/`none` | +| `--patch-csv` | 关 | 在 npy 之外再写一份长表 CSV | +| `--no-images` | 关 | 不生成每峰的 PNG | +| `--match` | 无 | 谱线表(CSV/JSON),用于给出谱线身份建议 | +| `--match-tolerance` | `1.5` | 匹配容差(nm) | + +## 导出内容 + +``` +peak_exports/hq_20261101_114417_f0000/ +├─ peaks_overview.png 全谱:上图原始信号+基线,下图归一化净信号+峰标注 +├─ peaks_all.csv 所有通道的全部候选峰(每行一个通道峰) +├─ selected_peaks.csv 选中的峰,一行一条物理谱线(只包含像素坐标等汇总) +├─ peaks.json 清单:录制、帧、ROI、黑电平、噪声、标定、阈值、每峰的逐通道细节 +└─ peaks/ + ├─ peak_P002.png 峰形图:全谱定位 + RAW 像素块 + 放大曲线与拟合 + ├─ peak_P002_shape.csv 峰形数据:逐通道 raw / 基线 / net / 拟合值 / 残差 + └─ peak_P002_raw_y572-978_x664-714.npy 该峰的 RAW 像素块(uint16;文件名即像素框) +``` + +`peak_Pxxx.png` 三块内容:上方是该峰在全谱中的位置;左下是把该峰附近的 RAW Bayer +像素块(未去马赛克,横轴/纵轴都是真实传感器行列号);右下是放大后的净信号、拟合曲线 +(虚线)、半高宽标记,图例里的 `x±…` 是拟合中心的误差。标题给出波长、传感器列、通道、 +SNR、FWHM;饱和峰带 `[SATURATED]`,标定范围外带 `[extrapolated]`,重叠双峰经联合拟合 +分开的带 `[pair fit]`。 + +### selected_peaks.csv 主要字段 + +| 字段 | 含义 | +| --- | --- | +| `id` | 峰编号 `P001`…(按传感器列排序,同一数据同参数可复现) | +| `wavelength_nm` | 波长(空气,nm);无标定时为空 | +| `extrapolated` | 1 = 在标定确认范围之外,属于外推 | +| `channels` / `primary_channel` | 该线出现在哪些 CFA 通道 / 用哪个通道出指标 | +| `sensor_x_px` | 峰中心的全传感器列坐标(亚像素,加权合并各通道拟合中心) | +| `center_spread_px` | 各通道中心的最大差,判断跨通道一致性 | +| `sample_index` / `sample_step_px` | 该峰在降维谱数组里的下标 / 采样步长(2 px) | +| `dispersion_nm_per_px` | 该处色散 | +| `height_net_adu`、`snr`、`fwhm_px`、`fwhm_nm`、`area_net_adu_px` | 峰高(扣基线)、信噪比、半高宽(像素与 nm)、净面积 | +| `baseline_adu`、`raw_adu`、`saturated` | 基线值、峰顶原始值、是否饱和 | +| `slit_y_px` | 该谱线在缝方向(行方向)的强度加权中心行 | +| `roi_x0..roi_y1` | 使用的 ROI(全传感器坐标,半开区间) | +| `patch_x0..patch_y1` | 导出的 RAW 像素块框(全传感器坐标,半开区间) | +| `frame`、`frames_averaged` | 帧号、平均帧数 | +| `image_png`、`shape_csv`、`raw_npy`、`raw_csv` | 对应文件(相对输出目录) | +| `match_species`、`match_nm`、`match_delta_nm` | 谱线表匹配结果(未启用时为空) | + +`peaks_all.csv` 另外给出逐通道的 `center_parabola_px`、`center_fit_px`、 +`center_err_px`(拟合中心的形式误差)、`fwhm_halfmax_px`(半高宽直接测量值,可与拟合值 +对照)、`fit_ok`、`fit_rms_adu`、`deblended`,便于判断某个峰的拟合是否可信。实际采用的 +中心由 `center_source` 标明:`fit` = 高斯/洛伦兹拟合中心,`apex` = 拟合被拒后用峰顶三点 +抛物线,`sample` = 平顶/饱和导致抛物线也不可用时的采样点。重叠双峰的两个分量共用同一个 +`fwhm_px`(见下),因此两者线宽相同是正常的。 + +## 坐标约定 + +* 全部为 **0 起** 的全分辨率传感器坐标;像素 `x` 覆盖 `[x, x+1)`,第 0 行在画面顶部。 +* 降维谱每 2 px 一个采样点(CFA),拟合中心可以落在采样点之间,因此 `sensor_x_px` 是小数。 +* `roi`、`patch` 都是半开区间 `[x0,x1) × [y0,y1)`,与 `hq_spectrum.py` 的 ROI 定义一致。 +* `peak_*_raw_y572-978_x664-714.npy` 的第 0 行 = 传感器第 572 行,第 0 列 = 第 664 列 + (文件名里就是行列范围),可以直接 `np.load` 后与 `patch_x0…patch_y1` 对上。 + +## 方法说明 + +1. 用 `hq_spectrum.py` 的 `RecordingSource` 解码一帧(或若干帧均值):PC1B 解压成 + uint16 Bayer 图。 +2. `extract_planes` 按 BGGR 拆成 B、Gb+Gr(平均)、R 三个平面,沿缝方向(行)取 + mean/median,得到一个像素一个采样点的三条谱;x 用 `channel_x_positions` 换算成 + 全传感器列号(G 在 `x0+0.5+2k`,R/B 在 `x0+dx+2k`)。 +3. 基线:对每条谱做形态学开运算(先最小值滤波再最大值滤波,窗口默认 ±50 采样点), + 再去掉开运算结果的高频毛刺。它只用于**测量峰高/峰宽**,原始值单独导出。 +4. 噪声:中位绝对一阶差分 / (0.6745·√2),对孤立尖峰不敏感。 +5. 候选峰:局部极大值(平台取中点),要求峰高 ≥ max(高度比×通道最大, SNR 阈×噪声), + 且**地形突出度**(topographic prominence,向上走到比它更高的点为止,取两侧谷底较高者) + 达标,并与更强的峰间距 ≥ `--peak-distance`。用真实突出度而不是固定窗口,肩峰会被 + 自动排除,而真正分得开的双峰不会被并掉。 +6. 精化:加线性背景的高斯/洛伦兹拟合(自实现的 Levenberg–Marquardt,带边界投影), + 同时给出参数的形式误差。拟合不合格(不收敛、宽度顶到边界、残差 RMS > 20% 峰高)时 + **退回峰顶**(三点抛物线,`center_source=apex`),平顶/饱和导致抛物线也不可用时再用 + 检测到的采样点。**全程不使用质心**:质心是窗口的函数而不是峰的函数,同一个峰换个窗口 + 就换个值(本数据集里实测偏过 6 px 和 102 px),不能当峰位置用。 + +7. **双峰去混叠**:若两条相邻谱线的间距小于约 1.2 倍平均半高宽,就用「两个峰 + 一个 + 公共线宽 + 线性背景」做联合拟合。单峰拟合在重叠区会把两个中心都往中间拉(实测汞 + 577/579 会合成一条 579.5 nm),联合拟合才能把它们分开;标 `deblended=1`、`[pair fit]`。 +8. 跨通道合并:按**检测到的极大值位置**聚类(容差 `--group-tol`),同一通道只留 SNR + 最高者;同一条线在不同通道被检测到位置差几像素时,按拟合中心再合并一次 + (容差 1.5 px);峰中心按净峰高加权平均。 +9. 波长(**仅当你给了 `--calibration`**):`wavelength_nm = np.polyval(coefficients_descending, + sensor_x_px)`,与 `wavelength_player.py` 用的是同一个公式;超出该标定确认范围的峰标 + `extrapolated`。不给标定时这一列与 nm 相关的量都留空,图上的 x 轴就是传感器列。 + +## 已知限制 + +* **标定只对同一光学结构和传感器模式有效。** 标定来自 `hq_20261101_114417`,其确认范围 + 是 436–580 nm;其它录制的同一根谱线位置可能相差几个像素(几百分之一 nm 到 1 nm)。 + 需要更准就重新跑 `wavelength_calibration.py`。`--wavelength-offset` 只能在已确认是 + 固定系统偏移时使用,不能代替重新标定。 +* **谱线沿行方向不是一条直线(弯曲 + 漂移)。** 实测汞 546.1 nm(`hq_20261101_114417`): + 照亮带(约 363 行)上谱线中心是一条 S 形曲线,峰峰值 9.6 px(≈2.7 nm),按强度加权的 + 1σ 是 1.76 px,而**净线性漂移只有 2.6 px(0.41°)**。因此沿全部 970 行平均得到 + FWHM 10.35 px(2.95 nm),只用最亮的 70 行(`--rows 690,760`)得到 7.37 px(2.10 nm), + 中心相差 1.76 px(0.50 nm)。要测真实线宽就用 `--rows` 取窄行窗口。 + 导出的 RAW 像素块只有 ~50 px 宽却横跨几百行,`imshow` 的 `aspect="auto"` 会把横向 + 放大约 10 倍,看着比实际"斜"得多——上面的数字才是真实几何。 +* **饱和峰**:核心像素达到 65535 时标 `[SATURATED]`,此时峰高、线宽和中心都可能偏。 + 降低曝光/增益或缩短曝光时间重新录制更可靠。 +* **重叠双峰要小心。** 汞 576.96/579.07 nm 这类双线在默认 ROI 下间距只有约 6 px, + 而单条线 FWHM 约 7.5 px。程序会用公共线宽做联合拟合把它们分成两条(本例 + P004≈578.0 nm / P005≈580.4 nm),但分量位置的不确定度比孤立线大一个量级: + `center_err_px` 约 0.2–0.5 px,而孤立线只要 0.02–0.3 px。间距小于约 1 倍半高宽时 + 结果对线型假设很敏感,把它当"参考位置"而不是精密测量值。 +* **弱线/连续谱**:`--peak-height`、`--peak-prominence`、`--min-snr` 是相对的,暗弱录制 + 里需要适当调低;连成一片的宽峰会只报一个中心,或被 `--peak-distance` 合并。想把 + 靠得很近的线分开就把 `--peak-distance` 调小(最小 2 px = 一个采样点)。 +* 基线是形态学开运算的结果,宽于基线窗口的发射带会被部分当作基线;需要时可加大 + `--baseline-window` 或设为 0。 +* 波长单位是空气波长,未做光谱响应(强度)校正,强度仍是相机 ADU。 + +## 与其它脚本的关系 + +| 脚本 | 作用 | +| --- | --- | +| `hq_spectrum.py` | 解码/实时显示/回放的核心,本脚本复用它的解码、ROI、CFA 分离 | +| `hq_record.py` | 录制 RAW | +| `wavelength_calibration.py` | 用汞灯重新做像素→波长标定,产出 `wavelength_calibration/`;peak_finder 只读它的 JSON,从不改它 | +| `wavelength_player.py` | 带标定的回放 | +| `hq_average.py` | 多帧平均成新录制(本脚本用 `--average` 做同样的事,只是不落盘) | +| `peak_finder.py` | 本脚本:寻峰 + 峰形/像素坐标导出 | + +本脚本**只读取**标定,不会修改 `wavelength_calibration/` 里的任何文件。 + +## 测试 + +```powershell +# 录制不在程序目录附近时,用环境变量指过去,否则端到端用例会自动跳过 +$env:PEAK_FINDER_TEST_RECORDING = "C:\Users\peng0\Dev\Light\recordings\hq_20261101_114417" +$env:PEAK_FINDER_TEST_CALIBRATION = "C:\Users\peng0\Dev\Light\Programs\wavelength_calibration\calibration.json" +python -m unittest test_peak_finder -v +``` + +覆盖基线/噪声估计、亚像素中心、高斯/洛伦兹拟合、候选筛选、跨通道合并、选择语法、 +标定校验、谱线表匹配,以及一次真实的端到端导出(录制不在时会自动跳过)。 + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..fb22ed2 --- /dev/null +++ b/README.md @@ -0,0 +1,86 @@ +# spectrometer + +Raspberry Pi HQ Camera(IMX477)RAW 光谱工具集:录制、回放、寻峰、波长标定。 + +采集的是相机 **RAW** 数据(`BGGR_PISP_COMP1`,2028×1520,16 bit 容器),不做 ISP、 +不去马赛克、不做平滑;所有分析都基于按 CFA 分离后的 R/G/B 三条一维谱。 + +## 文件 + +| 文件 | 作用 | +| --- | --- | +| `hq_spectrum.py` | 核心库 + 实时播放器:PC1B 解码、CFA 分离、ROI、降维、预览、交互窗口 | +| `hq_record.py` | 把相机 RAW 流录成 `recordings/<时间戳>/{frames.bin,frames.jsonl,meta.json}` | +| `hq_average.py` | 多帧平均成一条新录制(降噪) | +| `spectro_core.py` | **共享核心**:模块加载、标定解析、基线/噪声/寻峰/拟合/双峰分解、CSV/JSON 输出 | +| `peak_finder.py` | 寻峰与峰信息导出:峰形图、峰形数据、像素坐标、RAW 像素块;`--pick` 交互选峰 | +| `wavelength_calibration.py` | 用汞灯谱线标定 像素→波长,产出 `wavelength_calibration/` | +| `wavelength_player.py` | 带已存标定的回放(波长轴、瀑布图) | +| `PEAK_FINDER.md` / `WAVELENGTH_PLAYER.md` | 两个工具的详细说明 | +| `test_peak_finder.py` / `test_wavelength_player.py` | 单元测试(含真实录制的端到端用例) | +| `wavelength_calibration/` | 一次汞灯标定的结果:`calibration.json` 及残差、候选、逐像素波长表 | + +`anal.py` 是 2026-10-23 的 `hq_spectrum.py` 旧副本,函数一一对应、无独有内容,属于 +遗留文件,可删。 + +## 依赖 + +``` +pip install -r requirements.txt +``` + +- Python ≥ 3.10,**NumPy + Matplotlib**,不需要 SciPy。 +- 树莓派上采集还需要 `picamera2`(系统包)和 `python3-tk`(交互窗口)。 +- 在 PC 上分析已有录制只需要 NumPy + Matplotlib。 + +## 快速开始 + +录制(树莓派): + +```bash +python3 hq_record.py --output recordings --exposure 66351 --gain 16 --duration 10 +``` + +回放 / 看谱(PC 或树莓派): + +```bash +python3 hq_spectrum.py --input recordings/hq_20261101_114417 --fps 15 +python3 wavelength_player.py --input recordings/hq_20261101_114417 \ + --calibration wavelength_calibration/calibration.json +``` + +寻峰并导出(`--calibration` 只是**应用**已有的标定,不给就只输出像素坐标): + +```bash +python peak_finder.py --input recordings/hq_20261101_114417 \ + --calibration wavelength_calibration/calibration.json --select all +python peak_finder.py --input recordings/hq_20261101_114417 --pick +``` + +重新标定: + +```bash +python wavelength_calibration.py --input recordings/hq_20261101_114417 --headless +# 看清 candidates.png 后指定:候选编号:波长 +python wavelength_calibration.py --input recordings/hq_20261101_114417 --headless \ + --assign "B1:404.6563,B2:435.8328,G1:546.0735,G2:576.9598,G3:579.0663" +``` + +## 测试 + +```bash +python -m unittest test_peak_finder test_wavelength_player -v +``` + +端到端用例需要录制与标定;录制不在脚本附近时用环境变量指过去,否则自动跳过: + +```powershell +$env:PEAK_FINDER_TEST_RECORDING = "C:\Users\peng0\Dev\Light\recordings\hq_20261101_114417" +$env:PEAK_FINDER_TEST_CALIBRATION = "wavelength_calibration\calibration.json" +``` + +## 仓库里不放什么 + +`recordings/`(原始录制,单个约 470 MB)、`peak_exports/`(导出结果)、`__pycache__/` +都在 `.gitignore` 里。标定结果 `wavelength_calibration/` 是**放**进来的,因为它决定了 +所有波长读数。 diff --git a/WAVELENGTH_PLAYER.md b/WAVELENGTH_PLAYER.md new file mode 100644 index 0000000..7d7cceb --- /dev/null +++ b/WAVELENGTH_PLAYER.md @@ -0,0 +1,29 @@ +# 使用标定结果播放录制 + +将 `wavelength_player.py` 放在 `hq_spectrum.py` 同目录,使用原来的 Python/NumPy/Matplotlib 环境。 +原播放器不需要修改。默认标定路径是新脚本旁 `wavelength_calibration/calibration.json`。 + +在 Programs 目录运行,把下面的 `other_recording` 换成实际录制目录: + +```powershell +python wavelength_player.py --input recordings/other_recording --calibration wavelength_calibration/calibration.json +``` + +默认暂停,沿用原播放器的播放/暂停、进度条、逐帧、倍速、循环、ROI 框选、`s` 保存和 `q` 退出。加 `--autoplay` 立即播放。默认在目标录制中自动检测 ROI;也可使用 `--roi calibration` 复用标定 ROI,或 `--roi 400,600,1400,900` 指定新 ROI。 + +```powershell +# 用已有汞灯录制检查标定回放 +python wavelength_player.py --input recordings/hq_20261024_135709 --roi calibration +# 无窗口检查前两帧并导出最后一帧截图及光谱 CSV +python wavelength_player.py --input recordings/other_recording --headless --frames 2 --save-prefix replay_check +# 全部参数 +python wavelength_player.py --help +``` + +直接使用 JSON 中的降幂系数,不重新拟合。R/G/B 各自完整传感器列坐标转换为 nm,ROI 改动不会把像素坐标重新从零开始。线性、二次及波长递减方向均支持;瀑布图显示时重采样到等距波长网格,导出的 R/G/B 光谱仍保留各自原始采样坐标。绿色背景表示确认点覆盖的波长范围,范围外为外推。 + +默认不平滑、不降噪、不扣背景,保留包括负值的黑电平校正信号。各帧采用录制自己的黑电平元数据,不沿用标定灯录制的 4096;可用 `--black` 手动覆盖(单位为解码后的 ADU)。归一化副本仅用于寻峰和辅助显示。导出 CSV 沿用原播放器字段,其中 `signal_adu` 和 `signal_smoothed` 在本程序中相同,均未经平滑。强度单位仍为相机 ADU,未做光谱响应校正。 + +标定要求光学结构、传感器模式及坐标几何保持一致。若 JSON 指向的原标定录制仍存在,会比较图像宽高,不一致即拒绝;若原录制已移走,会提示无法核对尺寸。相同宽高也不能保证光学位置未改变。 + +测试:`python -m unittest test_wavelength_player -v`。 diff --git a/anal.py b/anal.py new file mode 100644 index 0000000..5acf74d --- /dev/null +++ b/anal.py @@ -0,0 +1,1507 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +hq_spectrum.py +============== +Raspberry Pi HQ Camera (IMX477) 12-bit RAW, live R/G/B spectral monitor. + +The program follows the stream selected by rpicam-hello for the HQ camera: + + sensor mode : 2028x1520 (12 bit, packed) + raw stream : BGGR_PISP_COMP1 / SBGGR12_1X12 + sensor Bayer order : BGGR + CFE format : PC1B + +It captures the raw stream, *not* the ISP preview. The four Bayer sites are +split into B, Gb/Gr and R planes, a rectangular ROI is reduced along the slit +direction, and the three one-dimensional spectra are drawn with Matplotlib. + +The Pi 5 path delivers PiSP-compressed PC1B data (format BGGR_PISP_COMP1). +That buffer is expanded with the same decompressor that Picamera2 uses for +DNG output. On a Pi 4 (or when an uncompressed raw stream is selected), the +SBGGR12 16-bit-container path is used directly. Both paths are supported. + +Typical use on the Pi +--------------------- + # dependencies (Raspberry Pi OS, use the system Python) + sudo apt install python3-picamera2 python3-numpy python3-matplotlib python3-tk + + # list the raw modes your camera exposes + python3 hq_spectrum.py --list-modes + + # live display, auto ROI, manual exposure taken from rpicam-hello + python3 hq_spectrum.py --exposure 33013 --gain 16 + + # fixed ROI in full-resolution sensor pixels: x0,y0,x1,y1 + python3 hq_spectrum.py --roi 300,420,1750,980 --exposure 33013 --gain 16 + + # ROI as fractions of the frame + python3 hq_spectrum.py --roi 0.12,0.30,0.85,0.65 + + # two-point wavelength calibration made from known lamp lines + python3 hq_spectrum.py --calibrate 420:405,920:546 --exposure 33013 --gain 16 + +Keys in the live window +----------------------- + s : save the current figure as PNG and the profiles as CSV + q : quit + +Draw a new box with the mouse to change the orange/yellow ROI. + +Demo (works on a normal PC, no camera required) +----------------------------------------------- + python3 hq_spectrum.py --demo --frames 1 --headless --save-prefix demo + +Notes +----- +* The x axis is sensor column pixels until a wavelength calibration is given. + The R/G/B curves are broad CFA responses, not narrow-band spectrometer + channels; use them to see line positions and relative changes. +* No demosaicing or colour processing is applied. That is intentional: it + keeps the spectral lines from being interpolated across neighbouring pixels. +* Auto exposure is convenient for finding a signal but makes intensity ratios + unstable. For measurements use fixed ExposureTime and AnalogueGain. +* The preview is auto-stretched for visibility; the curves show black-level + corrected raw ADU values. +""" + +from __future__ import annotations + +import argparse +import csv +import re +import sys +import time +import traceback +from collections import deque +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional + +import numpy as np + +CHANNELS = ("R", "G", "B") +CHANNEL_COLORS = {"R": "#e43d45", "G": "#21a45a", "B": "#2f6fdd"} +BAYER_LAYOUT = { + "RGGB": (("R", 0, 0), ("G", 0, 1), ("G", 1, 0), ("B", 1, 1)), + "BGGR": (("B", 0, 0), ("G", 0, 1), ("G", 1, 0), ("R", 1, 1)), + "GRBG": (("G", 0, 0), ("R", 0, 1), ("B", 1, 0), ("G", 1, 1)), + "GBRG": (("G", 0, 0), ("B", 0, 1), ("R", 1, 0), ("G", 1, 1)), +} + + +# --------------------------------------------------------------------------- +# Small data containers +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ROI: + """A half-open ROI in full-resolution sensor coordinates: [x0,x1) x [y0,y1).""" + + x0: int + y0: int + x1: int + y1: int + + @property + def width(self) -> int: + return self.x1 - self.x0 + + @property + def height(self) -> int: + return self.y1 - self.y0 + + def text(self) -> str: + return f"({self.x0},{self.y0})-({self.x1},{self.y1})" + + +@dataclass +class SpectrumResult: + roi: ROI + x_px: dict[str, np.ndarray] + x_plot: dict[str, np.ndarray] + signal: dict[str, np.ndarray] + signal_raw: dict[str, np.ndarray] + normalized: dict[str, np.ndarray] + peaks: dict[str, np.ndarray] + preview: np.ndarray + full_size: tuple[int, int] + raw_format: str + bit_depth: int + black: float + full_scale: float + saturated_fraction: float + metadata: dict + x_unit: str + calibration_text: str + timestamp: float + + +class Calibration: + """Polynomial pixel -> wavelength calibration.""" + + def __init__(self, pairs: list[tuple[float, float]], order: int = 1): + if len(pairs) < 2: + raise ValueError("calibration needs at least two pixel:wavelength pairs") + arr = np.asarray(sorted(pairs), dtype=float) + if np.any(np.diff(arr[:, 0]) <= 0): + raise ValueError("calibration pixel positions must be distinct") + self.pixels = arr[:, 0] + self.wavelengths = arr[:, 1] + degree = max(1, min(int(order), len(pairs) - 1)) + self.coeff = np.polyfit(self.pixels, self.wavelengths, degree) + pred = np.polyval(self.coeff, self.pixels) + self.rms_nm = float(np.sqrt(np.mean((pred - self.wavelengths) ** 2))) + + def __call__(self, x: np.ndarray) -> np.ndarray: + return np.polyval(self.coeff, np.asarray(x, dtype=float)) + + def describe(self) -> str: + terms = [] + degree = len(self.coeff) - 1 + for i, c in enumerate(self.coeff): + power = degree - i + terms.append(f"{c:.6g}*x^{power}" if power else f"{c:.6g}") + return "lambda(nm) = " + " + ".join(terms) + f"; rms={self.rms_nm:.2f} nm" + + +# --------------------------------------------------------------------------- +# Bayer / raw helpers +# --------------------------------------------------------------------------- + + +def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Live R/G/B spectral analysis of Raspberry Pi HQ Camera RAW frames.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--width", type=int, default=2028, help="raw stream width") + parser.add_argument("--height", type=int, default=1520, help="raw stream height") + parser.add_argument("--fps", type=float, default=5.0, help="requested capture/display rate") + parser.add_argument( + "--roi", + default="auto", + help="'auto' or x0,y0,x1,y1 in sensor pixels; values <= 1 are treated as fractions", + ) + parser.add_argument( + "--calibrate", + default=None, + metavar="PX:NM[,PX:NM...]", + help="pixel:wavelength pairs, e.g. 420:405,920:546", + ) + parser.add_argument("--poly-order", type=int, default=1, help="polynomial order for calibration") + parser.add_argument( + "--smooth", type=int, default=3, + help="moving-average width in sensor pixels (channel samples are 2 px apart)", + ) + parser.add_argument( + "--reduce", + choices=("mean", "median"), + default="mean", + help="row reduction inside the ROI", + ) + parser.add_argument("--black", type=float, default=None, help="black level override in ADU") + parser.add_argument("--full-scale", type=float, default=None, help="white level override in ADU") + parser.add_argument("--exposure", type=float, default=None, help="manual exposure time in microseconds") + parser.add_argument("--gain", type=float, default=None, help="manual analogue gain") + parser.add_argument( + "--raw-format", + default=None, + help="optional Picamera2 raw format, e.g. SBGGR12 or BGGR_PISP_COMP1", + ) + parser.add_argument("--peak-prominence", type=float, default=0.08, help="normalized peak prominence") + parser.add_argument("--peak-height", type=float, default=0.035, help="minimum normalized peak height") + parser.add_argument("--peak-distance", type=float, default=12.0, help="minimum peak spacing in sensor pixels") + parser.add_argument("--max-peaks", type=int, default=5, help="maximum peaks detected per channel") + parser.add_argument("--waterfall-rows", type=int, default=240, help="number of frames in the waterfall") + parser.add_argument("--demo", action="store_true", help="use a synthetic 12-bit BGGR frame") + parser.add_argument("--headless", action="store_true", help="do not open a window (Agg backend)") + parser.add_argument("--frames", type=int, default=0, help="stop after N frames; 0 means run until closed") + parser.add_argument("--save-prefix", default=None, help="save final dashboard/CSV using this path prefix") + parser.add_argument("--no-selector", action="store_true", help="disable mouse ROI selection") + parser.add_argument("--list-modes", action="store_true", help="list camera raw modes and exit") + parser.add_argument("--verbose", action="store_true", help="print extra raw/ROI diagnostics") + args = parser.parse_args(argv) + + if args.width < 64 or args.height < 64: + parser.error("width and height must be at least 64") + if args.fps <= 0: + parser.error("--fps must be positive") + if args.max_peaks < 0: + parser.error("--max-peaks must be >= 0") + if args.waterfall_rows < 2: + parser.error("--waterfall-rows must be >= 2") + if args.frames < 0: + parser.error("--frames must be >= 0") + return args + + +def pattern_from_format(fmt: str) -> str: + match = re.search(r"([RGB]{4})", str(fmt).upper()) + if not match or match.group(1) not in BAYER_LAYOUT: + raise ValueError(f"cannot determine Bayer order from raw format {fmt!r}") + return match.group(1) + + +def bit_depth_from_format(fmt: str) -> int: + if "PISP_COMP" in str(fmt).upper(): + return 16 + match = re.search(r"([RGB]{4})(\d+)", str(fmt).upper()) + return int(match.group(2)) if match else 16 + + +def make_roi(x0: float, y0: float, x1: float, y1: float, width: int, height: int) -> ROI: + """Clamp and align a ROI to even sensor coordinates (needed by the Bayer grid).""" + + def edge(value: float, upper: int) -> int: + return int(round(max(0.0, min(float(upper), float(value))))) + + xa, xb = sorted((edge(x0, width), edge(x1, width))) + ya, yb = sorted((edge(y0, height), edge(y1, height))) + width_max = width - (width % 2) + height_max = height - (height % 2) + + xa = min(xa, width_max - 2) + ya = min(ya, height_max - 2) + xa -= xa % 2 + ya -= ya % 2 + xb = min(xb, width_max) + yb = min(yb, height_max) + xb = max(xa + 2, xb - ((xb - xa) % 2)) + yb = max(ya + 2, yb - ((yb - ya) % 2)) + return ROI(int(xa), int(ya), int(xb), int(yb)) + + +def parse_roi(value: str, width: int, height: int) -> Optional[ROI]: + value = value.strip().lower() + if value == "auto": + return None + parts = [p.strip() for p in value.split(",")] + if len(parts) != 4: + raise ValueError("--roi must be 'auto' or x0,y0,x1,y1") + try: + numbers = [float(p) for p in parts] + except ValueError as exc: + raise ValueError("ROI values must be numbers") from exc + if not all(np.isfinite(v) for v in numbers): + raise ValueError("ROI values must be finite") + if all(0.0 <= v <= 1.0 for v in numbers): + numbers = [numbers[0] * width, numbers[1] * height, numbers[2] * width, numbers[3] * height] + roi = make_roi(*numbers, width, height) + if roi.width < 24 or roi.height < 8: + raise ValueError("ROI is too small") + return roi + + +def decompress_pisp_comp1(array: np.ndarray) -> np.ndarray: + """Expand a PiSP PC1B/COMP1 raw buffer to a 2-D uint16 Bayer image. + + Adapted from Picamera2's helpers.decompress() (BSD-2-Clause, + Raspberry Pi Ltd.); see THIRD_PARTY_NOTICES.md. Keeping a local copy + makes the analyser work with older Picamera2 versions and avoids + depending on a helper method. + """ + + array = np.ascontiguousarray(array) + if array.dtype != np.uint8: + array = array.view(np.uint8) + if array.ndim != 2: + raise ValueError(f"unexpected PC1B buffer shape {array.shape}") + # PC1B works in 8-byte groups. Some libcamera versions report the + # compressed stride as the natural width (e.g. 2028) rather than a padded + # value, so zero-pad the tail of each row before expanding it. + if array.shape[1] % 8: + pad = (-array.shape[1]) % 8 + array = np.pad(array, ((0, 0), (0, pad)), mode="constant") + + offset = 2048 + words = array.view(np.int32) + words = words.reshape((words.shape[0], words.shape[1] // 2, 2)) + qmode = words & 3 + pix0 = (words >> 2) & 511 + pix1 = ((words >> 11) & 127) - 64 + pix2 = (words >> 18) & 127 + pix3 = (words >> 25) & 127 + + q1 = np.copy(pix0) + q2 = pix1 + 448 + np.maximum(pix0, pix0 - pix1, where=(qmode * pix0 < 768), out=q1) + np.maximum(pix0, pix0 + pix1, where=(qmode * pix0 < 768), out=q2) + q0 = np.minimum(1536 >> qmode, np.maximum(0, q1 - 64)) + pix2 + q3 = np.minimum(1536 >> qmode, np.maximum(0, q2 - 64)) + pix3 + np.maximum(np.maximum(16 * q0, 32 * (q0 - 160)), 64 * qmode * q0, out=pix0) + np.maximum(np.maximum(16 * q1, 32 * (q1 - 160)), 64 * qmode * q1, out=pix1) + np.maximum(np.maximum(16 * q2, 32 * (q2 - 160)), 64 * qmode * q2, out=pix2) + np.maximum(np.maximum(16 * q3, 32 * (q3 - 160)), 64 * qmode * q3, out=pix3) + + q2 = (words >> 2) & 32767 + q3 = (words >> 17) & 32767 + q0 = (q2 & 15) + 16 * ((q2 >> 8) // 11) + q1 = (q2 >> 4) % 176 + q2 = (q3 & 15) + 16 * ((q3 >> 8) // 11) + q3 = (q3 >> 4) % 176 + np.maximum(256 * q0, 512 * (q0 - 47), out=pix0, where=(qmode == 3)) + np.maximum(256 * q1, 512 * (q1 - 47), out=pix1, where=(qmode == 3)) + np.maximum(256 * q2, 512 * (q2 - 47), out=pix2, where=(qmode == 3)) + np.maximum(256 * q3, 512 * (q3 - 47), out=pix3, where=(qmode == 3)) + + res = np.stack((pix0, pix1, pix2, pix3), axis=2).reshape(array.shape) + return np.clip(res + offset, 0, 65535).astype(np.uint16) + + +def unpack_csi2(data: np.ndarray, bit_depth: int) -> np.ndarray: + """Unpack MIPI CSI-2 packed 10/12-bit rows (stride-sized raw rows).""" + + height, stride = data.shape + if bit_depth == 12: + if stride % 3: + raise ValueError(f"12-bit packed stride {stride} is not a multiple of 3") + width = stride // 3 * 2 + groups = np.ascontiguousarray(data).reshape(height, stride // 3, 3).astype(np.uint16) + out = np.empty((height, width), dtype=np.uint16) + out[:, 0::2] = groups[:, :, 0] | ((groups[:, :, 1] & 0x0F) << 8) + out[:, 1::2] = (groups[:, :, 1] >> 4) | (groups[:, :, 2] << 4) + return out + if bit_depth == 10: + if stride % 5: + raise ValueError(f"10-bit packed stride {stride} is not a multiple of 5") + width = stride // 5 * 4 + groups = np.ascontiguousarray(data).reshape(height, stride // 5, 5).astype(np.uint16) + out = np.empty((height, width), dtype=np.uint16) + out[:, 0::4] = groups[:, :, 0] | ((groups[:, :, 1] & 0x03) << 8) + out[:, 1::4] = (groups[:, :, 1] >> 2) | ((groups[:, :, 2] & 0x0F) << 6) + out[:, 2::4] = (groups[:, :, 2] >> 4) | ((groups[:, :, 3] & 0x3F) << 4) + out[:, 3::4] = (groups[:, :, 3] >> 6) | (groups[:, :, 4] << 2) + return out + raise ValueError(f"unsupported packed bit depth {bit_depth}") + + +def decode_raw(raw: np.ndarray, config: dict, fmt: str) -> tuple[np.ndarray, int]: + """Turn a Picamera2 raw buffer into a 2-D uint16 Bayer image. + + Picamera2's raw stream is normally a uint8 byte buffer of shape + (height, stride), including for 16-bit container formats. A few code + paths / future versions can hand over an already-unpacked uint16 array; + both representations are accepted here. + """ + + raw = np.asarray(raw) + width, height = (int(v) for v in config["size"]) + stride = int(config.get("stride") or (width * 2)) + fmt = str(fmt) + upper = fmt.upper() + + if raw.ndim == 1: + if raw.dtype == np.uint8: + raw = raw.reshape(height, stride) + elif raw.dtype.itemsize == 2: + raw = raw.reshape(height, -1) + else: + raise ValueError(f"unsupported raw dtype {raw.dtype}") + if raw.ndim != 2 or raw.shape[0] != height: + raise ValueError(f"raw buffer shape {raw.shape} does not match {height}x{width}") + + if "PISP_COMP1" in upper: + if raw.dtype != np.uint8: + raise ValueError("PC1B raw buffer must be a uint8 byte buffer") + image = decompress_pisp_comp1(raw) + if image.shape[1] < width: + raise ValueError(f"decompressed raw has only {image.shape[1]} columns, expected {width}") + return np.ascontiguousarray(image[:, :width]), 16 + + bit_depth = bit_depth_from_format(fmt) + mask = (1 << bit_depth) - 1 if bit_depth < 16 else None + + if raw.dtype.itemsize == 2: + # Already unpacked samples: shape is (height, stride_in_samples). + if raw.shape[1] < width: + raise ValueError( + f"unpacked raw has only {raw.shape[1]} columns, expected {width}; format={fmt!r}" + ) + image = raw[:, :width].astype(np.uint16, copy=True) + if mask is not None: + image &= mask + return image, bit_depth + + if "CSI2P" in upper or "_1X12" in upper or "_1X10" in upper: + # Stride is often padded (e.g. to 32 bytes). Unpack only the real row + # payload, otherwise the padding is interpreted as extra pixels. + row_bytes = (width * bit_depth + 7) // 8 + if raw.shape[1] < row_bytes: + raise ValueError(f"packed raw stride {raw.shape[1]} is too small for {width} pixels") + image = unpack_csi2(np.ascontiguousarray(raw[:, :row_bytes]), bit_depth) + if image.shape[1] < width: + raise ValueError(f"unpacked raw has only {image.shape[1]} columns, expected {width}") + return np.ascontiguousarray(image[:, :width]), bit_depth + + if bit_depth == 8: + if raw.shape[1] < width: + raise ValueError(f"raw stride {raw.shape[1]} is narrower than {width}") + return np.ascontiguousarray(raw[:, :width]).astype(np.uint16), 8 + + need = width * 2 + if raw.shape[1] < need: + raise ValueError( + f"raw stride {raw.shape[1]} is too small for {width} 16-bit samples; " + f"format={fmt!r}" + ) + image = np.ascontiguousarray(raw[:, :need]).view(" tuple[float, float]: + full = float(full_override) if full_override is not None else float((1 << bit_depth) - 1) + if black_override is not None: + black = float(black_override) + else: + value = meta.get("BlackLevel", 0) + if isinstance(value, (tuple, list, np.ndarray)): + value = float(np.mean(value)) + black = float(value) if value is not None else 0.0 + if not np.isfinite(black) or black < 0: + black = 0.0 + if black >= full: + # A 12-bit sensor can report a 16-bit-scaled black level when the + # metadata and the selected stream disagree. Rescale instead of + # subtracting a nonsense value. + black = black * full / 65535.0 + return black, full + + +# --------------------------------------------------------------------------- +# Preview, ROI and spectrum processing +# --------------------------------------------------------------------------- + + +def extract_planes(raw: np.ndarray, pattern: str, roi: ROI) -> dict[str, np.ndarray]: + """Extract R, G and B planes from a BGGR/RGGB/... ROI. + + The two green sites are averaged. All returned planes have the same shape + because the ROI is aligned to even coordinates. + """ + + if roi.width < 2 or roi.height < 2: + raise ValueError(f"ROI is empty or too small: {roi.text()}") + crop = raw[roi.y0:roi.y1, roi.x0:roi.x1] + if crop.size == 0 or crop.shape[0] < 2 or crop.shape[1] < 2: + raise ValueError(f"ROI {roi.text()} does not overlap the raw frame") + layout = BAYER_LAYOUT[pattern] + by_color: dict[str, list[np.ndarray]] = {c: [] for c in CHANNELS} + for color, dy, dx in layout: + by_color[color].append(crop[dy::2, dx::2]) + + planes: dict[str, np.ndarray] = {} + for color, arrays in by_color.items(): + if color == "G" and len(arrays) == 2: + rows = min(a.shape[0] for a in arrays) + cols = min(a.shape[1] for a in arrays) + planes[color] = (arrays[0][:rows, :cols].astype(np.float32) + + arrays[1][:rows, :cols].astype(np.float32)) * 0.5 + else: + planes[color] = arrays[0].astype(np.float32) + return planes + + +def channel_x_positions(roi: ROI, pattern: str, count_g: int, count_rb: int) -> dict[str, np.ndarray]: + """Full-resolution sensor x coordinate for every CFA sample.""" + + positions = {} + for color, dy, dx in BAYER_LAYOUT[pattern]: + if color == "G": + positions[color] = roi.x0 + 0.5 + 2.0 * np.arange(count_g, dtype=float) + else: + positions[color] = roi.x0 + float(dx) + 2.0 * np.arange(count_rb, dtype=float) + return positions + + +def smooth_vector(y: np.ndarray, width: int) -> np.ndarray: + width = int(width) + if width <= 1 or y.size == 0: + return y.astype(np.float32, copy=True) + width = min(width, y.size) + if width <= 1: + return y.astype(np.float32, copy=True) + # Symmetric edge padding works for both odd and even window widths. + left = (width - 1) // 2 + right = width // 2 + padded = np.pad(y.astype(np.float32), (left, right), mode="edge") + kernel = np.ones(width, dtype=np.float32) / width + return np.convolve(padded, kernel, mode="valid").astype(np.float32) + + +def find_peaks_normalized(y: np.ndarray, x: np.ndarray, min_distance: int, + prominence: float, min_height: float, + max_peaks: int) -> np.ndarray: + """A small scipy-free local-maximum/prominence finder.""" + + n = y.size + if n < 3 or max_peaks <= 0: + return np.empty(0, dtype=int) + candidates = np.where((y[1:-1] >= y[:-2]) & (y[1:-1] > y[2:]))[0] + 1 + if candidates.size == 0: + return np.empty(0, dtype=int) + + # Highest first, then suppress neighbours. + candidates = candidates[np.argsort(y[candidates])[::-1]] + min_distance = max(1, int(min_distance)) + prom_window = max(8, min_distance * 4) + kept: list[int] = [] + for idx in candidates: + if y[idx] < min_height: + continue + if any(abs(int(idx) - k) < min_distance for k in kept): + continue + left = y[max(0, idx - prom_window):idx] + right = y[idx + 1:min(n, idx + 1 + prom_window)] + if left.size == 0 or right.size == 0: + base = 0.0 + else: + base = max(float(left.min()), float(right.min())) + if float(y[idx]) - base < prominence: + continue + kept.append(int(idx)) + if len(kept) >= max_peaks: + break + + kept.sort(key=lambda i: float(x[i])) + return np.asarray(kept, dtype=int) + + +def make_preview(raw: np.ndarray, pattern: str, target_width: int = 640) -> np.ndarray: + """Quick, ISP-free RGB preview by binning the four Bayer sites.""" + + height = raw.shape[0] - (raw.shape[0] % 2) + width = raw.shape[1] - (raw.shape[1] % 2) + sub = raw[:height, :width] + layout = BAYER_LAYOUT[pattern] + by_color: dict[str, list[np.ndarray]] = {c: [] for c in CHANNELS} + for color, dy, dx in layout: + by_color[color].append(sub[dy::2, dx::2].astype(np.float32)) + + channels = [] + for color in CHANNELS: + arrays = by_color[color] + plane = sum(arrays).astype(np.float32) + plane /= len(arrays) + channels.append(plane) + rgb = np.stack(channels, axis=-1) + + factor = max(1, int(np.ceil(rgb.shape[1] / float(target_width)))) + if factor > 1: + hh = (rgb.shape[0] // factor) * factor + ww = (rgb.shape[1] // factor) * factor + rgb = rgb[:hh, :ww].reshape(hh // factor, factor, ww // factor, factor, 3).mean(axis=(1, 3)) + + # Auto-stretch per channel, subsample percentile computation for speed. + flat = rgb.reshape(-1, 3) + sample = flat[::3] if flat.shape[0] > 5000 else flat + lo = np.percentile(sample, 1.0, axis=0) + hi = np.percentile(sample, 99.6, axis=0) + scale = np.maximum(hi - lo, 1.0) + out = np.clip((rgb - lo) / scale, 0.0, 1.0) + return np.power(out, 0.8, dtype=np.float32) + + +def detect_auto_roi(preview: np.ndarray, full_width: int, full_height: int) -> Optional[ROI]: + """Find a horizontal spectral band from a stretched preview.""" + + height, width = preview.shape[:2] + if height < 8 or width < 8: + return None + luminance = 0.299 * preview[..., 0] + 0.587 * preview[..., 1] + 0.114 * preview[..., 2] + block = max(2, min(12, min(height, width) // 24)) + hh = (height // block) * block + ww = (width // block) * block + if hh < block or ww < block: + return None + blocks = luminance[:hh, :ww].reshape(hh // block, block, ww // block, block).mean(axis=(1, 3)) + + background = float(np.median(blocks)) + mad = float(np.median(np.abs(blocks - background))) + high = float(np.percentile(blocks, 99.9)) + threshold = background + max(8.0 * 1.4826 * mad, 0.02 * (high - background)) + mask = blocks > threshold + if int(mask.sum()) < 4: + return None + + row_counts = mask.sum(axis=1) + active_rows = np.where(row_counts >= max(1.0, 0.12 * row_counts.max()))[0] + if active_rows.size == 0: + return None + r0, r1 = int(active_rows.min()), int(active_rows.max()) + 1 + band = mask[r0:r1, :] + col_counts = band.sum(axis=0) + active_cols = np.where(col_counts > 0)[0] + if active_cols.size == 0: + return None + c0, c1 = int(active_cols.min()), int(active_cols.max()) + 1 + + scale_x = full_width / float(width) + scale_y = full_height / float(height) + x0 = c0 * block * scale_x + x1 = c1 * block * scale_x + y0 = r0 * block * scale_y + y1 = r1 * block * scale_y + + # Keep some dark baseline and make sure a narrow band still has context. + margin_x = 0.035 * full_width + margin_y = 0.035 * full_height + x0, x1 = x0 - margin_x, x1 + margin_x + y0, y1 = y0 - margin_y, y1 + margin_y + if x1 - x0 < 0.18 * full_width: + center = 0.5 * (x0 + x1) + x0, x1 = center - 0.09 * full_width, center + 0.09 * full_width + if y1 - y0 < 0.08 * full_height: + center = 0.5 * (y0 + y1) + y0, y1 = center - 0.04 * full_height, center + 0.04 * full_height + roi = make_roi(x0, y0, x1, y1, full_width, full_height) + if roi.width < 32 or roi.height < 8: + return None + return roi + + +def process_frame(raw: np.ndarray, meta: dict, roi: ROI, pattern: str, + args: argparse.Namespace, calibration: Optional[Calibration], + full_size: tuple[int, int], raw_format: str, + bit_depth: int) -> SpectrumResult: + planes = extract_planes(raw, pattern, roi) + if args.reduce == "median": + profiles = {c: np.median(planes[c], axis=0).astype(np.float32) for c in CHANNELS} + else: + profiles = {c: np.mean(planes[c], axis=0, dtype=np.float32) for c in CHANNELS} + + counts_g = min(planes["R"].shape[1], planes["B"].shape[1], planes["G"].shape[1]) + counts_rb = min(planes["R"].shape[1], planes["B"].shape[1]) + positions = channel_x_positions(roi, pattern, counts_g, counts_rb) + + black, full_scale = resolve_levels(meta, bit_depth, args.black, args.full_scale) + signal = {} + signal_raw = {} + normalized = {} + peaks = {} + x_px = {} + x_plot = {} + unit = "nm" if calibration is not None else "px" + peak_distance_samples = max(1, int(round(args.peak_distance / 2.0))) + smooth_samples = 1 if args.smooth <= 1 else max(1, (args.smooth + 1) // 2) + + for channel in CHANNELS: + n = min(profiles[channel].size, positions[channel].size) + y = np.maximum(profiles[channel][:n] - black, 0.0).astype(np.float32) + ys = smooth_vector(y, smooth_samples) + sig = np.clip(ys, 0.0, None) + base = float(np.percentile(sig, 5.0)) if sig.size else 0.0 + norm = np.clip(sig - base, 0.0, None) + peak = float(norm.max()) if norm.size else 0.0 + if peak > 0: + norm /= peak + else: + norm[:] = 0.0 + + x = positions[channel][:n] + x_display = calibration(x) if calibration is not None else x.copy() + idx = find_peaks_normalized( + norm, x_display, peak_distance_samples, + args.peak_prominence, args.peak_height, args.max_peaks + ) + + x_px[channel] = x + x_plot[channel] = x_display + signal[channel] = sig + signal_raw[channel] = y + normalized[channel] = norm + peaks[channel] = idx + + preview = make_preview(raw, pattern) + roi_raw = raw[roi.y0:roi.y1, roi.x0:roi.x1] + saturated = float(np.mean(roi_raw >= (0.995 * full_scale))) if roi_raw.size else 0.0 + return SpectrumResult( + roi=roi, + x_px=x_px, + x_plot=x_plot, + signal=signal, + signal_raw=signal_raw, + normalized=normalized, + peaks=peaks, + preview=preview, + full_size=full_size, + raw_format=raw_format, + bit_depth=bit_depth, + black=black, + full_scale=full_scale, + saturated_fraction=saturated, + metadata=meta, + x_unit=unit, + calibration_text=calibration.describe() if calibration is not None else "uncalibrated (sensor x)", + timestamp=time.time(), + ) + + +# --------------------------------------------------------------------------- +# Synthetic source (also useful for testing on a laptop) +# --------------------------------------------------------------------------- + + +class DemoSource: + """A BGGR 12-bit synthetic spectral frame with a horizontal dispersion axis.""" + + description = "synthetic HQ-camera-like BGGR12 frame" + + def __init__(self, args: argparse.Namespace): + self.args = args + self.full_size = (args.width - args.width % 2, args.height - args.height % 2) + self.raw_format = "SBGGR12 (demo)" + self.bit_depth = 12 + self.pattern = "BGGR" + self.stride = self.full_size[0] * 2 + self.rng = np.random.default_rng(20261023) + self.frame_index = 0 + self.base = self._build_base() + self.meta = { + "ExposureTime": 33013, + "AnalogueGain": 16.0, + "DigitalGain": 1.0, + "BlackLevel": 256, + "Lux": 12.0, + "SensorTimestamp": time.monotonic_ns(), + "Demo": True, + } + + def _gaussian(self, x: np.ndarray, center: float, sigma: float) -> np.ndarray: + return np.exp(-0.5 * ((x - center) / sigma) ** 2) + + def _build_base(self) -> np.ndarray: + width, height = self.full_size + w2, h2 = width // 2, height // 2 + x = 2.0 * np.arange(w2) + 0.5 + x0, x1 = 0.12 * width, 0.88 * width + wavelength = np.interp(x, (x0, x1), (380.0, 760.0), left=np.nan, right=np.nan) + valid = np.isfinite(wavelength) + wl = np.where(valid, wavelength, 550.0) + + # Mercury/neon-like line mixture on a small continuum. + radiance = np.where(valid, 0.10 + 0.00002 * (wl - 380.0), 0.0) + for center, amplitude, sigma in ( + (404.7, 0.40, 1.7), + (435.8, 1.00, 2.1), + (546.1, 0.92, 2.4), + (577.0, 0.45, 2.0), + (579.1, 0.40, 2.0), + (611.6, 0.34, 2.3), + (696.5, 0.48, 3.0), + ): + radiance = radiance + amplitude * self._gaussian(wl, center, sigma) * valid + + # Broad CFA response curves (illustrative, not a measured QE curve). + response = { + "R": 0.95 * self._gaussian(wl, 615.0, 78.0) + 0.10 * self._gaussian(wl, 540.0, 55.0), + "G": 0.88 * self._gaussian(wl, 545.0, 68.0) + 0.12 * self._gaussian(wl, 455.0, 45.0) + 0.08 * self._gaussian(wl, 635.0, 55.0), + "B": 0.90 * self._gaussian(wl, 458.0, 55.0) + 0.08 * self._gaussian(wl, 535.0, 60.0), + } + + # Horizontal slit image: a soft vertical band with a little tilt and texture. + yy = np.arange(h2, dtype=np.float32) + tilt = 2.0 * np.sin(np.linspace(0.0, 1.0, w2) * np.pi) + center_y = 0.53 * h2 + tilt + vertical = np.exp(-0.5 * ((yy[:, None] - center_y[None, :]) / (0.075 * h2)) ** 2) + + vignette = 1.0 - 0.18 * ((x - 0.5 * width) / (0.5 * width)) ** 2 + channel_images = {} + for channel in CHANNELS: + signal = radiance[None, :] * response[channel][None, :] * vertical + signal *= 2600.0 * vignette[None, :] + signal += 256.0 + 18.0 * np.linspace(0.0, 1.0, w2)[None, :] + channel_images[channel] = signal.astype(np.float32) + + raw = np.empty((h2 * 2, w2 * 2), dtype=np.float32) + raw[0::2, 0::2] = channel_images["B"] + raw[0::2, 1::2] = channel_images["G"] + raw[1::2, 0::2] = channel_images["G"] + raw[1::2, 1::2] = channel_images["R"] + + # A few hot pixels make sure the row reduction is not blindly trusting + # a single pixel. Mean reduction still works because they are rare. + for _ in range(80): + raw[self.rng.integers(0, raw.shape[0]), self.rng.integers(0, raw.shape[1])] = 4095.0 + return raw + + def start(self) -> None: + pass + + def read(self) -> tuple[np.ndarray, dict]: + self.frame_index += 1 + flicker = 1.0 + 0.012 * np.sin(self.frame_index * 0.23) + noise = self.rng.normal(0.0, 3.0, self.base.shape).astype(np.float32) + raw = np.clip(self.base * flicker + noise, 0.0, 4095.0).astype(np.uint16) + meta = dict(self.meta) + meta["SensorTimestamp"] = time.monotonic_ns() + meta["Lux"] = float(12.0 + 0.3 * np.sin(self.frame_index * 0.17)) + return raw, meta + + def close(self) -> None: + pass + + +class CameraSource: + """Picamera2 raw source; decodes PC1B or uncompressed SBGGR12.""" + + def __init__(self, args: argparse.Namespace): + self.args = args + try: + from picamera2 import Picamera2 + except Exception as exc: # pragma: no cover - depends on the Pi + raise RuntimeError( + "Picamera2 is not available. On Raspberry Pi OS use the system Python:\n" + " sudo apt install python3-picamera2\n" + " python3 hq_spectrum.py ..." + ) from exc + self.picam2 = Picamera2() + model = str(self.picam2.camera_properties.get("Model", "Raspberry Pi camera")) + self.description = "HQ Camera (IMX477)" if "imx477" in model.lower() else model + self._configure() + + cfg = self.picam2.camera_config["raw"] + self.full_size = (int(cfg["size"][0]), int(cfg["size"][1])) + self.raw_format = str(cfg["format"]) + self.stride = int(cfg["stride"]) + self.pattern = pattern_from_format(self.raw_format) + self.bit_depth = bit_depth_from_format(self.raw_format) + self.camera_config_raw = cfg + self._started = False + + def _controls(self) -> dict: + controls: dict = {"FrameRate": float(self.args.fps)} + if self.args.exposure is not None or self.args.gain is not None: + controls["AeEnable"] = False + controls["ExposureTime"] = int(self.args.exposure if self.args.exposure is not None else 10000) + controls["AnalogueGain"] = float(self.args.gain if self.args.gain is not None else 1.0) + return controls + + def _configure(self) -> None: + requested = self.args.raw_format.upper() if self.args.raw_format else None + + def configure_with(use_format: Optional[str]) -> None: + cfg = {"size": (self.args.width, self.args.height)} + if use_format: + cfg["format"] = use_format + config = self.picam2.create_video_configuration( + main={"size": (640, 480)}, + raw=cfg, + controls=self._controls(), + buffer_count=4, + ) + self.picam2.configure(config) + + try: + configure_with(requested) + except Exception as exc: + if not requested: + raise + print(f"warning: raw format {requested!r} was rejected ({exc}); retrying with automatic format", + file=sys.stderr) + configure_with(None) + + def start(self) -> None: + self.picam2.start() + self._started = True + + def read(self) -> tuple[np.ndarray, dict]: + request = self.picam2.capture_request() + try: + raw_bytes = request.make_array("raw") + meta = request.get_metadata() + # Decode while the request is still alive; make_array() normally + # copies, but a future Picamera2 might return a mapped view. + image, _ = decode_raw(raw_bytes, self.camera_config_raw, self.raw_format) + finally: + request.release() + return image, meta + + def close(self) -> None: + try: + if self._started: + self.picam2.stop() + finally: + self.picam2.close() + + +def list_camera_modes() -> None: + try: + from picamera2 import Picamera2 + except Exception as exc: + raise RuntimeError("Picamera2 is not available. Try: sudo apt install python3-picamera2") from exc + + cam = Picamera2() + try: + model = cam.camera_properties.get("Model", "unknown camera") + print(f"Camera: {model}") + print(f"{'format':26s} {'size':>12s} {'bit':>3s} {'fps':>6s}") + print("-" * 56) + for mode in cam.sensor_modes: + size = mode.get("size", ("?", "?")) + bits = mode.get("bit_depth", "?") + fps = mode.get("fps", "?") + print(f"{str(mode.get('format', '?')):26s} {size[0]:5d}x{size[1]:<5d} {str(bits):>3s} {str(fps):>6s}") + finally: + cam.close() + + +# --------------------------------------------------------------------------- +# Matplotlib dashboard +# --------------------------------------------------------------------------- + + +class Dashboard: + def __init__(self, plt, args: argparse.Namespace, full_size: tuple[int, int], + description: str, calibration_text: str, on_roi=None): + self.plt = plt + self.args = args + self.full_w, self.full_h = full_size + self.on_roi = on_roi + self.selector = None + self.save_callback = None + self.last_result: Optional[SpectrumResult] = None + self.headless = args.headless + self.quit_requested = False + + if not self.headless: + plt.ion() + self.fig = plt.figure(figsize=(13.2, 8.2), constrained_layout=True) + try: + self.fig.canvas.manager.set_window_title("HQ Camera RAW spectrum") + except Exception: + pass + self.fig.suptitle(f"HQ Camera RAW R/G/B spectrum monitor — {description}", fontsize=11) + + gs = self.fig.add_gridspec(2, 2, width_ratios=(1.18, 1.0), height_ratios=(1.08, 1.0)) + self.ax_preview = self.fig.add_subplot(gs[0, 0]) + self.ax_spec = self.fig.add_subplot(gs[0, 1]) + self.ax_water = self.fig.add_subplot(gs[1, 0]) + self.ax_norm = self.fig.add_subplot(gs[1, 1]) + + self.im_preview = self.ax_preview.imshow( + np.zeros((2, 2, 3), dtype=np.float32), + extent=(0, self.full_w, self.full_h, 0), + origin="upper", + interpolation="nearest", + aspect="equal", + ) + self.ax_preview.set_xlim(0, self.full_w) + self.ax_preview.set_ylim(self.full_h, 0) + self.ax_preview.set_xlabel("sensor x / px") + self.ax_preview.set_ylabel("sensor y / px") + selector_wanted = not self.headless and not args.no_selector + preview_title = ("RAW preview (auto-stretched) — drag a box to set ROI" + if selector_wanted else "RAW preview (auto-stretched) and ROI") + self.ax_preview.set_title(preview_title, fontsize=8.5) + self.roi_patch = self._make_roi_patch(0, 0, 10, 10) + self.ax_preview.add_patch(self.roi_patch) + self.status_text = self.ax_preview.text( + 0.012, 0.985, "", transform=self.ax_preview.transAxes, + va="top", ha="left", fontsize=7.1, family="monospace", color="white", + bbox=dict(facecolor="black", alpha=0.58, edgecolor="none", pad=3.5), + zorder=20, + ) + + # Absolute spectra. + self.raw_lines = {} + self.smooth_lines = {} + for channel in CHANNELS: + (raw_line,) = self.ax_spec.plot([], [], color=CHANNEL_COLORS[channel], + lw=0.55, alpha=0.22) + (smooth_line,) = self.ax_spec.plot([], [], color=CHANNEL_COLORS[channel], + lw=1.55, label=channel) + self.raw_lines[channel] = raw_line + self.smooth_lines[channel] = smooth_line + self.ax_spec.set_title(f"R / G / B profiles: {args.reduce} over ROI rows", fontsize=9) + self.ax_spec.set_ylabel("black-corrected signal / ADU") + self.ax_spec.grid(alpha=0.24) + self.ax_spec.legend(loc="upper right", ncol=3, fontsize=7, framealpha=0.85) + + # Normalized spectra + peak markers. + self.norm_lines = {} + self.peak_markers = {} + for channel in CHANNELS: + (line,) = self.ax_norm.plot([], [], color=CHANNEL_COLORS[channel], + lw=1.45, label=channel) + (marker,) = self.ax_norm.plot([], [], linestyle="none", marker="o", ms=3.2, + color=CHANNEL_COLORS[channel], mec="white", mew=0.35) + self.norm_lines[channel] = line + self.peak_markers[channel] = marker + self.ax_norm.set_title("normalized profiles (baseline removed) and peaks", fontsize=9) + self.ax_norm.set_ylabel("normalized") + self.ax_norm.set_ylim(0, 1.08) + self.ax_norm.grid(alpha=0.24) + self.ax_norm.legend(loc="upper right", ncol=3, fontsize=7, framealpha=0.85) + + # Waterfall. + self.im_water = self.ax_water.imshow( + np.zeros((2, 2), dtype=np.float32), + extent=(0, self.full_w, 0, 2), + origin="lower", aspect="auto", cmap="inferno", vmin=0.0, vmax=1.0, + interpolation="nearest", + ) + self.ax_water.set_title("G-channel waterfall (normalized)", fontsize=9) + self.ax_water.set_ylabel("recent frames (old → new)") + self.ax_water.set_xlim(0, self.full_w) + + self.calibration_text = calibration_text + self.ax_norm.set_xlabel(self._xlabel()) + self.ax_water.set_xlabel(self._xlabel(), fontsize=8) + + if selector_wanted: + from matplotlib.widgets import RectangleSelector + selector_style = dict(facecolor="none", edgecolor="#43d9ff", linewidth=1.1, + linestyle="--", alpha=0.95) + try: + self.selector = RectangleSelector( + self.ax_preview, + self._on_select, + useblit=False, + button=[1], + minspanx=8, + minspany=4, + spancoords="data", + props=selector_style, + ) + except TypeError: + # Matplotlib < 3.5 used "rectprops" for the same thing. + self.selector = RectangleSelector( + self.ax_preview, + self._on_select, + useblit=False, + button=[1], + minspanx=8, + minspany=4, + spancoords="data", + rectprops=selector_style, + ) + + self.fig.canvas.mpl_connect("key_press_event", self._on_key) + if not self.headless: + plt.show(block=False) + + def _xlabel(self) -> str: + if "uncalibrated" in self.calibration_text: + return "sensor column / px (uncalibrated)" + return "wavelength / nm" + + def _make_roi_patch(self, x0: int, y0: int, w: int, h: int): + from matplotlib.patches import Rectangle + return Rectangle((x0, y0), w, h, fill=False, edgecolor="#ffd166", + linewidth=1.5, zorder=15) + + def _on_select(self, eclick, erelease) -> None: + if (eclick.xdata is None or erelease.xdata is None or + eclick.ydata is None or erelease.ydata is None or self.on_roi is None): + return + x0, x1 = sorted((eclick.xdata, erelease.xdata)) + y0, y1 = sorted((eclick.ydata, erelease.ydata)) + roi = make_roi(x0, y0, x1, y1, self.full_w, self.full_h) + if roi.width < 24 or roi.height < 8: + return + self.on_roi(roi) + + def _on_key(self, event) -> None: + if event.key == "s" and self.save_callback is not None: + self.save_callback() + elif event.key == "q": + # Do not close the canvas here: the main loop still has to save + # the final figure if --save-prefix was given. + self.quit_requested = True + + def update(self, result: SpectrumResult, waterfall: np.ndarray, fps: float, + status_text: str) -> None: + self.last_result = result + self.im_preview.set_data(result.preview) + self.roi_patch.set_bounds(result.roi.x0, result.roi.y0, result.roi.width, result.roi.height) + self.status_text.set_text(status_text) + + valid_x = [v for v in result.x_plot.values() if v.size] + if valid_x: + x_min = min(float(v.min()) for v in valid_x) + x_max = max(float(v.max()) for v in valid_x) + else: + x_min, x_max = 0.0, float(self.full_w) + if not np.isfinite(x_min) or not np.isfinite(x_max) or x_max <= x_min: + x_min, x_max = 0.0, float(self.full_w) + for channel in CHANNELS: + x = result.x_plot[channel] + self.raw_lines[channel].set_data(x, result.signal_raw[channel]) + self.smooth_lines[channel].set_data(x, result.signal[channel]) + self.norm_lines[channel].set_data(x, result.normalized[channel]) + idx = result.peaks[channel] + self.peak_markers[channel].set_data(x[idx], result.normalized[channel][idx]) + self.ax_spec.set_xlim(x_min, x_max) + valid_y = [v for v in result.signal.values() if v.size] + y_max = max(float(v.max()) for v in valid_y) if valid_y else 1.0 + self.ax_spec.set_ylim(0.0, max(1.0, y_max * 1.1)) + self.ax_norm.set_xlim(x_min, x_max) + self.ax_water.set_xlim(x_min, x_max) + self.ax_norm.set_xlabel(self._xlabel()) + self.ax_water.set_xlabel(self._xlabel(), fontsize=8) + + if waterfall.size: + self.im_water.set_data(waterfall) + self.im_water.set_extent((x_min, x_max, 0, waterfall.shape[0])) + self.ax_water.set_ylim(0, max(1, waterfall.shape[0])) + + if self.headless: + # Redrawing a full figure for every frame is needlessly expensive + # in a batch/headless run. savefig() at the end draws the final + # state once. + return + self.fig.canvas.draw_idle() + self.fig.canvas.flush_events() + + def pump(self, seconds: float, headless: bool) -> None: + if seconds <= 0: + return + if headless: + time.sleep(seconds) + else: + self.plt.pause(seconds) + + def save(self, prefix: str) -> tuple[Path, Path]: + if self.last_result is None: + raise RuntimeError("no frame to save yet") + png, csv_path = save_result(self.fig, self.last_result, prefix) + return png, csv_path + + def close(self) -> None: + self.plt.close(self.fig) + + +# --------------------------------------------------------------------------- +# Save helpers +# --------------------------------------------------------------------------- + + +def _safe_prefix(prefix: str) -> Path: + path = Path(prefix).expanduser() + if path.suffix.lower() in (".png", ".csv"): + path = path.with_suffix("") + path.parent.mkdir(parents=True, exist_ok=True) + return path + + +def save_result(fig, result: SpectrumResult, prefix: str) -> tuple[Path, Path]: + base = _safe_prefix(prefix) + png_path = base.with_suffix(".png") + csv_path = base.with_suffix(".csv") + fig.savefig(png_path, dpi=150) + + with open(csv_path, "w", newline="", encoding="utf-8") as handle: + handle.write(f"# hq_spectrum snapshot {datetime.now().isoformat(timespec='seconds')}\n") + handle.write(f"# roi={result.roi.text()} size={result.roi.width}x{result.roi.height}\n") + handle.write(f"# raw_format={result.raw_format} bit_depth={result.bit_depth} " + f"black={result.black:g} full_scale={result.full_scale:g}\n") + handle.write(f"# calibration={result.calibration_text}\n") + handle.write(f"# peaks_{result.x_unit}: R={format_peak_values(result, 'R', 5)} " + f"G={format_peak_values(result, 'G', 5)} " + f"B={format_peak_values(result, 'B', 5)}\n") + writer = csv.writer(handle) + writer.writerow(["channel", "x_px", "x_display", "x_unit", + "signal_adu", "signal_smoothed", "normalized"]) + for channel in CHANNELS: + x_px = result.x_px[channel] + x_plot = result.x_plot[channel] + n = min(x_px.size, x_plot.size, result.signal_raw[channel].size, + result.signal[channel].size, result.normalized[channel].size) + for i in range(n): + writer.writerow([ + channel, + f"{x_px[i]:.3f}", + f"{x_plot[i]:.4f}", + result.x_unit, + f"{result.signal_raw[channel][i]:.3f}", + f"{result.signal[channel][i]:.3f}", + f"{result.normalized[channel][i]:.6f}", + ]) + return png_path, csv_path + + +def top_peak_indices(result: SpectrumResult, channel: str, count: int) -> np.ndarray: + """Strongest peaks, then returned in left-to-right order for display.""" + + idx = result.peaks[channel] + if idx.size == 0 or count <= 0: + return np.empty(0, dtype=int) + heights = result.normalized[channel][idx] + chosen = idx[np.argsort(heights)[::-1][:count]] + return chosen[np.argsort(result.x_plot[channel][chosen])] + + +def format_peak_values(result: SpectrumResult, channel: str, count: int = 1) -> str: + idx = top_peak_indices(result, channel, count) + if not idx.size: + return "—" + return ", ".join(f"{result.x_plot[channel][i]:.1f}" for i in idx) + + +# --------------------------------------------------------------------------- +# Main loop +# --------------------------------------------------------------------------- + + +def make_calibration(args: argparse.Namespace) -> Optional[Calibration]: + if not args.calibrate: + return None + pairs: list[tuple[float, float]] = [] + for token in args.calibrate.split(","): + token = token.strip() + if not token: + continue + if ":" not in token: + raise ValueError(f"bad calibration token {token!r}; expected PX:NM") + px, nm = token.split(":", 1) + pairs.append((float(px), float(nm))) + if len(pairs) < 2: + raise ValueError("--calibrate needs at least two PX:NM pairs") + return Calibration(pairs, args.poly_order) + + +def _meta_number(meta: dict, key: str, default: float = 0.0) -> float: + value = meta.get(key, default) + try: + return float(value) if value is not None else default + except (TypeError, ValueError): + return default + + +def status_for(result: SpectrumResult, fps: float) -> str: + meta = result.metadata + exposure = _meta_number(meta, "ExposureTime") + gain = _meta_number(meta, "AnalogueGain") + fps_text = f"{fps:4.1f}" if fps > 0 else " --" + peaks = { + channel: format_peak_values(result, channel, 3) for channel in CHANNELS + } + text = ( + f"{result.raw_format} {result.full_size[0]}x{result.full_size[1]} " + f"{result.bit_depth}-bit buf\n" + f"ROI {result.roi.width}x{result.roi.height} @ {result.roi.text()} " + f"{fps_text} fps\n" + f"exp {exposure / 1000.0:7.2f} ms gain {gain:4.1f} black {result.black:.0f} " + f"sat {result.saturated_fraction * 100.0:.2f}%\n" + f"peaks [{result.x_unit}]\n" + f" R {peaks['R']}\n" + f" G {peaks['G']}\n" + f" B {peaks['B']}" + ) + return text + + +def run(args: argparse.Namespace) -> int: + if args.list_modes: + try: + list_camera_modes() + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + # Import Matplotlib only after the backend has been chosen. + import matplotlib + if args.headless: + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + try: + calibration = make_calibration(args) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + try: + manual_roi = parse_roi(args.roi, args.width, args.height) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + source = None + dashboard = None + state: dict = {"roi": manual_roi, "result": None} + saved_on_exit = False + + try: + if args.demo: + source = DemoSource(args) + if args.headless and args.frames == 0: + args.frames = 1 + else: + source = CameraSource(args) + + if manual_roi is not None: + # Re-apply fractions/pixels to the actual raw size (normally the + # same as the requested size, but libcamera is the final authority). + manual_roi = parse_roi(args.roi, *source.full_size) + state["roi"] = manual_roi + + print(f"source : {source.description}") + print(f"raw stream : {source.raw_format} {source.full_size[0]}x{source.full_size[1]}") + print(f"stride : {getattr(source, 'stride', 'n/a')}") + print(f"Bayer order : {source.pattern} (decoded {source.bit_depth}-bit buffer)") + if not args.demo and "PISP_COMP" in str(source.raw_format).upper(): + print("note : PC1B/PiSP compressed raw is being decompressed in software") + + source.start() + # Let AE/AWB and the sensor settle; also fills the first buffers. + for _ in range(5): + try: + source.read() + except Exception as exc: + print(f"warning: warm-up read failed: {exc}", file=sys.stderr) + break + + roi = state["roi"] + period = 1.0 / max(args.fps, 0.1) + fps_est = 0.0 + previous_capture = None + frame_count = 0 + capture_error = False + waterfall_rows = max(2, int(args.waterfall_rows)) + waterfall = deque(maxlen=waterfall_rows) + last_width = None + + def on_roi(new_roi: ROI) -> None: + state["roi"] = new_roi + waterfall.clear() + print(f"ROI changed by mouse: {new_roi.text()} {new_roi.width}x{new_roi.height}") + + def do_save(prefix: Optional[str] = None) -> bool: + if dashboard is None or state.get("result") is None: + return False + target = prefix or args.save_prefix + if not target: + target = f"hq_spectrum_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + try: + png, csv_path = dashboard.save(target) + except Exception as exc: + print(f"save failed: {exc}", file=sys.stderr) + return False + print(f"saved: {png}") + print(f"saved: {csv_path}") + return True + + while True: + loop_start = time.perf_counter() + try: + raw, metadata = source.read() + except KeyboardInterrupt: + raise + except Exception as exc: + print(f"capture error: {exc}", file=sys.stderr) + capture_error = True + break + + # Pick up a ROI selected by the mouse in the dashboard. + roi = state["roi"] + if roi is None: + preview = make_preview(raw, source.pattern) + roi = detect_auto_roi(preview, *source.full_size) + if roi is None: + width, height = source.full_size + roi = make_roi(0, 0.30 * height, width, 0.70 * height, width, height) + print("auto ROI: no spectral band found, using full-width central strip") + else: + print(f"auto ROI : {roi.text()} {roi.width}x{roi.height}") + state["roi"] = roi + + result = process_frame( + raw, metadata, roi, source.pattern, args, calibration, + source.full_size, source.raw_format, source.bit_depth, + ) + state["result"] = result + frame_count += 1 + + if previous_capture is not None: + dt = time.perf_counter() - previous_capture + if dt > 0: + inst = 1.0 / dt + fps_est = inst if fps_est == 0.0 else 0.90 * fps_est + 0.10 * inst + previous_capture = time.perf_counter() + + norm_g = result.normalized["G"] + if last_width is not None and last_width != norm_g.size: + waterfall.clear() + last_width = norm_g.size + waterfall.append(norm_g.copy()) + wf = np.asarray(waterfall, dtype=np.float32) + if wf.size == 0: + wf = np.zeros((1, 2), dtype=np.float32) + + if dashboard is None: + dashboard = Dashboard( + plt, args, source.full_size, source.description, + result.calibration_text, on_roi=on_roi, + ) + dashboard.save_callback = do_save + + dashboard.update(result, wf, fps_est, status_for(result, fps_est)) + if args.verbose and frame_count == 1: + print(f"raw range : {int(raw.min())}..{int(raw.max())} " + f"p1={float(np.percentile(raw, 1)):.1f} p99={float(np.percentile(raw, 99)):.1f}") + print(f"black/full : {result.black:.1f}/{result.full_scale:.1f}") + print(f"calibration : {result.calibration_text}") + + if args.frames and frame_count >= args.frames: + break + if not args.headless and (dashboard.quit_requested or + not plt.fignum_exists(dashboard.fig.number)): + break + + elapsed = time.perf_counter() - loop_start + dashboard.pump(period - elapsed, args.headless) + + if args.save_prefix and dashboard is not None and state.get("result") is not None: + saved_on_exit = do_save(args.save_prefix) + elif args.headless and dashboard is not None and state.get("result") is not None and not args.demo: + print("headless run finished; use --save-prefix to write the dashboard and CSV") + + return 1 if capture_error else 0 + except (RuntimeError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("\ninterrupted") + return 130 + except Exception as exc: + if args.verbose: + traceback.print_exc() + print(f"error: {exc}", file=sys.stderr) + return 1 + finally: + # Save before closing the Matplotlib canvas, otherwise figure.savefig() + # may be operating on a torn-down backend. + if args.save_prefix and dashboard is not None and state.get("result") is not None and not saved_on_exit: + try: + png, csv_path = dashboard.save(args.save_prefix) + print(f"saved: {png}") + print(f"saved: {csv_path}") + except Exception as exc: + print(f"warning: could not save final snapshot: {exc}", file=sys.stderr) + if dashboard is not None: + try: + dashboard.close() + except Exception: + pass + if source is not None: + try: + source.close() + except Exception: + pass + + +def main(argv: Optional[list[str]] = None) -> int: + args = parse_args(argv) + return run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/hq_average.py b/hq_average.py new file mode 100644 index 0000000..d9e68c5 --- /dev/null +++ b/hq_average.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Average hq_record recording directories using the existing player's decoder.""" +from __future__ import annotations +import argparse +import json +import os +from pathlib import Path +import sys +import time +import uuid + +import numpy as np + +try: # shared core lives beside this script + import spectro_core as core +except ImportError as exc: # pragma: no cover - placement error + raise SystemExit('hq_average.py needs spectro_core.py in the same directory ' + '(or on sys.path)') from exc + + +def load_player(source, module_dir=None): + """Decoder module: --module-dir, then this folder, then the project above.""" + for directory in ([Path(module_dir)] if module_dir + else [Path(__file__).resolve().parent, Path(source).parent.parent]): + if (directory / 'hq_spectrum.py').is_file(): + return core.load_hq(directory) + return core.load_hq() + + +def write_json(path, value): + core.write_json(path, value) + + +def average(source, output, group=0, drop_last=False, module_dir=None): + source, output = Path(source).resolve(), Path(output).resolve() + if group < 0: + raise ValueError('group must be >= 0') + if source == output or source in output.parents or output in source.parents: + raise ValueError('Input and output must be separate, non-nested directories') + if output.exists(): + raise ValueError(f'Output already exists: {output}') + meta = json.loads((source / 'meta.json').read_text(encoding='utf-8')) + if meta.get('kind') != 'hq_raw_recording' or meta.get('format_version') != 1: + raise ValueError('Expected hq_raw_recording format_version 1') + w, h, stride = (int(meta[k]) for k in ('width', 'height', 'stride')) + fb = int(meta['frame_bytes']) + if min(w, h, stride) <= 0 or fb != h * stride: + raise ValueError('Invalid dimensions/stride/frame_bytes') + size = (source / 'frames.bin').stat().st_size + count, remainder = divmod(size, fb) + if remainder or not count or int(meta.get('frames', count)) != count: + raise ValueError('Truncated recording or frame count mismatch; repair source separately') + if not meta.get('completed'): + raise ValueError('Source recording is not marked completed') + records = [] + metadata_path = source / 'frames.jsonl' + if metadata_path.exists(): + for i, line in enumerate(metadata_path.read_text(encoding='utf-8').splitlines()): + row = json.loads(line) + if not isinstance(row, dict): + raise ValueError(f'Invalid metadata row {i}') + nested = row.pop('metadata', None) + if isinstance(nested, dict): + row = {**nested, **row} + for key, expected in [('i', i), ('offset', i * fb), ('bytes', fb)]: + if key in row and row[key] != expected: + raise ValueError(f'Incorrect metadata {key} in row {i}') + records.append(row) + if len(records) != count: + raise ValueError('frames.jsonl count mismatch') + else: + records = [{} for _ in range(count)] + # Exposure/gain changes are retained as ranges in each averaged frame's provenance. + player = load_player(source, module_dir) + pattern = str(meta.get('pattern') or player.pattern_from_format(meta['raw_format'])) + if pattern not in ('BGGR', 'RGGB', 'GBRG', 'GRBG'): + raise ValueError('Unsupported Bayer pattern') + config = {'size': (w, h), 'stride': stride} + n = group or count + groups = [(a, min(a + n, count)) for a in range(0, count, n) + if not (drop_last and min(a + n, count) - a < n)] + if not groups: + raise ValueError('No output frames; reduce group size or omit --drop-last') + output.parent.mkdir(parents=True, exist_ok=True) + temp = output.parent / (output.name + '.partial-' + uuid.uuid4().hex[:8]) + temp.mkdir() + started = time.monotonic() + try: + depth = None + out_fb = w * h * 2 + with (source / 'frames.bin').open('rb') as inp, (temp / 'frames.bin').open('wb') as raw_out, (temp / 'frames.jsonl').open('w', encoding='utf-8') as mout: + for oi, (a, b) in enumerate(groups): + acc = np.zeros((h, w), dtype=np.float64) + inp.seek(a * fb) + for index in range(a, b): + data = inp.read(fb) + if len(data) != fb: + raise ValueError('Input changed or was truncated while reading') + image, bits = player.decode_raw(np.frombuffer(data, dtype=np.uint8).reshape(h, stride), config, meta['raw_format']) + if depth is not None and depth != bits: + raise ValueError('Decoded bit depth changed') + depth = bits + acc += image + image = np.clip(np.rint(acc / (b - a)), 0, (1 << depth) - 1).astype(' recordings/hq_YYYYmmdd_HHMMSS/ + hq_spectrum.py --input REC_DIR -> replay / analysis + +The recorder does not import Matplotlib and does not compute spectra. It only +pulls the Picamera2 RAW stream, queues each raw byte buffer, and writes it to +`frames.bin`. A background writer thread keeps disk I/O away from the camera +capture loop. + +Recording layout +---------------- + REC_DIR/ + meta.json camera/format/stride/frame count + frames.bin raw byte buffers, one after another, no decoding + frames.jsonl one line of camera metadata per frame + +For the usual Pi 5 HQ Camera configuration the raw stream is +`BGGR_PISP_COMP1` (PC1B). Each frame is height*stride bytes; at 2028x1520 +that is typically about 3.1 MB/frame. At 15 fps that is about 47 MB/s, so a +USB SSD/NVMe is preferable to a slow SD card. + +Examples +-------- + # list modes first if needed + python3 hq_record.py --list-modes + + # 10 seconds, 15 fps, manual exposure copied from rpicam-hello + python3 hq_record.py --duration 10 --fps 15 \ + --exposure 33013 --gain 16 \ + --record-dir recordings/hg_lamp + + # unlimited until Ctrl+C + python3 hq_record.py --duration 0 --fps 15 \ + --record-dir recordings/hg_long + + # analyze later (on the Pi or on a PC) + python3 hq_spectrum.py --input recordings/hg_lamp --fps 15 + +The recorder is deliberately independent of the analyzer: a slow Matplotlib +redraw can no longer cause dropped frames or change the exposure timing. +""" + +from __future__ import annotations + +import argparse +import json +import queue +import shutil +import sys +import threading +import time +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Optional + +import numpy as np + +try: + from hq_spectrum import CameraSource, list_camera_modes +except ImportError as exc: # pragma: no cover - import error path + raise SystemExit( + "hq_record.py must sit next to hq_spectrum.py " + "(run it from the hq_spectrum directory)." + ) from exc + + +class RawWriter(threading.Thread): + """Write queued raw buffers and metadata lines in the background.""" + + def __init__(self, frames_path: Path, metadata_path: Path, frame_bytes: int, + queue_size: int): + super().__init__(name="hq-raw-writer", daemon=False) + self.frames_path = frames_path + self.metadata_path = metadata_path + self.frame_bytes = frame_bytes + self.queue: queue.Queue = queue.Queue(maxsize=max(2, queue_size)) + self.frames_written = 0 + self.error: Optional[BaseException] = None + self.finished = threading.Event() + + def put(self, raw: np.ndarray, metadata: dict, host_time_ns: int) -> None: + if self.error is not None: + raise RuntimeError(f"writer failed: {self.error}") + raw = np.ascontiguousarray(raw) + if raw.dtype != np.uint8 or raw.nbytes != self.frame_bytes: + raise ValueError( + f"unexpected raw buffer {raw.shape} {raw.dtype} {raw.nbytes}B; " + f"expected {self.frame_bytes}B uint8" + ) + item = (raw, dict(metadata), int(host_time_ns)) + while True: + try: + self.queue.put(item, timeout=0.5) + return + except queue.Full: + if self.error is not None: + raise RuntimeError(f"writer failed: {self.error}") + + def run(self) -> None: + try: + with open(self.frames_path, "wb", buffering=4 * 1024 * 1024) as frames, \ + open(self.metadata_path, "w", encoding="utf-8", buffering=1024 * 1024) as meta_out: + while True: + item = self.queue.get() + if item is None: + break + raw, camera_meta, host_time_ns = item + offset = self.frames_written * self.frame_bytes + record = dict(camera_meta) + record.update({ + "i": self.frames_written, + "offset": offset, + "bytes": raw.nbytes, + "host_time_ns": host_time_ns, + }) + frames.write(memoryview(raw).cast("B")) + meta_out.write(json.dumps(record, default=str, separators=(",", ":")) + "\n") + self.frames_written += 1 + if self.frames_written % 30 == 0: + frames.flush() + meta_out.flush() + frames.flush() + meta_out.flush() + self.finished.set() + except BaseException as exc: # noqa: BLE001 - report writer failure to main + self.error = exc + self.finished.set() + + def stop(self) -> None: + if self.is_alive(): + try: + self.queue.put(None, timeout=2.0) + except queue.Full: + pass + self.join(timeout=30.0) + if self.is_alive(): + raise RuntimeError("writer thread did not stop within 30 seconds") + if self.error is not None: + raise RuntimeError(f"writer failed: {self.error}") + + +def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Record Raspberry Pi HQ Camera RAW frames for later spectrum analysis.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--width", type=int, default=2028, help="raw stream width") + parser.add_argument("--height", type=int, default=1520, help="raw stream height") + parser.add_argument("--fps", type=float, default=15.0, help="requested frame rate") + parser.add_argument("--duration", type=float, default=10.0, + help="seconds to record; 0 means until Ctrl+C") + parser.add_argument("--frames", type=int, default=0, + help="optional frame limit; 0 means no limit") + parser.add_argument("--record-dir", default=None, + help="output directory (default recordings/hq_YYYYmmdd_HHMMSS)") + parser.add_argument("--queue-size", type=int, default=8, + help="raw frames buffered in RAM while the writer catches up") + parser.add_argument("--exposure", type=float, default=None, + help="manual exposure time in microseconds") + parser.add_argument("--gain", type=float, default=None, help="manual analogue gain") + parser.add_argument("--raw-format", default=None, + help="optional raw format, e.g. SBGGR12 or BGGR_PISP_COMP1") + parser.add_argument("--list-modes", action="store_true", + help="list camera raw modes and exit") + args = parser.parse_args(argv) + if args.width < 64 or args.height < 64: + parser.error("width and height must be at least 64") + if args.fps <= 0: + parser.error("--fps must be positive") + if args.duration < 0: + parser.error("--duration must be >= 0") + if args.frames < 0: + parser.error("--frames must be >= 0") + if args.queue_size < 2: + parser.error("--queue-size must be >= 2") + return args + + +def write_json(path: Path, data: dict) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + with open(tmp, "w", encoding="utf-8") as handle: + json.dump(data, handle, indent=2, default=str) + handle.write("\n") + tmp.replace(path) + + +def prepare_directory(path: Path) -> None: + if path.exists(): + if not path.is_dir(): + raise RuntimeError(f"{path} exists and is not a directory") + if any(path.iterdir()): + raise RuntimeError( + f"{path} is not empty; choose a new --record-dir " + f"(the recorder never overwrites old data)" + ) + else: + path.mkdir(parents=True, exist_ok=True) + + +def main(argv: Optional[list[str]] = None) -> int: + args = parse_args(argv) + if args.list_modes: + list_camera_modes() + return 0 + + cam_args = SimpleNamespace( + width=args.width, + height=args.height, + fps=args.fps, + exposure=args.exposure, + gain=args.gain, + raw_format=args.raw_format, + ) + + source = None + writer = None + meta = None + record_dir = None + enqueued = 0 + started = None + exit_code = 0 + + try: + source = CameraSource(cam_args) + print(f"camera : {source.description}") + print(f"raw stream : {source.raw_format} " + f"{source.full_size[0]}x{source.full_size[1]}") + print(f"stride : {source.stride}") + print(f"Bayer : {source.pattern} decoded {source.bit_depth}-bit buffer") + print(f"target rate : {args.fps:g} fps") + if "PISP_COMP" in source.raw_format.upper(): + print("note : recording the compressed PC1B buffer; no decode on the Pi") + + source.start() + last_meta = {} + # Discard warm-up frames; the recording should start with settled AE. + for _ in range(5): + _, last_meta = source.read_raw() + + # Get one sample only to validate the actual buffer size. + sample_raw, sample_meta = source.read_raw() + last_meta = sample_meta + if sample_raw.dtype != np.uint8: + raise RuntimeError(f"unexpected raw dtype {sample_raw.dtype}; expected uint8 bytes") + frame_bytes = int(sample_raw.nbytes) + expected_bytes = source.stride * source.full_size[1] + if frame_bytes != expected_bytes: + raise RuntimeError( + f"raw buffer is {frame_bytes} B but stride*height is {expected_bytes} B; " + f"refusing to write a recording that the analyzer would misread" + ) + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + record_dir = Path(args.record_dir).expanduser() if args.record_dir else \ + Path("recordings") / f"hq_{stamp}" + prepare_directory(record_dir) + frames_path = record_dir / "frames.bin" + metadata_path = record_dir / "frames.jsonl" + meta_path = record_dir / "meta.json" + + data_rate = frame_bytes * args.fps + free = shutil.disk_usage(record_dir).free + print(f"output : {record_dir}") + print(f"frame size : {frame_bytes / 1e6:.3f} MB") + print(f"data rate : {data_rate / 1e6:.1f} MB/s") + print(f"free space : {free / 1e9:.2f} GB") + planned_frames = None + if args.frames: + planned_frames = args.frames + if args.duration: + duration_frames = int(args.duration * args.fps) + planned_frames = duration_frames if planned_frames is None else min(planned_frames, duration_frames) + if planned_frames is not None: + needed = frame_bytes * planned_frames * 1.05 + print(f"planned size : {needed / 1e9:.2f} GB for ~{planned_frames} frames") + if needed > free: + raise RuntimeError( + f"not enough free space: need ~{needed / 1e9:.2f} GB, " + f"have {free / 1e9:.2f} GB" + ) + + meta = { + "format_version": 1, + "kind": "hq_raw_recording", + "created_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "camera": source.description, + "raw_format": source.raw_format, + "width": source.full_size[0], + "height": source.full_size[1], + "stride": source.stride, + "frame_bytes": frame_bytes, + "storage_dtype": "uint8", + "pattern": source.pattern, + "bit_depth": source.bit_depth, + "fps_requested": args.fps, + "exposure_us": args.exposure, + "gain": args.gain, + "black_level": last_meta.get("BlackLevel"), + "duration_limit_s": args.duration, + "frame_limit": args.frames, + "frames": 0, + "completed": False, + } + write_json(meta_path, meta) + + writer = RawWriter(frames_path, metadata_path, frame_bytes, args.queue_size) + writer.start() + # The first sample was taken to validate the format, so it is also the + # first frame of the recording. + writer.put(sample_raw, last_meta, time.time_ns()) + enqueued = 1 + + started = time.monotonic() + last_report = started + while True: + if args.frames and enqueued >= args.frames: + break + elapsed = time.monotonic() - started + if args.duration and elapsed >= args.duration: + break + raw, camera_meta = source.read_raw() + writer.put(raw, camera_meta, time.time_ns()) + enqueued += 1 + now = time.monotonic() + if now - last_report >= 1.0: + written = writer.frames_written + rate = enqueued / max(now - started, 1e-6) + print(f"recording : {enqueued} frames captured, {written} written, " + f"{rate:.1f} fps") + last_report = now + + elapsed = time.monotonic() - started + writer.stop() + writer_error = writer.error + writer = None + if writer_error is not None: + raise RuntimeError(f"writer failed: {writer_error}") + + meta.update({ + "frames": int(enqueued), + "duration_s": round(elapsed, 3), + "average_capture_fps": round(enqueued / max(elapsed, 1e-6), 3), + "completed": True, + "ended_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "frames_file": "frames.bin", + "metadata_file": "frames.jsonl", + }) + write_json(meta_path, meta) + size = frames_path.stat().st_size + print(f"done : {enqueued} frames, {size / 1e9:.2f} GB, " + f"{enqueued / max(elapsed, 1e-6):.1f} fps") + print(f"analyze with : python3 hq_spectrum.py --input \"{record_dir}\"") + return 0 + except KeyboardInterrupt: + print("\ninterrupted; finalizing recording", file=sys.stderr) + exit_code = 130 + return exit_code + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + exit_code = 1 + return exit_code + finally: + if source is not None: + try: + source.close() + except Exception: + pass + if writer is not None: + try: + writer.stop() + except Exception as exc: + print(f"writer shutdown error: {exc}", file=sys.stderr) + exit_code = 1 + if meta is not None and record_dir is not None: + try: + meta.update({ + "frames": int(writer.frames_written) if writer is not None else int(enqueued), + "completed": exit_code == 0 and meta.get("completed", False), + "ended_at": datetime.now().astimezone().isoformat(timespec="seconds"), + }) + write_json(record_dir / "meta.json", meta) + except Exception as exc: + print(f"warning: could not finalize meta.json: {exc}", file=sys.stderr) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/hq_spectrum.py b/hq_spectrum.py new file mode 100644 index 0000000..438839e --- /dev/null +++ b/hq_spectrum.py @@ -0,0 +1,2409 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +hq_spectrum.py +============== +Raspberry Pi HQ Camera (IMX477) 12-bit RAW, live R/G/B spectral monitor. + +The program follows the stream selected by rpicam-hello for the HQ camera: + + sensor mode : 2028x1520 (12 bit, packed) + raw stream : BGGR_PISP_COMP1 / SBGGR12_1X12 + sensor Bayer order : BGGR + CFE format : PC1B + +It captures the raw stream, *not* the ISP preview. The four Bayer sites are +split into B, Gb/Gr and R planes, a rectangular ROI is reduced along the slit +direction, and the three one-dimensional spectra are drawn with Matplotlib. + +The Pi 5 path delivers PiSP-compressed PC1B data (format BGGR_PISP_COMP1). +That buffer is expanded with the same decompressor that Picamera2 uses for +DNG output. On a Pi 4 (or when an uncompressed raw stream is selected), the +SBGGR12 16-bit-container path is used directly. Both paths are supported. + +Typical use on the Pi +--------------------- + # dependencies (Raspberry Pi OS, use the system Python) + sudo apt install python3-picamera2 python3-numpy python3-matplotlib python3-tk + + # list the raw modes your camera exposes + python3 hq_spectrum.py --list-modes + + # live display, auto ROI, manual exposure taken from rpicam-hello + python3 hq_spectrum.py --exposure 33013 --gain 16 + + # fixed ROI in full-resolution sensor pixels: x0,y0,x1,y1 + python3 hq_spectrum.py --roi 300,420,1750,980 --exposure 33013 --gain 16 + + # ROI as fractions of the frame + python3 hq_spectrum.py --roi 0.12,0.30,0.85,0.65 + + # two-point wavelength calibration made from known lamp lines + python3 hq_spectrum.py --calibrate 420:405,920:546 --exposure 33013 --gain 16 + + # replay/analyze a recording made by hq_record.py + python3 hq_spectrum.py --input recordings/hg_lamp --fps 15 + +Keys in the live window +----------------------- + s : save the current figure as PNG and the profiles as CSV + q : quit + +Draw a new box with the mouse to change the orange/yellow ROI. + +Demo (works on a normal PC, no camera required) +----------------------------------------------- + python3 hq_spectrum.py --demo --frames 1 --headless --save-prefix demo + +Notes +----- +* The x axis is sensor column pixels until a wavelength calibration is given. + The R/G/B curves are broad CFA responses, not narrow-band spectrometer + channels; use them to see line positions and relative changes. +* No demosaicing or colour processing is applied. That is intentional: it + keeps the spectral lines from being interpolated across neighbouring pixels. +* Auto exposure is convenient for finding a signal but makes intensity ratios + unstable. For measurements use fixed ExposureTime and AnalogueGain. +* The preview is auto-stretched for visibility; the curves show black-level + corrected raw ADU values. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import sys +import time +import traceback +from collections import deque +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional + +import numpy as np + +CHANNELS = ("R", "G", "B") +CHANNEL_COLORS = {"R": "#e43d45", "G": "#21a45a", "B": "#2f6fdd"} +BAYER_LAYOUT = { + "RGGB": (("R", 0, 0), ("G", 0, 1), ("G", 1, 0), ("B", 1, 1)), + "BGGR": (("B", 0, 0), ("G", 0, 1), ("G", 1, 0), ("R", 1, 1)), + "GRBG": (("G", 0, 0), ("R", 0, 1), ("B", 1, 0), ("G", 1, 1)), + "GBRG": (("G", 0, 0), ("B", 0, 1), ("R", 1, 0), ("G", 1, 1)), +} + + +# --------------------------------------------------------------------------- +# Small data containers +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ROI: + """A half-open ROI in full-resolution sensor coordinates: [x0,x1) x [y0,y1).""" + + x0: int + y0: int + x1: int + y1: int + + @property + def width(self) -> int: + return self.x1 - self.x0 + + @property + def height(self) -> int: + return self.y1 - self.y0 + + def text(self) -> str: + return f"({self.x0},{self.y0})-({self.x1},{self.y1})" + + +@dataclass +class SpectrumResult: + roi: ROI + x_px: dict[str, np.ndarray] + x_plot: dict[str, np.ndarray] + signal: dict[str, np.ndarray] + signal_raw: dict[str, np.ndarray] + normalized: dict[str, np.ndarray] + peaks: dict[str, np.ndarray] + preview: np.ndarray + full_size: tuple[int, int] + raw_format: str + bit_depth: int + black: float + full_scale: float + saturated_fraction: float + metadata: dict + x_unit: str + calibration_text: str + timestamp: float + + +@dataclass +class PlayerState: + """Transport state for the replay player.""" + + total_frames: int + fps: float + current: int = 0 + playing: bool = False + loop: bool = True + speed: float = 1.0 + pending_seek: Optional[int] = None + needs_reschedule: bool = False + + def period(self) -> float: + return 1.0 / max(self.fps * self.speed, 0.01) + + def request(self, frame: int, pause: bool = True) -> None: + self.pending_seek = max(0, min(self.total_frames - 1, int(frame))) + if pause: + self.playing = False + + +class Calibration: + """Polynomial pixel -> wavelength calibration.""" + + def __init__(self, pairs: list[tuple[float, float]], order: int = 1): + if len(pairs) < 2: + raise ValueError("calibration needs at least two pixel:wavelength pairs") + arr = np.asarray(sorted(pairs), dtype=float) + if np.any(np.diff(arr[:, 0]) <= 0): + raise ValueError("calibration pixel positions must be distinct") + self.pixels = arr[:, 0] + self.wavelengths = arr[:, 1] + degree = max(1, min(int(order), len(pairs) - 1)) + self.coeff = np.polyfit(self.pixels, self.wavelengths, degree) + pred = np.polyval(self.coeff, self.pixels) + self.rms_nm = float(np.sqrt(np.mean((pred - self.wavelengths) ** 2))) + + def __call__(self, x: np.ndarray) -> np.ndarray: + return np.polyval(self.coeff, np.asarray(x, dtype=float)) + + def describe(self) -> str: + terms = [] + degree = len(self.coeff) - 1 + for i, c in enumerate(self.coeff): + power = degree - i + terms.append(f"{c:.6g}*x^{power}" if power else f"{c:.6g}") + return "lambda(nm) = " + " + ".join(terms) + f"; rms={self.rms_nm:.2f} nm" + + +# --------------------------------------------------------------------------- +# Bayer / raw helpers +# --------------------------------------------------------------------------- + + +def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Live R/G/B spectral analysis of Raspberry Pi HQ Camera RAW frames.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--width", type=int, default=2028, help="raw stream width") + parser.add_argument("--height", type=int, default=1520, help="raw stream height") + parser.add_argument( + "--fps", type=float, default=5.0, + help="capture/display rate for live mode; the GUI player uses the recording's fps", + ) + parser.add_argument( + "--roi", + default="auto", + help="'auto' or x0,y0,x1,y1 in sensor pixels; values <= 1 are treated as fractions", + ) + parser.add_argument( + "--calibrate", + default=None, + metavar="PX:NM[,PX:NM...]", + help="pixel:wavelength pairs, e.g. 420:405,920:546", + ) + parser.add_argument("--poly-order", type=int, default=1, help="polynomial order for calibration") + parser.add_argument( + "--smooth", type=int, default=3, + help="moving-average width in sensor pixels (channel samples are 2 px apart)", + ) + parser.add_argument( + "--reduce", + choices=("mean", "median"), + default="mean", + help="row reduction inside the ROI", + ) + parser.add_argument("--black", type=float, default=None, help="black level override in ADU") + parser.add_argument("--full-scale", type=float, default=None, help="white level override in ADU") + parser.add_argument("--exposure", type=float, default=None, help="manual exposure time in microseconds") + parser.add_argument("--gain", type=float, default=None, help="manual analogue gain") + parser.add_argument( + "--raw-format", + default=None, + help="optional Picamera2 raw format, e.g. SBGGR12 or BGGR_PISP_COMP1", + ) + parser.add_argument("--peak-prominence", type=float, default=0.08, help="normalized peak prominence") + parser.add_argument("--peak-height", type=float, default=0.035, help="minimum normalized peak height") + parser.add_argument("--peak-distance", type=float, default=12.0, help="minimum peak spacing in sensor pixels") + parser.add_argument("--max-peaks", type=int, default=5, help="maximum peaks detected per channel") + parser.add_argument("--waterfall-rows", type=int, default=240, help="number of frames in the waterfall") + parser.add_argument("--demo", action="store_true", help="use a synthetic 12-bit BGGR frame") + parser.add_argument( + "--input", + default=None, + metavar="REC_DIR", + help="replay/analyze a recording made by hq_record.py", + ) + parser.add_argument( + "--no-loop", + action="store_true", + help="with --input: stop at the end instead of looping playback", + ) + parser.add_argument( + "--autoplay", + action="store_true", + help="with --input: start playing immediately instead of paused", + ) + parser.add_argument("--headless", action="store_true", help="do not open a window (Agg backend)") + parser.add_argument( + "--no-blit", + action="store_true", + help="redraw the whole figure per frame instead of blitting the changed artists", + ) + parser.add_argument( + "--frames", type=int, default=0, + help="stop after N frames for live/headless mode; ignored by the GUI player", + ) + parser.add_argument("--save-prefix", default=None, help="save final dashboard/CSV using this path prefix") + parser.add_argument("--no-selector", action="store_true", help="disable mouse ROI selection") + parser.add_argument("--list-modes", action="store_true", help="list camera raw modes and exit") + parser.add_argument("--verbose", action="store_true", help="print extra raw/ROI diagnostics") + args = parser.parse_args(argv) + + if args.width < 64 or args.height < 64: + parser.error("width and height must be at least 64") + if args.fps <= 0: + parser.error("--fps must be positive") + if args.max_peaks < 0: + parser.error("--max-peaks must be >= 0") + if args.waterfall_rows < 2: + parser.error("--waterfall-rows must be >= 2") + if args.frames < 0: + parser.error("--frames must be >= 0") + if args.demo and args.input: + parser.error("--demo and --input are mutually exclusive") + return args + + +def pattern_from_format(fmt: str) -> str: + match = re.search(r"([RGB]{4})", str(fmt).upper()) + if not match or match.group(1) not in BAYER_LAYOUT: + raise ValueError(f"cannot determine Bayer order from raw format {fmt!r}") + return match.group(1) + + +def bit_depth_from_format(fmt: str) -> int: + if "PISP_COMP" in str(fmt).upper(): + return 16 + match = re.search(r"([RGB]{4})(\d+)", str(fmt).upper()) + return int(match.group(2)) if match else 16 + + +def make_roi(x0: float, y0: float, x1: float, y1: float, width: int, height: int) -> ROI: + """Clamp and align a ROI to even sensor coordinates (needed by the Bayer grid).""" + + def edge(value: float, upper: int) -> int: + return int(round(max(0.0, min(float(upper), float(value))))) + + xa, xb = sorted((edge(x0, width), edge(x1, width))) + ya, yb = sorted((edge(y0, height), edge(y1, height))) + width_max = width - (width % 2) + height_max = height - (height % 2) + + xa = min(xa, width_max - 2) + ya = min(ya, height_max - 2) + xa -= xa % 2 + ya -= ya % 2 + xb = min(xb, width_max) + yb = min(yb, height_max) + xb = max(xa + 2, xb - ((xb - xa) % 2)) + yb = max(ya + 2, yb - ((yb - ya) % 2)) + return ROI(int(xa), int(ya), int(xb), int(yb)) + + +def parse_roi(value: str, width: int, height: int) -> Optional[ROI]: + value = value.strip().lower() + if value == "auto": + return None + parts = [p.strip() for p in value.split(",")] + if len(parts) != 4: + raise ValueError("--roi must be 'auto' or x0,y0,x1,y1") + try: + numbers = [float(p) for p in parts] + except ValueError as exc: + raise ValueError("ROI values must be numbers") from exc + if not all(np.isfinite(v) for v in numbers): + raise ValueError("ROI values must be finite") + if all(0.0 <= v <= 1.0 for v in numbers): + numbers = [numbers[0] * width, numbers[1] * height, numbers[2] * width, numbers[3] * height] + roi = make_roi(*numbers, width, height) + if roi.width < 24 or roi.height < 8: + raise ValueError("ROI is too small") + return roi + + +def decompress_pisp_comp1(array: np.ndarray) -> np.ndarray: + """Expand a PiSP PC1B/COMP1 raw buffer to a 2-D uint16 Bayer image. + + Adapted from Picamera2's helpers.decompress() (BSD-2-Clause, + Raspberry Pi Ltd.); see THIRD_PARTY_NOTICES.md. Keeping a local copy + makes the analyser work with older Picamera2 versions and avoids + depending on a helper method. + """ + + array = np.ascontiguousarray(array) + if array.dtype != np.uint8: + array = array.view(np.uint8) + if array.ndim != 2: + raise ValueError(f"unexpected PC1B buffer shape {array.shape}") + # PC1B works in 8-byte groups. Some libcamera versions report the + # compressed stride as the natural width (e.g. 2028) rather than a padded + # value, so zero-pad the tail of each row before expanding it. + if array.shape[1] % 8: + pad = (-array.shape[1]) % 8 + array = np.pad(array, ((0, 0), (0, pad)), mode="constant") + + offset = 2048 + words = array.view(np.int32) + words = words.reshape((words.shape[0], words.shape[1] // 2, 2)) + qmode = words & 3 + pix0 = (words >> 2) & 511 + pix1 = ((words >> 11) & 127) - 64 + pix2 = (words >> 18) & 127 + pix3 = (words >> 25) & 127 + + # qmode 0..2: four deltas around a 9-bit base value. + low = qmode * pix0 < 768 + q1 = np.copy(pix0) + q2 = pix1 + 448 + np.maximum(pix0, pix0 - pix1, where=low, out=q1) + np.maximum(pix0, pix0 + pix1, where=low, out=q2) + gain = np.right_shift(1536, qmode) + q0 = np.minimum(gain, np.maximum(q1 - 64, 0)) + pix2 + q3 = np.minimum(gain, np.maximum(q2 - 64, 0)) + pix3 + np.maximum(np.maximum(16 * q0, 32 * (q0 - 160)), 64 * qmode * q0, out=pix0) + np.maximum(np.maximum(16 * q1, 32 * (q1 - 160)), 64 * qmode * q1, out=pix1) + np.maximum(np.maximum(16 * q2, 32 * (q2 - 160)), 64 * qmode * q2, out=pix2) + np.maximum(np.maximum(16 * q3, 32 * (q3 - 160)), 64 * qmode * q3, out=pix3) + + # qmode 3 is a second, independent 16-bit coding of the same word that + # overrides the values above. On ordinary sensor data roughly one word in + # ten thousand uses it, so decode just those words: the full-frame version + # of this block costs ~30 % of the whole frame. + q3mode = qmode == 3 + if q3mode.any(): + index = np.flatnonzero(q3mode) + word = words.reshape(-1)[index] + high = (word >> 2) & 32767 + low = (word >> 17) & 32767 + codes = ((high & 15) + 16 * ((high >> 8) // 11), + (high >> 4) % 176, + (low & 15) + 16 * ((low >> 8) // 11), + (low >> 4) % 176) + for plane, code in zip((pix0, pix1, pix2, pix3), codes): + plane.reshape(-1)[index] = np.maximum(256 * code, 512 * (code - 47)) + + # Every 8-byte group holds four sites x two words, and the flattened + # memory order is group -> site -> word, so write the pixels straight into + # the uint16 result instead of stacking four full-frame int32 planes and + # converting the 12 MB copy afterwards. + res = np.empty(array.shape, dtype=np.uint16) + groups = res.reshape(res.shape[0], -1, 8) + for index, plane in enumerate((pix0, pix1, pix2, pix3)): + np.add(plane, offset, out=plane) + np.minimum(plane, 65535, out=plane) + groups[:, :, 2 * index] = plane[:, :, 0] + groups[:, :, 2 * index + 1] = plane[:, :, 1] + return res + + +def unpack_csi2(data: np.ndarray, bit_depth: int) -> np.ndarray: + """Unpack MIPI CSI-2 packed 10/12-bit rows (stride-sized raw rows).""" + + height, stride = data.shape + if bit_depth == 12: + if stride % 3: + raise ValueError(f"12-bit packed stride {stride} is not a multiple of 3") + width = stride // 3 * 2 + groups = np.ascontiguousarray(data).reshape(height, stride // 3, 3).astype(np.uint16) + out = np.empty((height, width), dtype=np.uint16) + out[:, 0::2] = groups[:, :, 0] | ((groups[:, :, 1] & 0x0F) << 8) + out[:, 1::2] = (groups[:, :, 1] >> 4) | (groups[:, :, 2] << 4) + return out + if bit_depth == 10: + if stride % 5: + raise ValueError(f"10-bit packed stride {stride} is not a multiple of 5") + width = stride // 5 * 4 + groups = np.ascontiguousarray(data).reshape(height, stride // 5, 5).astype(np.uint16) + out = np.empty((height, width), dtype=np.uint16) + out[:, 0::4] = groups[:, :, 0] | ((groups[:, :, 1] & 0x03) << 8) + out[:, 1::4] = (groups[:, :, 1] >> 2) | ((groups[:, :, 2] & 0x0F) << 6) + out[:, 2::4] = (groups[:, :, 2] >> 4) | ((groups[:, :, 3] & 0x3F) << 4) + out[:, 3::4] = (groups[:, :, 3] >> 6) | (groups[:, :, 4] << 2) + return out + raise ValueError(f"unsupported packed bit depth {bit_depth}") + + +def decode_raw(raw: np.ndarray, config: dict, fmt: str) -> tuple[np.ndarray, int]: + """Turn a Picamera2 raw buffer into a 2-D uint16 Bayer image. + + Picamera2's raw stream is normally a uint8 byte buffer of shape + (height, stride), including for 16-bit container formats. A few code + paths / future versions can hand over an already-unpacked uint16 array; + both representations are accepted here. + """ + + raw = np.asarray(raw) + width, height = (int(v) for v in config["size"]) + stride = int(config.get("stride") or (width * 2)) + fmt = str(fmt) + upper = fmt.upper() + + if raw.ndim == 1: + if raw.dtype == np.uint8: + raw = raw.reshape(height, stride) + elif raw.dtype.itemsize == 2: + raw = raw.reshape(height, -1) + else: + raise ValueError(f"unsupported raw dtype {raw.dtype}") + if raw.ndim != 2 or raw.shape[0] != height: + raise ValueError(f"raw buffer shape {raw.shape} does not match {height}x{width}") + + if "PISP_COMP1" in upper: + if raw.dtype != np.uint8: + raise ValueError("PC1B raw buffer must be a uint8 byte buffer") + image = decompress_pisp_comp1(raw) + if image.shape[1] < width: + raise ValueError(f"decompressed raw has only {image.shape[1]} columns, expected {width}") + return np.ascontiguousarray(image[:, :width]), 16 + + bit_depth = bit_depth_from_format(fmt) + mask = (1 << bit_depth) - 1 if bit_depth < 16 else None + + if raw.dtype.itemsize == 2: + # Already unpacked samples: shape is (height, stride_in_samples). + if raw.shape[1] < width: + raise ValueError( + f"unpacked raw has only {raw.shape[1]} columns, expected {width}; format={fmt!r}" + ) + image = raw[:, :width].astype(np.uint16, copy=True) + if mask is not None: + image &= mask + return image, bit_depth + + if "CSI2P" in upper or "_1X12" in upper or "_1X10" in upper: + # Stride is often padded (e.g. to 32 bytes). Unpack only the real row + # payload, otherwise the padding is interpreted as extra pixels. + row_bytes = (width * bit_depth + 7) // 8 + if raw.shape[1] < row_bytes: + raise ValueError(f"packed raw stride {raw.shape[1]} is too small for {width} pixels") + image = unpack_csi2(np.ascontiguousarray(raw[:, :row_bytes]), bit_depth) + if image.shape[1] < width: + raise ValueError(f"unpacked raw has only {image.shape[1]} columns, expected {width}") + return np.ascontiguousarray(image[:, :width]), bit_depth + + if bit_depth == 8: + if raw.shape[1] < width: + raise ValueError(f"raw stride {raw.shape[1]} is narrower than {width}") + return np.ascontiguousarray(raw[:, :width]).astype(np.uint16), 8 + + need = width * 2 + if raw.shape[1] < need: + raise ValueError( + f"raw stride {raw.shape[1]} is too small for {width} 16-bit samples; " + f"format={fmt!r}" + ) + image = np.ascontiguousarray(raw[:, :need]).view("= full: + black = black * full / 65535.0 + + return black, full + + +# --------------------------------------------------------------------------- +# Preview, ROI and spectrum processing +# --------------------------------------------------------------------------- + + +def extract_planes(raw: np.ndarray, pattern: str, roi: ROI) -> dict[str, np.ndarray]: + """Extract R, G and B planes from a BGGR/RGGB/... ROI. + + The two green sites are averaged. All returned planes have the same shape + because the ROI is aligned to even coordinates. + """ + + if roi.width < 2 or roi.height < 2: + raise ValueError(f"ROI is empty or too small: {roi.text()}") + crop = raw[roi.y0:roi.y1, roi.x0:roi.x1] + if crop.size == 0 or crop.shape[0] < 2 or crop.shape[1] < 2: + raise ValueError(f"ROI {roi.text()} does not overlap the raw frame") + layout = BAYER_LAYOUT[pattern] + by_color: dict[str, list[np.ndarray]] = {c: [] for c in CHANNELS} + for color, dy, dx in layout: + by_color[color].append(crop[dy::2, dx::2]) + + planes: dict[str, np.ndarray] = {} + for color, arrays in by_color.items(): + if color == "G" and len(arrays) == 2: + rows = min(a.shape[0] for a in arrays) + cols = min(a.shape[1] for a in arrays) + planes[color] = (arrays[0][:rows, :cols].astype(np.float32) + + arrays[1][:rows, :cols].astype(np.float32)) * 0.5 + else: + planes[color] = arrays[0].astype(np.float32) + return planes + + +def channel_x_positions(roi: ROI, pattern: str, count_g: int, count_rb: int) -> dict[str, np.ndarray]: + """Full-resolution sensor x coordinate for every CFA sample.""" + + positions = {} + for color, dy, dx in BAYER_LAYOUT[pattern]: + if color == "G": + positions[color] = roi.x0 + 0.5 + 2.0 * np.arange(count_g, dtype=float) + else: + positions[color] = roi.x0 + float(dx) + 2.0 * np.arange(count_rb, dtype=float) + return positions + + +def smooth_vector(y: np.ndarray, width: int) -> np.ndarray: + width = int(width) + if width <= 1 or y.size == 0: + return y.astype(np.float32, copy=True) + width = min(width, y.size) + if width <= 1: + return y.astype(np.float32, copy=True) + # Symmetric edge padding works for both odd and even window widths. + left = (width - 1) // 2 + right = width // 2 + padded = np.pad(y.astype(np.float32), (left, right), mode="edge") + kernel = np.ones(width, dtype=np.float32) / width + return np.convolve(padded, kernel, mode="valid").astype(np.float32) + + +def find_peaks_normalized(y: np.ndarray, x: np.ndarray, min_distance: int, + prominence: float, min_height: float, + max_peaks: int) -> np.ndarray: + """A small scipy-free local-maximum/prominence finder.""" + + n = y.size + if n < 3 or max_peaks <= 0: + return np.empty(0, dtype=int) + candidates = np.where((y[1:-1] >= y[:-2]) & (y[1:-1] > y[2:]))[0] + 1 + if candidates.size == 0: + return np.empty(0, dtype=int) + + # Highest first, then suppress neighbours. + candidates = candidates[np.argsort(y[candidates])[::-1]] + min_distance = max(1, int(min_distance)) + prom_window = max(8, min_distance * 4) + kept: list[int] = [] + for idx in candidates: + if y[idx] < min_height: + continue + if any(abs(int(idx) - k) < min_distance for k in kept): + continue + left = y[max(0, idx - prom_window):idx] + right = y[idx + 1:min(n, idx + 1 + prom_window)] + if left.size == 0 or right.size == 0: + base = 0.0 + else: + base = max(float(left.min()), float(right.min())) + if float(y[idx]) - base < prominence: + continue + kept.append(int(idx)) + if len(kept) >= max_peaks: + break + + kept.sort(key=lambda i: float(x[i])) + return np.asarray(kept, dtype=int) + + +def _block_sum(plane: np.ndarray, factor: int) -> np.ndarray: + """Sum ``factor`` x ``factor`` blocks of a 2-D integer plane.""" + + height, width = plane.shape + height -= height % factor + width -= width % factor + if height <= 0 or width <= 0: + return plane + return plane[:height, :width].reshape( + height // factor, factor, width // factor, factor + ).sum(axis=(1, 3), dtype=np.uint32) + + +def make_preview(raw: np.ndarray, pattern: str, target_width: int = 640) -> np.ndarray: + """Quick, ISP-free RGB preview by binning the four Bayer sites. + + The whole frame is viewed as 2x2 Bayer cells, so every site is a plain + slice of that view and the arithmetic stays integer until the final + stretch. The 1st/99.6th percentiles are estimated from a coarse + subsample: a preview does not need a per-frame exact percentile and the + exact version is what made this function the most expensive part of a + frame. + """ + + height = raw.shape[0] - (raw.shape[0] % 2) + width = raw.shape[1] - (raw.shape[1] % 2) + if height < 2 or width < 2: + raise ValueError(f"frame {raw.shape} is too small for a preview") + sub = raw[:height, :width] + if not sub.flags.c_contiguous: + sub = np.ascontiguousarray(sub) + cells = sub.reshape(height // 2, 2, width // 2, 2) + layout = BAYER_LAYOUT[pattern] + + sites: dict[str, list[np.ndarray]] = {c: [] for c in CHANNELS} + for color, dy, dx in layout: + sites[color].append(cells[:, dy, :, dx]) + + factor = max(1, int(np.ceil((width // 2) / float(target_width)))) + channels = [] + for color in CHANNELS: + arrays = sites[color] + plane = arrays[0].astype(np.uint32) + for extra in arrays[1:]: + plane += extra + if len(arrays) > 1: + plane //= len(arrays) + if factor > 1: + plane = _block_sum(plane, factor) + channels.append(plane) + + sums = np.stack(channels, axis=-1) + # ~40k samples per channel are plenty for a 1 % / 99.6 % stretch and keep + # the percentile search at ~1 ms instead of ~15 ms for the full frame. + step = max(1, int(round(max(sums.shape[0], sums.shape[1]) / 256.0))) + sample = sums[::step, ::step].reshape(-1, 3) + lo, hi = np.percentile(sample, (1.0, 99.6), axis=0) + lo = lo.astype(np.float32) + scale = np.maximum((hi - lo).astype(np.float32), 1.0) + out = sums.astype(np.float32) + out -= lo + out /= scale + np.clip(out, 0.0, 1.0, out=out) + return np.power(out, 0.8, dtype=np.float32) + + +def detect_auto_roi(preview: np.ndarray, full_width: int, full_height: int) -> Optional[ROI]: + """Find a horizontal spectral band from a stretched preview.""" + + height, width = preview.shape[:2] + if height < 8 or width < 8: + return None + luminance = 0.299 * preview[..., 0] + 0.587 * preview[..., 1] + 0.114 * preview[..., 2] + block = max(2, min(12, min(height, width) // 24)) + hh = (height // block) * block + ww = (width // block) * block + if hh < block or ww < block: + return None + blocks = luminance[:hh, :ww].reshape(hh // block, block, ww // block, block).mean(axis=(1, 3)) + + background = float(np.median(blocks)) + mad = float(np.median(np.abs(blocks - background))) + high = float(np.percentile(blocks, 99.9)) + threshold = background + max(8.0 * 1.4826 * mad, 0.02 * (high - background)) + mask = blocks > threshold + if int(mask.sum()) < 4: + return None + + row_counts = mask.sum(axis=1) + active_rows = np.where(row_counts >= max(1.0, 0.12 * row_counts.max()))[0] + if active_rows.size == 0: + return None + r0, r1 = int(active_rows.min()), int(active_rows.max()) + 1 + band = mask[r0:r1, :] + col_counts = band.sum(axis=0) + active_cols = np.where(col_counts > 0)[0] + if active_cols.size == 0: + return None + c0, c1 = int(active_cols.min()), int(active_cols.max()) + 1 + + scale_x = full_width / float(width) + scale_y = full_height / float(height) + x0 = c0 * block * scale_x + x1 = c1 * block * scale_x + y0 = r0 * block * scale_y + y1 = r1 * block * scale_y + + # Keep some dark baseline and make sure a narrow band still has context. + margin_x = 0.035 * full_width + margin_y = 0.035 * full_height + x0, x1 = x0 - margin_x, x1 + margin_x + y0, y1 = y0 - margin_y, y1 + margin_y + if x1 - x0 < 0.18 * full_width: + center = 0.5 * (x0 + x1) + x0, x1 = center - 0.09 * full_width, center + 0.09 * full_width + if y1 - y0 < 0.08 * full_height: + center = 0.5 * (y0 + y1) + y0, y1 = center - 0.04 * full_height, center + 0.04 * full_height + roi = make_roi(x0, y0, x1, y1, full_width, full_height) + if roi.width < 32 or roi.height < 8: + return None + return roi + + +def process_frame(raw: np.ndarray, meta: dict, roi: ROI, pattern: str, + args: argparse.Namespace, calibration: Optional[Calibration], + full_size: tuple[int, int], raw_format: str, + bit_depth: int) -> SpectrumResult: + planes = extract_planes(raw, pattern, roi) + if args.reduce == "median": + profiles = {c: np.median(planes[c], axis=0).astype(np.float32) for c in CHANNELS} + else: + profiles = {c: np.mean(planes[c], axis=0, dtype=np.float32) for c in CHANNELS} + + counts_g = min(planes["R"].shape[1], planes["B"].shape[1], planes["G"].shape[1]) + counts_rb = min(planes["R"].shape[1], planes["B"].shape[1]) + positions = channel_x_positions(roi, pattern, counts_g, counts_rb) + + black, full_scale = resolve_levels(meta, bit_depth, args.black, args.full_scale) + signal = {} + signal_raw = {} + normalized = {} + peaks = {} + x_px = {} + x_plot = {} + unit = "nm" if calibration is not None else "px" + peak_distance_samples = max(1, int(round(args.peak_distance / 2.0))) + smooth_samples = 1 if args.smooth <= 1 else max(1, (args.smooth + 1) // 2) + + for channel in CHANNELS: + n = min(profiles[channel].size, positions[channel].size) + y = np.maximum(profiles[channel][:n] - black, 0.0).astype(np.float32) + ys = smooth_vector(y, smooth_samples) + sig = np.clip(ys, 0.0, None) + base = float(np.percentile(sig, 5.0)) if sig.size else 0.0 + norm = np.clip(sig - base, 0.0, None) + peak = float(norm.max()) if norm.size else 0.0 + if peak > 0: + norm /= peak + else: + norm[:] = 0.0 + + x = positions[channel][:n] + x_display = calibration(x) if calibration is not None else x.copy() + idx = find_peaks_normalized( + norm, x_display, peak_distance_samples, + args.peak_prominence, args.peak_height, args.max_peaks + ) + + x_px[channel] = x + x_plot[channel] = x_display + signal[channel] = sig + signal_raw[channel] = y + normalized[channel] = norm + peaks[channel] = idx + + preview = make_preview(raw, pattern) + roi_raw = raw[roi.y0:roi.y1, roi.x0:roi.x1] + saturated = float(np.mean(roi_raw >= (0.995 * full_scale))) if roi_raw.size else 0.0 + return SpectrumResult( + roi=roi, + x_px=x_px, + x_plot=x_plot, + signal=signal, + signal_raw=signal_raw, + normalized=normalized, + peaks=peaks, + preview=preview, + full_size=full_size, + raw_format=raw_format, + bit_depth=bit_depth, + black=black, + full_scale=full_scale, + saturated_fraction=saturated, + metadata=meta, + x_unit=unit, + calibration_text=calibration.describe() if calibration is not None else "uncalibrated (sensor x)", + timestamp=time.time(), + ) + + +# --------------------------------------------------------------------------- +# Synthetic source (also useful for testing on a laptop) +# --------------------------------------------------------------------------- + + +class DemoSource: + """A BGGR 12-bit synthetic spectral frame with a horizontal dispersion axis.""" + + description = "synthetic HQ-camera-like BGGR12 frame" + + def __init__(self, args: argparse.Namespace): + self.args = args + self.full_size = (args.width - args.width % 2, args.height - args.height % 2) + self.raw_format = "SBGGR12 (demo)" + self.bit_depth = 12 + self.pattern = "BGGR" + self.stride = self.full_size[0] * 2 + self.rng = np.random.default_rng(20261023) + self.frame_index = 0 + self.base = self._build_base() + self.meta = { + "ExposureTime": 33013, + "AnalogueGain": 16.0, + "DigitalGain": 1.0, + "BlackLevel": 256, + "Lux": 12.0, + "SensorTimestamp": time.monotonic_ns(), + "Demo": True, + } + + def _gaussian(self, x: np.ndarray, center: float, sigma: float) -> np.ndarray: + return np.exp(-0.5 * ((x - center) / sigma) ** 2) + + def _build_base(self) -> np.ndarray: + width, height = self.full_size + w2, h2 = width // 2, height // 2 + x = 2.0 * np.arange(w2) + 0.5 + x0, x1 = 0.12 * width, 0.88 * width + wavelength = np.interp(x, (x0, x1), (380.0, 760.0), left=np.nan, right=np.nan) + valid = np.isfinite(wavelength) + wl = np.where(valid, wavelength, 550.0) + + # Mercury/neon-like line mixture on a small continuum. + radiance = np.where(valid, 0.10 + 0.00002 * (wl - 380.0), 0.0) + for center, amplitude, sigma in ( + (404.7, 0.40, 1.7), + (435.8, 1.00, 2.1), + (546.1, 0.92, 2.4), + (577.0, 0.45, 2.0), + (579.1, 0.40, 2.0), + (611.6, 0.34, 2.3), + (696.5, 0.48, 3.0), + ): + radiance = radiance + amplitude * self._gaussian(wl, center, sigma) * valid + + # Broad CFA response curves (illustrative, not a measured QE curve). + response = { + "R": 0.95 * self._gaussian(wl, 615.0, 78.0) + 0.10 * self._gaussian(wl, 540.0, 55.0), + "G": 0.88 * self._gaussian(wl, 545.0, 68.0) + 0.12 * self._gaussian(wl, 455.0, 45.0) + 0.08 * self._gaussian(wl, 635.0, 55.0), + "B": 0.90 * self._gaussian(wl, 458.0, 55.0) + 0.08 * self._gaussian(wl, 535.0, 60.0), + } + + # Horizontal slit image: a soft vertical band with a little tilt and texture. + yy = np.arange(h2, dtype=np.float32) + tilt = 2.0 * np.sin(np.linspace(0.0, 1.0, w2) * np.pi) + center_y = 0.53 * h2 + tilt + vertical = np.exp(-0.5 * ((yy[:, None] - center_y[None, :]) / (0.075 * h2)) ** 2) + + vignette = 1.0 - 0.18 * ((x - 0.5 * width) / (0.5 * width)) ** 2 + channel_images = {} + for channel in CHANNELS: + signal = radiance[None, :] * response[channel][None, :] * vertical + signal *= 2600.0 * vignette[None, :] + signal += 256.0 + 18.0 * np.linspace(0.0, 1.0, w2)[None, :] + channel_images[channel] = signal.astype(np.float32) + + raw = np.empty((h2 * 2, w2 * 2), dtype=np.float32) + raw[0::2, 0::2] = channel_images["B"] + raw[0::2, 1::2] = channel_images["G"] + raw[1::2, 0::2] = channel_images["G"] + raw[1::2, 1::2] = channel_images["R"] + + # A few hot pixels make sure the row reduction is not blindly trusting + # a single pixel. Mean reduction still works because they are rare. + for _ in range(80): + raw[self.rng.integers(0, raw.shape[0]), self.rng.integers(0, raw.shape[1])] = 4095.0 + return raw + + def start(self) -> None: + pass + + def read(self) -> tuple[np.ndarray, dict]: + self.frame_index += 1 + flicker = 1.0 + 0.012 * np.sin(self.frame_index * 0.23) + noise = self.rng.normal(0.0, 3.0, self.base.shape).astype(np.float32) + raw = np.clip(self.base * flicker + noise, 0.0, 4095.0).astype(np.uint16) + meta = dict(self.meta) + meta["SensorTimestamp"] = time.monotonic_ns() + meta["Lux"] = float(12.0 + 0.3 * np.sin(self.frame_index * 0.17)) + return raw, meta + + def close(self) -> None: + pass + + +class CameraSource: + """Picamera2 raw source; decodes PC1B or uncompressed SBGGR12.""" + + def __init__(self, args: argparse.Namespace): + self.args = args + try: + from picamera2 import Picamera2 + except Exception as exc: # pragma: no cover - depends on the Pi + raise RuntimeError( + "Picamera2 is not available. On Raspberry Pi OS use the system Python:\n" + " sudo apt install python3-picamera2\n" + " python3 hq_spectrum.py ..." + ) from exc + self.picam2 = Picamera2() + model = str(self.picam2.camera_properties.get("Model", "Raspberry Pi camera")) + self.description = "HQ Camera (IMX477)" if "imx477" in model.lower() else model + self._configure() + + cfg = self.picam2.camera_config["raw"] + self.full_size = (int(cfg["size"][0]), int(cfg["size"][1])) + self.raw_format = str(cfg["format"]) + self.stride = int(cfg["stride"]) + self.pattern = pattern_from_format(self.raw_format) + self.bit_depth = bit_depth_from_format(self.raw_format) + self.camera_config_raw = cfg + self._started = False + + def _controls(self) -> dict: + controls: dict = {"FrameRate": float(self.args.fps)} + if self.args.exposure is not None or self.args.gain is not None: + controls["AeEnable"] = False + controls["ExposureTime"] = int(self.args.exposure if self.args.exposure is not None else 10000) + controls["AnalogueGain"] = float(self.args.gain if self.args.gain is not None else 1.0) + return controls + + def _configure(self) -> None: + requested = self.args.raw_format.upper() if self.args.raw_format else None + + def configure_with(use_format: Optional[str]) -> None: + cfg = {"size": (self.args.width, self.args.height)} + if use_format: + cfg["format"] = use_format + config = self.picam2.create_video_configuration( + main={"size": (320, 240)}, + raw=cfg, + controls=self._controls(), + buffer_count=4, + ) + self.picam2.configure(config) + + try: + configure_with(requested) + except Exception as exc: + if not requested: + raise + print(f"warning: raw format {requested!r} was rejected ({exc}); retrying with automatic format", + file=sys.stderr) + configure_with(None) + + def start(self) -> None: + self.picam2.start() + self._started = True + + def _capture(self, decode: bool): + request = self.picam2.capture_request() + try: + raw_bytes = request.make_array("raw") + meta = request.get_metadata() + image = None + if decode: + # Decode while the request is still alive; make_array() + # normally copies, but a future version might return a view. + image, _ = decode_raw(raw_bytes, self.camera_config_raw, self.raw_format) + finally: + request.release() + if decode: + return image, meta + return raw_bytes, meta + + def read_raw(self) -> tuple[np.ndarray, dict]: + """Return the raw byte buffer (height x stride) without decoding.""" + + return self._capture(False) + + def read(self) -> tuple[np.ndarray, dict]: + return self._capture(True) + + def close(self) -> None: + try: + if self._started: + self.picam2.stop() + finally: + self.picam2.close() + + +class RecordingSource: + """Replay a recording written by hq_record.py.""" + + def __init__(self, directory: str): + self.path = Path(directory).expanduser() + meta_path = self.path / "meta.json" + if not meta_path.is_file(): + raise RuntimeError(f"recording metadata not found: {meta_path}") + try: + with open(meta_path, "r", encoding="utf-8") as handle: + meta = json.load(handle) + except Exception as exc: + raise RuntimeError(f"cannot read {meta_path}: {exc}") from exc + + for key in ("width", "height", "raw_format", "stride"): + if key not in meta: + raise RuntimeError(f"{meta_path} is missing the {key!r} field") + self.meta = dict(meta) + self.base_meta = dict(meta) + if "BlackLevel" not in self.base_meta and "black_level" in meta: + self.base_meta["BlackLevel"] = meta["black_level"] + self.full_size = (int(meta["width"]), int(meta["height"])) + self.stride = int(meta["stride"]) + self.raw_format = str(meta["raw_format"]) + self.pattern = str(meta.get("pattern") or pattern_from_format(self.raw_format)) + self.bit_depth = int(meta.get("bit_depth") or bit_depth_from_format(self.raw_format)) + self.frame_bytes = int(meta.get("frame_bytes") or self.stride * self.full_size[1]) + if self.stride <= 0 or self.frame_bytes <= 0: + raise RuntimeError(f"{meta_path} has an invalid stride/frame_bytes") + if self.frame_bytes != self.stride * self.full_size[1]: + raise RuntimeError( + f"{meta_path}: frame_bytes={self.frame_bytes} does not match " + f"stride*height={self.stride * self.full_size[1]}" + ) + self.frames_path = self.path / "frames.bin" + self.metadata_path = self.path / "frames.jsonl" + if not self.frames_path.is_file(): + raise RuntimeError(f"frames file not found: {self.frames_path}") + self.description = f"replay {self.path.name}" + self.needs_warmup = False + self.index = 0 + self.total_frames = 0 + self._fp = None + self._meta_fp = None + self._meta_offsets: list[int] = [] + + def start(self) -> None: + file_size = self.frames_path.stat().st_size + file_frames = file_size // self.frame_bytes + meta_frames = int(self.meta.get("frames") or 0) + self.total_frames = min(file_frames, meta_frames) if meta_frames > 0 else file_frames + if self.total_frames <= 0: + raise RuntimeError(f"recording {self.path} contains no complete frames") + self._fp = open(self.frames_path, "rb") + self._meta_offsets = [] + if self.metadata_path.is_file(): + self._meta_fp = open(self.metadata_path, "r", encoding="utf-8") + # Index JSONL lines so the player can seek to an arbitrary frame. + while True: + pos = self._meta_fp.tell() + line = self._meta_fp.readline() + if not line: + break + self._meta_offsets.append(pos) + self._meta_fp.seek(0) + self.index = 0 + print(f"replay : {self.path} {self.total_frames} frames") + + def rewind(self) -> None: + self.index = 0 + if self._fp is not None: + self._fp.seek(0) + if self._meta_fp is not None: + self._meta_fp.seek(0) + + def _metadata_at(self, index: int) -> dict: + if self._meta_fp is None or index >= len(self._meta_offsets): + return {} + self._meta_fp.seek(self._meta_offsets[index]) + line = self._meta_fp.readline() + if not line: + return {} + try: + record = json.loads(line) + except json.JSONDecodeError: + return {} + if not isinstance(record, dict): + return {} + nested = record.pop("metadata", None) + if isinstance(nested, dict): + return {**nested, **record} + return record + + def read_at(self, index: int) -> tuple[np.ndarray, dict]: + if self._fp is None: + raise RuntimeError("RecordingSource.start() was not called") + if index < 0: + raise IndexError(index) + if index >= self.total_frames: + raise StopIteration + self._fp.seek(index * self.frame_bytes) + data = self._fp.read(self.frame_bytes) + if len(data) != self.frame_bytes: + raise StopIteration + raw = np.frombuffer(data, dtype=np.uint8).reshape(self.full_size[1], self.stride) + + meta = dict(self.base_meta) + meta.update(self._metadata_at(index)) + meta["_frame_index"] = index + 1 + meta["_frame_total"] = self.total_frames + image, _ = decode_raw( + raw, + {"size": self.full_size, "stride": self.stride}, + self.raw_format, + ) + self.index = index + 1 + return image, meta + + def read(self) -> tuple[np.ndarray, dict]: + if self._fp is None: + raise RuntimeError("RecordingSource.start() was not called") + if self.index >= self.total_frames: + raise StopIteration + return self.read_at(self.index) + + def close(self) -> None: + if self._fp is not None: + self._fp.close() + self._fp = None + if self._meta_fp is not None: + self._meta_fp.close() + self._meta_fp = None + + +def list_camera_modes() -> None: + try: + from picamera2 import Picamera2 + except Exception as exc: + raise RuntimeError("Picamera2 is not available. Try: sudo apt install python3-picamera2") from exc + + cam = Picamera2() + try: + model = cam.camera_properties.get("Model", "unknown camera") + print(f"Camera: {model}") + print(f"{'format':26s} {'size':>12s} {'bit':>3s} {'fps':>6s}") + print("-" * 56) + for mode in cam.sensor_modes: + size = mode.get("size", ("?", "?")) + bits = mode.get("bit_depth", "?") + fps = mode.get("fps", "?") + print(f"{str(mode.get('format', '?')):26s} {size[0]:5d}x{size[1]:<5d} {str(bits):>3s} {str(fps):>6s}") + finally: + cam.close() + + +# --------------------------------------------------------------------------- +# Matplotlib dashboard +# --------------------------------------------------------------------------- + + +def _nice_ceiling(value: float) -> float: + """Round up to 1/1.5/2/2.5/3/4/5/6/8 x a power of ten.""" + + if not np.isfinite(value) or value <= 0.0: + return 1.0 + exponent = 10.0 ** np.floor(np.log10(value)) + for factor in (1.0, 1.5, 2.0, 2.5, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0): + if value <= factor * exponent: + return float(factor * exponent) + return float(10.0 * exponent) + + +def _nice_floor(value: float) -> float: + """Round down to a nice number (mirror image of _nice_ceiling).""" + + if not np.isfinite(value) or value >= 0.0: + return 0.0 + return -_nice_ceiling(-value) + + +class Dashboard: + def __init__(self, plt, args: argparse.Namespace, full_size: tuple[int, int], + description: str, calibration_text: str, on_roi=None, + player: Optional[PlayerState] = None): + self.plt = plt + self.args = args + self.full_w, self.full_h = full_size + self.on_roi = on_roi + self.selector = None + self.save_callback = None + self.last_result: Optional[SpectrumResult] = None + self.headless = args.headless + self.quit_requested = False + self.player = player + self._player_syncing = False + self.btn_play = None + self.btn_loop = None + self.btn_speed = None + self.caption = None + self.ax_slider = None + self.player_slider = None + + # Blitting: keep a cached background and repaint only the artists whose + # data changed. A full Matplotlib draw of this dashboard costs several + # times more than the capture + analysis of a frame, and almost none of + # that time is spent on the curves or images themselves. + self.fast = (not self.headless) and not getattr(args, "no_blit", False) + self._bg = None + self._bg_state = None + self._bg_size = None + self._force_full = False + self._paint_groups: list[tuple] = [] + self._wf_rows = max(2, int(getattr(args, "waterfall_rows", 240))) + self._wf_buf: Optional[np.ndarray] = None + + # These default Matplotlib bindings collide with the player/live keys + # (s=save, q=quit, space/p=play, l=loop, arrows/home/end=seek). + for keymap in ("save", "quit", "home", "back", "forward", + "pan", "yscale", "xscale"): + plt.rcParams[f"keymap.{keymap}"] = [] + + if not self.headless: + plt.ion() + figsize = (13.2, 9.0) if player is not None else (13.2, 8.2) + if self.fast: + self.fig = plt.figure(figsize=figsize) + # In interactive mode every artist update schedules a full canvas + # redraw (Figure.stale -> draw_idle). That is exactly the cost the + # blitting path exists to avoid, so disable the automatic route for + # this figure; the backend's own resize/expose handlers still ask + # for redraws, and update() paints explicitly. + self.fig.stale_callback = None + else: + self.fig = plt.figure(figsize=figsize, constrained_layout=True) + try: + self.fig.canvas.manager.set_window_title("HQ Camera RAW spectrum") + except Exception: + pass + self.fig.suptitle(f"HQ Camera RAW R/G/B spectrum monitor — {description}", fontsize=11) + + if player is not None: + gs = self.fig.add_gridspec( + 3, 2, width_ratios=(1.18, 1.0), + height_ratios=(1.08, 1.0, 0.17), hspace=0.28, + ) + else: + gs = self.fig.add_gridspec(2, 2, width_ratios=(1.18, 1.0), + height_ratios=(1.08, 1.0)) + self.ax_preview = self.fig.add_subplot(gs[0, 0]) + self.ax_spec = self.fig.add_subplot(gs[0, 1]) + self.ax_water = self.fig.add_subplot(gs[1, 0]) + self.ax_norm = self.fig.add_subplot(gs[1, 1]) + if player is not None: + self._build_player_controls(gs[2, :]) + if self.fast: + # constrained_layout re-solves the complete layout (text extents + # included) on every single draw. With a fixed figure size that + # is pure overhead, so reserve the margins once instead. + if player is not None: + self.fig.subplots_adjust(left=0.058, right=0.988, + top=0.938, bottom=0.052) + else: + self.fig.subplots_adjust(left=0.058, right=0.988, + top=0.928, bottom=0.072) + + self.im_preview = self.ax_preview.imshow( + np.zeros((2, 2, 3), dtype=np.float32), + extent=(0, self.full_w, self.full_h, 0), + origin="upper", + interpolation="nearest", + aspect="equal", + ) + self.ax_preview.set_xlim(0, self.full_w) + self.ax_preview.set_ylim(self.full_h, 0) + self.ax_preview.set_xlabel("sensor x / px") + self.ax_preview.set_ylabel("sensor y / px") + selector_wanted = not self.headless and not args.no_selector + preview_title = ("RAW preview (auto-stretched) — drag a box to set ROI" + if selector_wanted else "RAW preview (auto-stretched) and ROI") + self.ax_preview.set_title(preview_title, fontsize=8.5) + self.roi_patch = self._make_roi_patch(0, 0, 10, 10) + self.ax_preview.add_patch(self.roi_patch) + self.status_text = self.ax_preview.text( + 0.012, 0.985, "", transform=self.ax_preview.transAxes, + va="top", ha="left", fontsize=7.1, family="monospace", color="white", + bbox=dict(facecolor="black", alpha=0.58, edgecolor="none", pad=3.5), + zorder=20, + ) + + # Absolute spectra. + self.raw_lines = {} + self.smooth_lines = {} + for channel in CHANNELS: + (raw_line,) = self.ax_spec.plot([], [], color=CHANNEL_COLORS[channel], + lw=0.55, alpha=0.22) + (smooth_line,) = self.ax_spec.plot([], [], color=CHANNEL_COLORS[channel], + lw=1.55, label=channel) + self.raw_lines[channel] = raw_line + self.smooth_lines[channel] = smooth_line + self.ax_spec.set_title(f"R / G / B profiles: {args.reduce} over ROI rows", fontsize=9) + self.ax_spec.set_ylabel("black-corrected signal / ADU") + self.ax_spec.grid(alpha=0.24) + self.ax_spec.legend(loc="upper right", ncol=3, fontsize=7, framealpha=0.85) + + # Normalized spectra + peak markers. + self.norm_lines = {} + self.peak_markers = {} + for channel in CHANNELS: + (line,) = self.ax_norm.plot([], [], color=CHANNEL_COLORS[channel], + lw=1.45, label=channel) + (marker,) = self.ax_norm.plot([], [], linestyle="none", marker="o", ms=3.2, + color=CHANNEL_COLORS[channel], mec="white", mew=0.35) + self.norm_lines[channel] = line + self.peak_markers[channel] = marker + self.ax_norm.set_title("normalized profiles (baseline removed) and peaks", fontsize=9) + self.ax_norm.set_ylabel("normalized") + self.ax_norm.set_ylim(0, 1.08) + self.ax_norm.grid(alpha=0.24) + self.ax_norm.legend(loc="upper right", ncol=3, fontsize=7, framealpha=0.85) + + # Waterfall. + waterfall_height = self._wf_rows if self.fast else 2 + self.im_water = self.ax_water.imshow( + np.zeros((2, 2), dtype=np.float32), + extent=(0, self.full_w, 0, waterfall_height), + origin="lower", aspect="auto", cmap="inferno", vmin=0.0, vmax=1.0, + interpolation="nearest", + ) + self.ax_water.set_title("G-channel waterfall (normalized)", fontsize=9) + self.ax_water.set_ylabel("recent frames (old → new)") + self.ax_water.set_xlim(0, self.full_w) + if self.fast: + # A fixed frame count keeps the axis (and therefore the blitted + # background) stable while the deque fills up. + self.ax_water.set_ylim(0, self._wf_rows) + + self.calibration_text = calibration_text + self.ax_norm.set_xlabel(self._xlabel()) + self.ax_water.set_xlabel(self._xlabel(), fontsize=8) + + if selector_wanted: + from matplotlib.widgets import RectangleSelector + selector_style = dict(facecolor="none", edgecolor="#43d9ff", linewidth=1.1, + linestyle="--", alpha=0.95) + try: + self.selector = RectangleSelector( + self.ax_preview, + self._on_select, + useblit=False, + button=[1], + minspanx=8, + minspany=4, + spancoords="data", + props=selector_style, + ) + except TypeError: + # Matplotlib < 3.5 used "rectprops" for the same thing. + self.selector = RectangleSelector( + self.ax_preview, + self._on_select, + useblit=False, + button=[1], + minspanx=8, + minspany=4, + spancoords="data", + rectprops=selector_style, + ) + + self.fig.canvas.mpl_connect("key_press_event", self._on_key) + self._build_paint_groups() + if self.fast: + self.fig.canvas.mpl_connect("resize_event", self._invalidate_background) + self.fig.canvas.mpl_connect("button_press_event", self._on_button_press) + self.fig.canvas.mpl_connect("button_release_event", self._on_button_release) + if not self.headless: + plt.show(block=False) + # The axes positions are only final after the first layout pass. + self._update_caption() + + def _build_paint_groups(self) -> None: + """Artists that change from frame to frame, grouped by their axes.""" + + groups: list[tuple] = [ + (self.ax_preview, [self.im_preview, self.roi_patch, self.status_text]), + (self.ax_spec, list(self.raw_lines.values()) + list(self.smooth_lines.values())), + (self.ax_norm, list(self.norm_lines.values()) + list(self.peak_markers.values())), + (self.ax_water, [self.im_water]), + ] + if self.player is not None: + for button in (self.btn_play, self.btn_start, self.btn_prev, + self.btn_next, self.btn_end, self.btn_loop, self.btn_speed): + if button is not None: + groups.append((button.ax, [button.ax.patch, button.label])) + slider = self.player_slider + if slider is not None: + artists = [getattr(slider, name, None) for name in + ("track", "poly", "vline", "_handle", "label", "valtext")] + groups.append((slider.ax, [a for a in artists if a is not None])) + if self.caption is not None: + # Figure-level text above the transport row. It is not clipped to + # an axes, so it is drawn directly into the renderer. + groups.append((None, [self.caption])) + self._paint_groups = groups + + def _invalidate_background(self, event=None) -> None: + self._bg = None + + def _on_button_press(self, event) -> None: + # RectangleSelector needs a real redraw while the box is being dragged. + if self.fast and event.inaxes is self.ax_preview: + self._force_full = True + + def _on_button_release(self, event) -> None: + self._force_full = False + if self.fast: + self._render() + + def _view_state(self) -> tuple: + """Everything that, when changed, requires a full redraw.""" + + return ( + tuple(self.ax_spec.get_xlim()), tuple(self.ax_spec.get_ylim()), + tuple(self.ax_norm.get_xlim()), tuple(self.ax_water.get_xlim()), + tuple(self.ax_water.get_ylim()), + self.ax_norm.get_xlabel(), self.ax_water.get_xlabel(), + ) + + def _capture_background(self) -> None: + """Redraw the static parts of the figure and cache them. + + The data artists are hidden for this one draw: they are painted on top + of the cached background every frame, so baking them into it would + leave one stale copy of every curve visible for ever (the traces appear + never to be erased). + """ + + canvas = self.fig.canvas + hidden = [] + for _axes, artists in self._paint_groups: + for artist in artists: + if artist.get_visible(): + artist.set_visible(False) + hidden.append(artist) + try: + canvas.draw() + finally: + for artist in hidden: + artist.set_visible(True) + self._bg = canvas.copy_from_bbox(self.fig.bbox) + + def _render(self) -> None: + canvas = self.fig.canvas + if not self.fast: + canvas.draw_idle() + return + size = tuple(canvas.get_width_height()) + state = self._view_state() + if (self._force_full or self._bg is None or size != self._bg_size + or state != self._bg_state): + self._capture_background() + self._bg_size = size + self._bg_state = state + canvas.restore_region(self._bg) + renderer = None + for axes, artists in self._paint_groups: + for artist in artists: + if axes is None: + # Not clipped to an axes (e.g. the transport caption), so + # it has to go straight into the renderer. + if renderer is None: + renderer = self.fig.canvas.get_renderer() + artist.draw(renderer) + else: + axes.draw_artist(artist) + canvas.blit(self.fig.bbox) + + def _xlabel(self) -> str: + if "uncalibrated" in self.calibration_text: + return "sensor column / px (uncalibrated)" + return "wavelength / nm" + + def _build_player_controls(self, spec) -> None: + from matplotlib.widgets import Button, Slider + + player = self.player + if player is None: + return + cg = spec.subgridspec( + 1, 8, + width_ratios=(0.78, 0.44, 0.44, 0.44, 0.44, 5.2, 0.95, 0.95), + wspace=0.12, + ) + self.ax_play = self.fig.add_subplot(cg[0, 0]) + self.ax_start = self.fig.add_subplot(cg[0, 1]) + self.ax_prev = self.fig.add_subplot(cg[0, 2]) + self.ax_next = self.fig.add_subplot(cg[0, 3]) + self.ax_end = self.fig.add_subplot(cg[0, 4]) + self.ax_slider = self.fig.add_subplot(cg[0, 5]) + self.ax_loop = self.fig.add_subplot(cg[0, 6]) + self.ax_speed = self.fig.add_subplot(cg[0, 7]) + + label_kw = dict(fontsize=7.5, color="#1a1a1a") + self.btn_play = Button(self.ax_play, "Play", color="#dce9ff", hovercolor="#c2d8ff") + self.btn_start = Button(self.ax_start, "|<", color="#eeeeee", hovercolor="#dddddd") + self.btn_prev = Button(self.ax_prev, "<", color="#eeeeee", hovercolor="#dddddd") + self.btn_next = Button(self.ax_next, ">", color="#eeeeee", hovercolor="#dddddd") + self.btn_end = Button(self.ax_end, ">|", color="#eeeeee", hovercolor="#dddddd") + self.btn_loop = Button(self.ax_loop, "Loop: on", color="#eeeeee", hovercolor="#dddddd") + self.btn_speed = Button(self.ax_speed, f"{player.speed:g}x", color="#eeeeee", hovercolor="#dddddd") + for button in (self.btn_play, self.btn_start, self.btn_prev, self.btn_next, + self.btn_end, self.btn_loop, self.btn_speed): + button.label.set_fontsize(label_kw["fontsize"]) + button.label.set_color(label_kw["color"]) + + self.player_slider = Slider( + self.ax_slider, "", 0, max(0, player.total_frames - 1), + valinit=player.current, valstep=1, valfmt="%0.0f", + color="#ffd166", initcolor="none", + ) + self.player_slider.valtext.set_visible(False) + # Frame/time caption. A figure-level Text, not an axes title: in the + # blitting path the caption is redrawn every frame, and titles cannot + # be hidden/redrawn independently (hiding any title of the axes hides + # all of them). + self.caption = self.fig.text(0.0, 0.0, "", ha="left", va="bottom", fontsize=7.3) + if self.fast: + # Slider.set_val() asks for a full canvas redraw on every update; + # the blitting path repaints the slider itself instead. + self.player_slider.drawon = False + self._refresh_player(draw=False) + + self.btn_play.on_clicked(self._toggle_play) + self.btn_start.on_clicked(lambda event: self._player_seek(0)) + self.btn_prev.on_clicked(lambda event: self._player_step(-1)) + self.btn_next.on_clicked(lambda event: self._player_step(1)) + self.btn_end.on_clicked(lambda event: self._player_seek(player.total_frames - 1)) + self.btn_loop.on_clicked(self._toggle_loop) + self.btn_speed.on_clicked(self._cycle_speed) + self.player_slider.on_changed(self._on_player_slider) + + def _player_title(self) -> str: + player = self.player + if player is None: + return "" + seconds = player.current / max(player.fps, 0.01) + total_s = player.total_frames / max(player.fps, 0.01) + return (f"frame {player.current + 1}/{player.total_frames} " + f"t={seconds:.2f}s / {total_s:.2f}s") + + def _update_caption(self) -> None: + """Keep the frame/time caption glued just above the transport slider.""" + + if self.caption is None or self.ax_slider is None: + return + self.caption.set_text(self._player_title()) + position = self.ax_slider.get_position() + self.caption.set_position((position.x0 + 0.002, position.y1 + 0.006)) + + def _refresh_player(self, draw: bool = True) -> None: + player = self.player + if player is None or self.player_slider is None: + return + player.current = max(0, min(player.current, player.total_frames - 1)) + self.btn_play.label.set_text("Pause" if player.playing else "Play") + self.btn_loop.label.set_text(f"Loop: {'on' if player.loop else 'off'}") + self.btn_speed.label.set_text(f"{player.speed:g}x") + self._update_caption() + self._player_syncing = True + try: + self.player_slider.set_val(player.current) + finally: + self._player_syncing = False + if draw: + self._render() + + def set_player_position(self, frame: int, redraw: bool = True) -> None: + if self.player is None: + return + self.player.current = frame + self._refresh_player(draw=redraw) + + def _toggle_play(self, event=None) -> None: + player = self.player + if player is None: + return + if not player.playing and player.current >= player.total_frames - 1: + player.pending_seek = 0 + player.playing = not player.playing + if player.playing: + player.needs_reschedule = True + self._refresh_player() + + def _player_seek(self, frame: int) -> None: + player = self.player + if player is None: + return + player.playing = False + player.request(frame) + self._refresh_player() + + def _player_step(self, delta: int) -> None: + player = self.player + if player is None: + return + self._player_seek(player.current + delta) + + def _on_player_slider(self, value) -> None: + if self._player_syncing or self.player is None: + return + player = self.player + player.playing = False + player.pending_seek = max(0, min(player.total_frames - 1, int(round(value)))) + self._refresh_player() + + def _toggle_loop(self, event=None) -> None: + if self.player is None: + return + self.player.loop = not self.player.loop + self._refresh_player() + + def _cycle_speed(self, event=None) -> None: + if self.player is None: + return + speeds = (0.25, 0.5, 1.0, 2.0, 4.0) + current = self.player.speed + try: + index = min(range(len(speeds)), key=lambda i: abs(speeds[i] - current)) + except ValueError: + index = 2 + self.player.speed = speeds[(index + 1) % len(speeds)] + self.player.needs_reschedule = True + self._refresh_player() + + def _make_roi_patch(self, x0: int, y0: int, w: int, h: int): + from matplotlib.patches import Rectangle + return Rectangle((x0, y0), w, h, fill=False, edgecolor="#ffd166", + linewidth=1.5, zorder=15) + + def _on_select(self, eclick, erelease) -> None: + if (eclick.xdata is None or erelease.xdata is None or + eclick.ydata is None or erelease.ydata is None or self.on_roi is None): + return + x0, x1 = sorted((eclick.xdata, erelease.xdata)) + y0, y1 = sorted((eclick.ydata, erelease.ydata)) + roi = make_roi(x0, y0, x1, y1, self.full_w, self.full_h) + if roi.width < 24 or roi.height < 8: + return + self.on_roi(roi) + + def _on_key(self, event) -> None: + key = event.key + if key == "s" and self.save_callback is not None: + self.save_callback() + elif key == "q": + # Do not close the canvas here: the main loop still has to save + # the final figure if --save-prefix was given. + self.quit_requested = True + elif self.player is not None: + if key in (" ", "space", "p"): + self._toggle_play() + elif key == "left": + self._player_step(-1) + elif key == "right": + self._player_step(1) + elif key == "home": + self._player_seek(0) + elif key == "end": + self._player_seek(self.player.total_frames - 1) + elif key == "l": + self._toggle_loop() + + def _target_y_limits(self, result: SpectrumResult) -> tuple[float, float]: + """Y range of the absolute spectra; rounded so it seldom changes.""" + + valid = [v for v in result.signal.values() if v.size] + y_max = max(float(v.max()) for v in valid) if valid else 1.0 + return 0.0, _nice_ceiling(max(1.0, y_max * 1.1)) + + def _update_limits(self, result: SpectrumResult) -> None: + valid_x = [v for v in result.x_plot.values() if v.size] + if valid_x: + x_min = min(float(v.min()) for v in valid_x) + x_max = max(float(v.max()) for v in valid_x) + else: + x_min, x_max = 0.0, float(self.full_w) + if not np.isfinite(x_min) or not np.isfinite(x_max) or x_max <= x_min: + x_min, x_max = 0.0, float(self.full_w) + if tuple(self.ax_spec.get_xlim()) != (x_min, x_max): + self.ax_spec.set_xlim(x_min, x_max) + self.ax_norm.set_xlim(x_min, x_max) + self.ax_water.set_xlim(x_min, x_max) + if self.fast: + self.im_water.set_extent((x_min, x_max, 0, self._wf_rows)) + + target = self._target_y_limits(result) + current = tuple(self.ax_spec.get_ylim()) + # Hysteresis: grow immediately, shrink only when the current range is + # more than ~1.8x larger than needed. Without this the y axis (and + # with it the whole cached background) would change every frame. + if target[1] > current[1] or target[1] * 1.8 < current[1] or target[0] != current[0]: + self.ax_spec.set_ylim(*target) + + def _set_waterfall(self, waterfall: np.ndarray, x_min: float, x_max: float) -> None: + if self.fast: + rows = self._wf_rows + width = waterfall.shape[1] + if self._wf_buf is None or self._wf_buf.shape[1] != width: + self._wf_buf = np.zeros((rows, width), dtype=np.float32) + count = min(rows, waterfall.shape[0]) + if count < rows: + self._wf_buf[: rows - count] = 0.0 + self._wf_buf[rows - count:] = waterfall[-count:] + self.im_water.set_data(self._wf_buf) + else: + self.im_water.set_data(waterfall) + self.im_water.set_extent((x_min, x_max, 0, waterfall.shape[0])) + self.ax_water.set_ylim(0, max(1, waterfall.shape[0])) + + def update(self, result: SpectrumResult, waterfall: np.ndarray, fps: float, + status_text: str) -> None: + self.last_result = result + self.im_preview.set_data(result.preview) + self.roi_patch.set_bounds(result.roi.x0, result.roi.y0, result.roi.width, result.roi.height) + self.status_text.set_text(status_text) + + for channel in CHANNELS: + x = result.x_plot[channel] + self.raw_lines[channel].set_data(x, result.signal_raw[channel]) + self.smooth_lines[channel].set_data(x, result.signal[channel]) + self.norm_lines[channel].set_data(x, result.normalized[channel]) + idx = result.peaks[channel] + self.peak_markers[channel].set_data(x[idx], result.normalized[channel][idx]) + + self._update_limits(result) + self.ax_norm.set_xlabel(self._xlabel()) + self.ax_water.set_xlabel(self._xlabel(), fontsize=8) + + if waterfall.size: + self._set_waterfall(waterfall, *self.ax_spec.get_xlim()) + + if self.headless: + # Redrawing a full figure for every frame is needlessly expensive + # in a batch/headless run. savefig() at the end draws the final + # state once. + return + self._render() + if not self.fast: + self.fig.canvas.flush_events() + + def pump(self, seconds: float, headless: bool) -> None: + if headless: + if seconds > 0: + time.sleep(seconds) + return + if seconds <= 0: + # Nothing to wait for, but the window still has to breathe. While + # the machine is behind schedule the loop must not skip the GUI + # event loop: that is what makes the window freeze ("not + # responding") and swallows clicks on Pause. + self.fig.canvas.flush_events() + return + if self.fast: + # plt.pause() redraws the whole figure as soon as it is marked + # stale, which would throw away the blitted frame; only pump the + # GUI event queue here. + self.fig.canvas.start_event_loop(seconds) + else: + self.plt.pause(seconds) + + def save(self, prefix: str) -> tuple[Path, Path]: + if self.last_result is None: + raise RuntimeError("no frame to save yet") + png, csv_path = save_result(self.fig, self.last_result, prefix) + # savefig() renders through its own renderer; drop the cached + # background so the next frame starts from a fresh full draw. + self._bg = None + return png, csv_path + + def close(self) -> None: + self.plt.close(self.fig) + + +# --------------------------------------------------------------------------- +# Save helpers +# --------------------------------------------------------------------------- + + +def _safe_prefix(prefix: str) -> Path: + path = Path(prefix).expanduser() + if path.suffix.lower() in (".png", ".csv"): + path = path.with_suffix("") + path.parent.mkdir(parents=True, exist_ok=True) + return path + + +def write_csv(result: SpectrumResult, csv_path: Path) -> Path: + """Write one profile snapshot; shared by every front end.""" + + with open(csv_path, "w", newline="", encoding="utf-8") as handle: + handle.write(f"# hq_spectrum snapshot {datetime.now().isoformat(timespec='seconds')}\n") + handle.write(f"# roi={result.roi.text()} size={result.roi.width}x{result.roi.height}\n") + handle.write(f"# raw_format={result.raw_format} bit_depth={result.bit_depth} " + f"black={result.black:g} full_scale={result.full_scale:g}\n") + handle.write(f"# calibration={result.calibration_text}\n") + handle.write(f"# peaks_{result.x_unit}: R={format_peak_values(result, 'R', 5)} " + f"G={format_peak_values(result, 'G', 5)} " + f"B={format_peak_values(result, 'B', 5)}\n") + writer = csv.writer(handle) + writer.writerow(["channel", "x_px", "x_display", "x_unit", + "signal_adu", "signal_smoothed", "normalized"]) + for channel in CHANNELS: + x_px = result.x_px[channel] + x_plot = result.x_plot[channel] + n = min(x_px.size, x_plot.size, result.signal_raw[channel].size, + result.signal[channel].size, result.normalized[channel].size) + for i in range(n): + writer.writerow([ + channel, + f"{x_px[i]:.3f}", + f"{x_plot[i]:.4f}", + result.x_unit, + f"{result.signal_raw[channel][i]:.3f}", + f"{result.signal[channel][i]:.3f}", + f"{result.normalized[channel][i]:.6f}", + ]) + return csv_path + + +def save_result(fig, result: SpectrumResult, prefix: str) -> tuple[Path, Path]: + base = _safe_prefix(prefix) + png_path = base.with_suffix(".png") + csv_path = base.with_suffix(".csv") + fig.savefig(png_path, dpi=150) + write_csv(result, csv_path) + return png_path, csv_path + + +def top_peak_indices(result: SpectrumResult, channel: str, count: int) -> np.ndarray: + """Strongest peaks, then returned in left-to-right order for display.""" + + idx = result.peaks[channel] + if idx.size == 0 or count <= 0: + return np.empty(0, dtype=int) + heights = result.normalized[channel][idx] + chosen = idx[np.argsort(heights)[::-1][:count]] + return chosen[np.argsort(result.x_plot[channel][chosen])] + + +def format_peak_values(result: SpectrumResult, channel: str, count: int = 1) -> str: + idx = top_peak_indices(result, channel, count) + if not idx.size: + return "—" + return ", ".join(f"{result.x_plot[channel][i]:.1f}" for i in idx) + + +# --------------------------------------------------------------------------- +# Main loop +# --------------------------------------------------------------------------- + + +def make_calibration(args: argparse.Namespace) -> Optional[Calibration]: + if not args.calibrate: + return None + pairs: list[tuple[float, float]] = [] + for token in args.calibrate.split(","): + token = token.strip() + if not token: + continue + if ":" not in token: + raise ValueError(f"bad calibration token {token!r}; expected PX:NM") + px, nm = token.split(":", 1) + pairs.append((float(px), float(nm))) + if len(pairs) < 2: + raise ValueError("--calibrate needs at least two PX:NM pairs") + return Calibration(pairs, args.poly_order) + + +def _meta_number(meta: dict, key: str, default: float = 0.0) -> float: + value = meta.get(key, default) + try: + return float(value) if value is not None else default + except (TypeError, ValueError): + return default + + +def status_for(result: SpectrumResult, fps: float) -> str: + meta = result.metadata + exposure = _meta_number(meta, "ExposureTime") + gain = _meta_number(meta, "AnalogueGain") + fps_text = f"{fps:4.1f}" if fps > 0 else " --" + peaks = { + channel: format_peak_values(result, channel, 3) for channel in CHANNELS + } + frame_line = "" + if "_frame_index" in meta: + frame_line = f"frame {meta['_frame_index']}/{meta.get('_frame_total', '?')}\n" + text = ( + f"{result.raw_format} {result.full_size[0]}x{result.full_size[1]} " + f"{result.bit_depth}-bit buf\n" + f"ROI {result.roi.width}x{result.roi.height} @ {result.roi.text()} " + f"{fps_text} fps\n" + f"{frame_line}" + f"exp {exposure / 1000.0:7.2f} ms gain {gain:4.1f} black {result.black:.0f} " + f"sat {result.saturated_fraction * 100.0:.2f}%\n" + f"peaks [{result.x_unit}]\n" + f" R {peaks['R']}\n" + f" G {peaks['G']}\n" + f" B {peaks['B']}" + ) + return text + + +def run_replay_player(args: argparse.Namespace, plt, calibration: Optional[Calibration], + manual_roi: Optional[ROI]) -> int: + """Interactive player for a recording: play/pause, scrub, step, speed, loop.""" + + source = None + dashboard = None + state_result: list[Optional[SpectrumResult]] = [None] + saved = False + + try: + source = RecordingSource(args.input) + source.start() + if args.exposure is not None or args.gain is not None: + print("note : --exposure/--gain are ignored during replay", file=sys.stderr) + if args.frames: + print("note : --frames is ignored in player mode; use the timeline", file=sys.stderr) + + first_raw, first_meta = source.read_at(0) + if manual_roi is not None: + roi = parse_roi(args.roi, *source.full_size) + else: + preview = make_preview(first_raw, source.pattern) + roi = detect_auto_roi(preview, *source.full_size) + if roi is None: + width, height = source.full_size + roi = make_roi(0, 0.30 * height, width, 0.70 * height, width, height) + print("auto ROI: no spectral band found, using full-width central strip") + else: + print(f"auto ROI : {roi.text()} {roi.width}x{roi.height}") + roi_holder = [roi] + + first_result = process_frame( + first_raw, first_meta, roi, source.pattern, args, calibration, + source.full_size, source.raw_format, source.bit_depth, + ) + state_result[0] = first_result + + recorded_fps = float( + source.meta.get("average_capture_fps") + or source.meta.get("fps_requested") + or args.fps + or 15.0 + ) + if recorded_fps <= 0: + recorded_fps = 15.0 + player = PlayerState( + total_frames=source.total_frames, + fps=recorded_fps, + current=0, + playing=bool(args.autoplay), + loop=not args.no_loop, + ) + waterfall = deque(maxlen=max(2, int(args.waterfall_rows))) + waterfall.append(first_result.normalized["G"].copy()) + dashboard_ref: list[Optional[Dashboard]] = [None] + + def on_roi(new_roi: ROI) -> None: + roi_holder[0] = new_roi + waterfall.clear() + player.playing = False + player.pending_seek = player.current + if dashboard_ref[0] is not None: + dashboard_ref[0]._refresh_player() + print(f"ROI changed by mouse: {new_roi.text()} {new_roi.width}x{new_roi.height}") + + dashboard = Dashboard( + plt, args, source.full_size, source.description, + first_result.calibration_text, on_roi=on_roi, player=player, + ) + dashboard_ref[0] = dashboard + + def paint(result: SpectrumResult, fps_now: float) -> None: + wf = np.asarray(waterfall, dtype=np.float32) + if wf.size == 0: + wf = np.zeros((1, 2), dtype=np.float32) + # Move the transport widgets first; update() then repaints the + # whole dashboard (including them) in a single pass. + dashboard.set_player_position(player.current, redraw=False) + dashboard.update(result, wf, fps_now, status_for(result, fps_now)) + + def do_save(prefix: Optional[str] = None) -> bool: + if dashboard is None or state_result[0] is None: + return False + target = prefix or args.save_prefix + if not target: + target = f"hq_spectrum_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + try: + png, csv_path = dashboard.save(target) + except Exception as exc: + print(f"save failed: {exc}", file=sys.stderr) + return False + print(f"saved: {png}") + print(f"saved: {csv_path}") + return True + + dashboard.save_callback = do_save + player.current = 0 + paint(first_result, (player.fps * player.speed) if player.playing else 0.0) + next_deadline = time.monotonic() + player.period() + last_play_time = None + play_fps = 0.0 + + while plt.fignum_exists(dashboard.fig.number): + # Always give the GUI a slice of time, even when the pipeline is + # behind: otherwise the window stops repainting and Pause/keys are + # never delivered until playback ends. + dashboard.pump(0.0, False) + if dashboard.quit_requested: + break + if player.needs_reschedule: + player.needs_reschedule = False + next_deadline = time.monotonic() + player.period() + + if player.pending_seek is not None: + target = max(0, min(player.total_frames - 1, int(player.pending_seek))) + player.pending_seek = None + if target != player.current: + waterfall.clear() + raw, meta = source.read_at(target) + result = process_frame( + raw, meta, roi_holder[0], source.pattern, args, calibration, + source.full_size, source.raw_format, source.bit_depth, + ) + state_result[0] = result + player.current = target + waterfall.append(result.normalized["G"].copy()) + paint(result, 0.0) + next_deadline = time.monotonic() + player.period() + last_play_time = None + continue + + if player.playing: + now = time.monotonic() + if now < next_deadline: + dashboard.pump(min(0.05, next_deadline - now), False) + continue + target = source.index + if target >= source.total_frames: + if player.loop: + target = 0 + waterfall.clear() + else: + player.playing = False + dashboard._refresh_player() + dashboard.pump(0.05, False) + continue + raw, meta = source.read_at(target) + result = process_frame( + raw, meta, roi_holder[0], source.pattern, args, calibration, + source.full_size, source.raw_format, source.bit_depth, + ) + state_result[0] = result + player.current = target + waterfall.append(result.normalized["G"].copy()) + now_after = time.monotonic() + if last_play_time is not None: + dt = now_after - last_play_time + if dt > 0: + instant = 1.0 / dt + play_fps = instant if play_fps == 0 else 0.85 * play_fps + 0.15 * instant + last_play_time = now_after + paint(result, play_fps if play_fps > 0 else player.fps * player.speed) + next_deadline += player.period() + if next_deadline < time.monotonic(): + # Behind schedule (slow machine): run the next frame + # immediately instead of waiting for a whole extra period. + next_deadline = time.monotonic() + continue + + dashboard.pump(0.05, False) + + if args.save_prefix and state_result[0] is not None: + saved = do_save(args.save_prefix) + return 0 + except KeyboardInterrupt: + print("\ninterrupted") + return 130 + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + finally: + if args.save_prefix and dashboard is not None and state_result[0] is not None and not saved: + try: + png, csv_path = dashboard.save(args.save_prefix) + print(f"saved: {png}") + print(f"saved: {csv_path}") + except Exception as exc: + print(f"warning: could not save final snapshot: {exc}", file=sys.stderr) + if dashboard is not None: + try: + dashboard.close() + except Exception: + pass + if source is not None: + try: + source.close() + except Exception: + pass + + +def run(args: argparse.Namespace) -> int: + if args.list_modes: + try: + list_camera_modes() + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + # Import Matplotlib only after the backend has been chosen. + import matplotlib + if args.headless: + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + try: + calibration = make_calibration(args) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + try: + manual_roi = parse_roi(args.roi, args.width, args.height) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + if args.input and not args.headless: + return run_replay_player(args, plt, calibration, manual_roi) + + source = None + dashboard = None + state: dict = {"roi": manual_roi, "result": None} + saved_on_exit = False + + try: + if args.input: + source = RecordingSource(args.input) + elif args.demo: + source = DemoSource(args) + if args.headless and args.frames == 0: + args.frames = 1 + else: + source = CameraSource(args) + + if args.input and (args.exposure is not None or args.gain is not None): + print("note : --exposure/--gain are ignored during replay", file=sys.stderr) + + if manual_roi is not None: + # Re-apply fractions/pixels to the actual raw size (normally the + # same as the requested size, but libcamera is the final authority). + manual_roi = parse_roi(args.roi, *source.full_size) + state["roi"] = manual_roi + + print(f"source : {source.description}") + print(f"raw stream : {source.raw_format} {source.full_size[0]}x{source.full_size[1]}") + print(f"stride : {getattr(source, 'stride', 'n/a')}") + print(f"Bayer order : {source.pattern} (decoded {source.bit_depth}-bit buffer)") + if not args.demo and "PISP_COMP" in str(source.raw_format).upper(): + print("note : PC1B/PiSP compressed raw is being decompressed in software") + + source.start() + # Let AE/AWB and the sensor settle; also fills the first buffers. + # A recording already contains settled frames, so do not skip any. + if getattr(source, "needs_warmup", True): + for _ in range(5): + try: + source.read() + except Exception as exc: + print(f"warning: warm-up read failed: {exc}", file=sys.stderr) + break + + roi = state["roi"] + # Headless replay should run as fast as the CPU allows; no point in + # sleeping just to honour a playback frame rate on a batch job. + period = 0.0 if (args.input and args.headless) else 1.0 / max(args.fps, 0.1) + fps_est = 0.0 + previous_capture = None + frame_count = 0 + capture_error = False + waterfall_rows = max(2, int(args.waterfall_rows)) + waterfall = deque(maxlen=waterfall_rows) + last_width = None + + def on_roi(new_roi: ROI) -> None: + state["roi"] = new_roi + waterfall.clear() + print(f"ROI changed by mouse: {new_roi.text()} {new_roi.width}x{new_roi.height}") + + def do_save(prefix: Optional[str] = None) -> bool: + if dashboard is None or state.get("result") is None: + return False + target = prefix or args.save_prefix + if not target: + target = f"hq_spectrum_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + try: + png, csv_path = dashboard.save(target) + except Exception as exc: + print(f"save failed: {exc}", file=sys.stderr) + return False + print(f"saved: {png}") + print(f"saved: {csv_path}") + return True + + while True: + loop_start = time.perf_counter() + try: + raw, metadata = source.read() + except StopIteration: + if (args.input and not args.headless and not args.no_loop and + dashboard is not None and not dashboard.quit_requested): + source.rewind() + waterfall.clear() + last_width = None + previous_capture = None + print("replay : end of recording, looping") + continue + print("replay : end of recording") + break + except KeyboardInterrupt: + raise + except Exception as exc: + print(f"capture error: {exc}", file=sys.stderr) + capture_error = True + break + + # Pick up a ROI selected by the mouse in the dashboard. + roi = state["roi"] + if roi is None: + preview = make_preview(raw, source.pattern) + roi = detect_auto_roi(preview, *source.full_size) + if roi is None: + width, height = source.full_size + roi = make_roi(0, 0.30 * height, width, 0.70 * height, width, height) + print("auto ROI: no spectral band found, using full-width central strip") + else: + print(f"auto ROI : {roi.text()} {roi.width}x{roi.height}") + state["roi"] = roi + + result = process_frame( + raw, metadata, roi, source.pattern, args, calibration, + source.full_size, source.raw_format, source.bit_depth, + ) + state["result"] = result + frame_count += 1 + + if previous_capture is not None: + dt = time.perf_counter() - previous_capture + if dt > 0: + inst = 1.0 / dt + fps_est = inst if fps_est == 0.0 else 0.90 * fps_est + 0.10 * inst + previous_capture = time.perf_counter() + + norm_g = result.normalized["G"] + if last_width is not None and last_width != norm_g.size: + waterfall.clear() + last_width = norm_g.size + waterfall.append(norm_g.copy()) + wf = np.asarray(waterfall, dtype=np.float32) + if wf.size == 0: + wf = np.zeros((1, 2), dtype=np.float32) + + if dashboard is None: + dashboard = Dashboard( + plt, args, source.full_size, source.description, + result.calibration_text, on_roi=on_roi, + ) + dashboard.save_callback = do_save + + dashboard.update(result, wf, fps_est, status_for(result, fps_est)) + if args.input and (frame_count == 1 or frame_count % 30 == 0): + print(f"analyzing : frame {metadata.get('_frame_index', frame_count)}" + f"/{metadata.get('_frame_total', '?')}") + if args.verbose and frame_count == 1: + print(f"raw range : {int(raw.min())}..{int(raw.max())} " + f"p1={float(np.percentile(raw, 1)):.1f} p99={float(np.percentile(raw, 99)):.1f}") + print(f"black/full : {result.black:.1f}/{result.full_scale:.1f}") + print(f"calibration : {result.calibration_text}") + + if args.frames and frame_count >= args.frames: + break + if not args.headless and (dashboard.quit_requested or + not plt.fignum_exists(dashboard.fig.number)): + break + + elapsed = time.perf_counter() - loop_start + dashboard.pump(period - elapsed, args.headless) + + if args.save_prefix and dashboard is not None and state.get("result") is not None: + saved_on_exit = do_save(args.save_prefix) + elif args.headless and dashboard is not None and state.get("result") is not None and not args.demo: + print("headless run finished; use --save-prefix to write the dashboard and CSV") + + return 1 if capture_error else 0 + except (RuntimeError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("\ninterrupted") + return 130 + except Exception as exc: + if args.verbose: + traceback.print_exc() + print(f"error: {exc}", file=sys.stderr) + return 1 + finally: + # Save before closing the Matplotlib canvas, otherwise figure.savefig() + # may be operating on a torn-down backend. + if args.save_prefix and dashboard is not None and state.get("result") is not None and not saved_on_exit: + try: + png, csv_path = dashboard.save(args.save_prefix) + print(f"saved: {png}") + print(f"saved: {csv_path}") + except Exception as exc: + print(f"warning: could not save final snapshot: {exc}", file=sys.stderr) + if dashboard is not None: + try: + dashboard.close() + except Exception: + pass + if source is not None: + try: + source.close() + except Exception: + pass + + +def main(argv: Optional[list[str]] = None) -> int: + args = parse_args(argv) + return run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/line_tables/mercury_hg1.csv b/line_tables/mercury_hg1.csv new file mode 100644 index 0000000..2ef8afb --- /dev/null +++ b/line_tables/mercury_hg1.csv @@ -0,0 +1,6 @@ +wavelength_nm,species,label +404.6563,Hg I,Hg 405 +435.8328,Hg I,Hg 436 +546.0735,Hg I,Hg 546 +576.9598,Hg I,Hg 577 (unresolved with 579 in this instrument) +579.0663,Hg I,Hg 579 diff --git a/peak_finder.py b/peak_finder.py new file mode 100644 index 0000000..7080152 --- /dev/null +++ b/peak_finder.py @@ -0,0 +1,1411 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Peak finder and calibrated peak export for hq_record RAW recordings. + +What it does +------------ +1. Decodes one frame (or the mean of several frames) of a recording written by + ``hq_record.py`` through the decoder of ``hq_spectrum.py`` (PiSP PC1B -> + uint16 Bayer image). +2. Splits the ROI into the B / Gb+Gr / R planes and reduces it along the slit + into three one-dimensional spectra, exactly like ``hq_spectrum.py`` does. + Sensor columns are full-resolution sensor x coordinates, not ROI counters. +3. Estimates a morphological baseline and a noise level for every channel, so + peaks are found relative to the local continuum instead of the global zero. +4. Detects peaks, suppresses neighbours, and refines every centre with a + Gaussian/Lorentzian fit, falling back to the local vertex of the measured + maximum; FWHM, net height, area and SNR come from the refined profile. +5. Merges peaks that the same spectral line produces in the R, G and B CFA + channels into one physical line. +6. Converts sensor columns to wavelength with a saved calibration JSON + (``wavelength_calibration/calibration.json``), flags extrapolation, and + optionally matches a line table. +7. Exports what was asked for: for every selected peak a peak-shape image + (PNG), the peak-shape numbers (CSV), and the pixel coordinates of the peak + in the raw frame (CSV/JSON, plus an optional raw patch dump). + +Pixel coordinates +----------------- +All coordinates are 0-based full-resolution sensor coordinates; pixel ``x`` +covers ``[x, x+1)`` and row 0 is the top of the raw frame. The reduced +profile samples the sensor every 2 px, so a fitted peak centre is fractional +and not restricted to the sample grid. + +Typical use +----------- + # every detected peak, calibrated, with images and tables + python peak_finder.py --input recordings/hq_20261110_230441 --select all + + # only peaks between 540 and 560 nm, plus the two strongest lines + python peak_finder.py --input recordings/hq_20261024_135709 \ + --select nm:540-560,top:2 + + # look at the candidates first, then export a manual selection + python peak_finder.py --input recordings/hq_x --select none + python peak_finder.py --input recordings/hq_x --select id:3,id:7-9 + + # click peaks in a window instead of typing ids + python peak_finder.py --input recordings/hq_x --pick + + # average 8 frames to lower the noise before peak finding + python peak_finder.py --input recordings/hq_x --frame 10 --average 8 + +No smoothing, denoising or background subtraction is applied to the exported +raw signal. The baseline exists only to measure peak height and width, and it +is exported next to the raw values so the choice stays auditable. + +Requires ``hq_spectrum.py`` beside this file (or ``--module-dir``), NumPy and +Matplotlib. SciPy is not used. +""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import math +import sys +from pathlib import Path +from typing import Optional + +import numpy as np + +try: # shared core lives beside this script + from spectro_core import ( # noqa: F401 (re-exported for compatibility) + GAUSS_TO_FWHM, Calibration, _fit_pair, _jsonable, _num, attach_wavelength, + deblend_close_peaks, dedupe_channel_peaks, estimate_baseline, estimate_noise, + find_peak_candidates, fit_profile, group_channels, half_max_crossings, is_saturated, load_hq, + merge_coincident_lines, model_values, parabola_center, peak_prominence, + refine_peak, summarise_line, write_csv, write_json) +except ImportError as exc: # pragma: no cover - placement error + raise SystemExit('peak_finder.py needs spectro_core.py in the same directory ' + '(or on sys.path)') from exc + +PROGRAM_DIR = Path(__file__).resolve().parent +SHAPES = ("gauss", "lorentz") +GAUSS_TO_FWHM = 2.3548200450309493 + + +# --------------------------------------------------------------------------- +# Imports and small helpers +# --------------------------------------------------------------------------- + + + + + + + + + + +# --------------------------------------------------------------------------- +# Calibration +# --------------------------------------------------------------------------- + + + + +# --------------------------------------------------------------------------- +# Baseline, noise and local peak metrics +# --------------------------------------------------------------------------- + + + + + + + + + + + + + + + + +# --------------------------------------------------------------------------- +# Line-shape fitting (Levenberg-Marquardt, no SciPy) +# --------------------------------------------------------------------------- + + + + + + + + + + + + +# --------------------------------------------------------------------------- +# Peak detection and refinement +# --------------------------------------------------------------------------- + + + + + + + + +# --------------------------------------------------------------------------- +# Grouping peaks from the three CFA channels +# --------------------------------------------------------------------------- + + + + + + + + + + + + + + + + +# --------------------------------------------------------------------------- +# Selection +# --------------------------------------------------------------------------- + + +def _match_ids(token: str, lines: list[dict]) -> set[str]: + available = {line["id"] for line in lines} + numbers = [int(line["id"][1:]) for line in lines] + highest = max(numbers) if numbers else 0 + if "-" in token: + first, second = token.split("-", 1) + low, high = int(first), int(second) + if low > high: + low, high = high, low + chosen = {f"P{value:03d}" for value in range(low, high + 1)} + else: + chosen = {f"P{int(token):03d}"} + if not chosen: + raise ValueError(f"empty peak id selection {token!r}") + missing = sorted(chosen - available) + if missing: + raise ValueError(f"unknown peak id(s): {', '.join(missing)}") + return chosen + + +def _split_range(token: str): + """Split 'LOW-HIGH' (optionally with a leading minus) into two strings.""" + + body = token[1:] if token.startswith("-") else token + if "-" not in body: + return None + first, second = body.split("-", 1) + return first, second + + +def _match_wavelengths(token: str, lines: list[dict], half_width_nm: float) -> set[str]: + values = [line["wavelength_nm"] for line in lines if np.isfinite(line["wavelength_nm"])] + if not values: + raise ValueError("wavelength selection needs --calibration") + parts = _split_range(token) + try: + if parts is None: + low, high = float(token) - half_width_nm, float(token) + half_width_nm + else: + low, high = float(parts[0]), float(parts[1]) + except ValueError as exc: + raise ValueError(f"invalid wavelength selection {token!r}; use nm:LOW-HIGH") from exc + if low > high: + low, high = high, low + return {line["id"] for line in lines + if np.isfinite(line["wavelength_nm"]) and low <= line["wavelength_nm"] <= high} + + +def _fallback_half_width(lines: list[dict]) -> float: + """Half-width (nm) for a bare wavelength token, from the typical line spacing.""" + + wavelengths = [line["wavelength_nm"] for line in lines if np.isfinite(line["wavelength_nm"])] + if len(wavelengths) < 2: + return 1.0 + spacing = float(np.median(np.diff(np.sort(np.asarray(wavelengths, dtype=float))))) + return max(0.5, min(2.0, spacing)) if np.isfinite(spacing) and spacing > 0 else 1.0 + + +def parse_selection(text: str, lines: list[dict]) -> set[str]: + """Parse --select: all, none, ids, id ranges, nm ranges and top:N.""" + + tokens = [token.strip() for token in str(text).replace(";", ",").split(",") if token.strip()] + if not tokens: + raise ValueError("--select needs at least one token") + numbers = [int(line["id"][1:]) for line in lines] + highest = max(numbers) if numbers else 0 + half_width = _fallback_half_width(lines) + selected: set[str] = set() + for token in tokens: + low = token.lower() + if low in ("all", "*"): + selected.update(line["id"] for line in lines) + elif low in ("none", "no", "null"): + continue + elif low.startswith("top:"): + count = int(low[4:]) + if count < 1: + raise ValueError("top:N needs N >= 1") + ranked = sorted(lines, key=lambda item: (-item["snr"], item["sensor_x_px"])) + selected.update(item["id"] for item in ranked[:count]) + elif low.startswith("id:"): + selected.update(_match_ids(low[3:], lines)) + elif low.startswith("nm:"): + selected.update(_match_wavelengths(low[3:], lines, half_width)) + elif _split_range(low) is not None: + first, second = _split_range(low) + try: + left, right = float(first), float(second) + except ValueError as exc: + raise ValueError(f"cannot parse selection token {token!r}") from exc + integers = "." not in first and "." not in second + if integers and right <= highest: + selected.update(_match_ids(low, lines)) + elif left >= 200 and right >= 200: + selected.update(_match_wavelengths(low, lines, half_width)) + else: + raise ValueError(f"selection {token!r} is neither a peak id range nor a " + f"wavelength range; use id:N-M or nm:LOW-HIGH") + elif low.replace(".", "", 1).isdigit(): + if "." in low: + selected.update(_match_wavelengths(low, lines, half_width)) + else: + selected.update(_match_ids(low, lines)) + else: + raise ValueError(f"cannot parse selection token {token!r}") + return selected + + +# --------------------------------------------------------------------------- +# Line table matching +# --------------------------------------------------------------------------- + + +def load_line_table(path) -> list[dict]: + """Read a CSV/JSON line table with wavelength_nm, species and label columns.""" + + location = Path(path).expanduser() + if not location.is_file(): + raise ValueError(f"line table not found: {location}") + if location.suffix.lower() == ".json": + data = json.loads(location.read_text(encoding="utf-8-sig")) + rows = data.get("lines") if isinstance(data, dict) else data + if not isinstance(rows, list): + raise ValueError("line table JSON needs a list of lines") + table = [] + for row in rows: + if not isinstance(row, dict): + raise ValueError("line table rows must be objects") + value = row.get("wavelength_nm", row.get("nm")) + if value is None: + continue + table.append({"wavelength_nm": float(value), + "species": str(row.get("species", row.get("element", ""))), + "label": str(row.get("label", ""))}) + else: + with location.open("r", newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + if not reader.fieldnames: + raise ValueError("line table needs a header row") + fields = {name.strip().lower(): name for name in reader.fieldnames} + wavelength = fields.get("wavelength_nm") or fields.get("nm") or fields.get("wavelength") + if wavelength is None: + raise ValueError("line table needs a wavelength_nm column") + species = fields.get("species") or fields.get("element") or fields.get("atom") + label = fields.get("label") or fields.get("line") + table = [] + for row in reader: + try: + value = float(row[wavelength]) + except (TypeError, ValueError): + continue + table.append({"wavelength_nm": value, + "species": (row.get(species) or "").strip() if species else "", + "label": (row.get(label) or "").strip() if label else ""}) + if not table: + raise ValueError("line table contains no usable wavelengths") + return sorted(table, key=lambda item: item["wavelength_nm"]) + + +def match_lines(lines: list[dict], table: list[dict], tolerance_nm: float) -> None: + known = np.asarray([entry["wavelength_nm"] for entry in table], dtype=float) + for line in lines: + line.update({"match_species": "", "match_nm": float("nan"), + "match_delta_nm": float("nan"), "match_label": ""}) + if not np.isfinite(line["wavelength_nm"]) or known.size == 0: + continue + index = int(np.argmin(np.abs(known - line["wavelength_nm"]))) + delta = float(line["wavelength_nm"] - known[index]) + if abs(delta) <= tolerance_nm: + entry = table[index] + line.update({"match_species": entry["species"], "match_nm": entry["wavelength_nm"], + "match_delta_nm": delta, "match_label": entry["label"]}) + + +# --------------------------------------------------------------------------- +# Reading the recording +# --------------------------------------------------------------------------- + + +def resolve_rows(value, roi, height: int, hq): + """Intersect the ROI with an optional absolute row window.""" + + if not value: + return roi + parts = [part.strip() for part in str(value).split(",")] + if len(parts) != 2: + raise ValueError("--rows must be y0,y1 in sensor rows") + try: + y0, y1 = (int(round(float(part))) for part in parts) + except ValueError as exc: + raise ValueError("--rows values must be numbers") from exc + y0 = max(0, min(y0, height - 4)) + y1 = max(4, min(y1, height)) + if y1 - y0 < 4: + raise ValueError("--rows window is too small") + y0 -= y0 % 2 + y1 -= y1 % 2 + low, high = max(roi.y0, y0), min(roi.y1, y1) + if high - low < 4: + raise ValueError("--rows window does not overlap the ROI") + return hq.ROI(roi.x0, int(low), roi.x1, int(high)) + + +def read_frames(hq, source, args, tell): + """Read one frame or the mean of several; returns float image, raw copy, indices.""" + + total = int(source.total_frames) + if args.frame < 0 or args.frame >= total: + raise ValueError(f"--frame {args.frame} outside 0..{total - 1}") + count = max(1, int(args.average)) + if args.frame + count > total: + raise ValueError(f"--average {count} from frame {args.frame} exceeds {total} frames") + indices = list(range(args.frame, args.frame + count)) + show_progress = count > 1 and sys.stdout.isatty() + accumulation = None + original = None + frame_meta = None + for position, index in enumerate(indices): + image, meta = source.read_at(index) + if frame_meta is None: + frame_meta = meta + accumulation = (image.astype(np.float64) if accumulation is None + else accumulation + image) + if count == 1: + original = image + if show_progress: + tell(f"\r reading frame {position + 1}/{count}", end="") + if count > 1: + if show_progress: + tell("\r" + " " * 40, end="\r") + accumulation = accumulation / float(count) + return accumulation.astype(np.float32), original, indices, frame_meta or {} + + +# --------------------------------------------------------------------------- +# Spectrum construction +# --------------------------------------------------------------------------- + + +class SpectrumData: + """Reduced spectra plus the raw image needed by the exports.""" + + def __init__(self, hq, source, image, original, indices, roi, args, black, full_scale): + self.hq = hq + self.source = source + self.image = image + self.original = original + self.indices = indices + self.roi = roi + self.args = args + self.black = black + self.full_scale = full_scale + self.pattern = source.pattern + self.full_size = tuple(source.full_size) + self.raw_format = source.raw_format + self.bit_depth = source.bit_depth + self.step_px = 2.0 + self.x_px: dict[str, np.ndarray] = {} + self.raw: dict[str, np.ndarray] = {} + self.baseline: dict[str, np.ndarray] = {} + self.net: dict[str, np.ndarray] = {} + self.noise: dict[str, float] = {} + self._build() + + def _build(self) -> None: + hq = self.hq + planes = hq.extract_planes(self.image, self.pattern, self.roi) + counts_g = planes["G"].shape[1] + counts_rb = min(planes["R"].shape[1], planes["B"].shape[1]) + positions = hq.channel_x_positions(self.roi, self.pattern, counts_g, counts_rb) + window = 0 if self.args.baseline_window <= 0 else max( + 3, int(round(self.args.baseline_window / self.step_px))) + for channel in hq.CHANNELS: + if self.args.reduce == "median": + profile = np.median(planes[channel], axis=0).astype(np.float64) + else: + profile = np.mean(planes[channel], axis=0, dtype=np.float64) + values = profile - self.black + x = positions[channel][:values.size] + baseline = estimate_baseline(values, window, hq.smooth_vector) + net = values - baseline + self.x_px[channel] = x + self.raw[channel] = values + self.baseline[channel] = baseline + self.net[channel] = net + self.noise[channel] = estimate_noise(net) + + def detect(self) -> dict[str, list[dict]]: + args = self.args + result = {} + for channel in self.hq.CHANNELS: + net = self.net[channel] + noise = self.noise[channel] + top = float(np.max(net)) if net.size else 0.0 + if top <= 0: + result[channel] = [] + continue + min_height = max(args.peak_height * top, args.min_snr * noise) + min_prominence = max(args.peak_prominence * top, 0.4 * args.min_snr * noise) + min_distance = max(1, int(round(args.peak_distance / self.step_px))) + candidates = find_peak_candidates(net, min_distance, min_height, + min_prominence, args.max_peaks) + peaks = [] + for candidate in candidates: + index = candidate["sample"] + x = self.x_px[channel][index] + saturated = is_saturated(self.image, self.roi, x, self.full_scale) + peaks.append(refine_peak( + self.x_px[channel], net, self.raw[channel], candidate, noise, self.step_px, + max(1, int(round(args.fwhm_span / self.step_px))), + max(0, int(round(args.fit_window / self.step_px))), + args.shape, saturated, + baseline_value=float(self.baseline[channel][index]))) + result[channel] = deblend_close_peaks( + self.x_px[channel], net, dedupe_channel_peaks(peaks, args.peak_distance), + args.shape, self.step_px, noise) + return result + + +# --------------------------------------------------------------------------- +# Plotting +# --------------------------------------------------------------------------- + + +def _axis_wavelength(ax, calibration: Optional[Calibration]) -> None: + if calibration is None or calibration.width is None: + return + try: + secondary = ax.secondary_xaxis("top", functions=(calibration, calibration.inverse)) + secondary.set_xlabel("Wavelength / nm (air)") + secondary.tick_params(labelsize=8) + except (ValueError, TypeError): + pass + + +def plot_overview(path: Path, spectrum: SpectrumData, lines: list[dict], + calibration, selected: set[str], meta: dict) -> None: + import matplotlib.pyplot as plt + + hq = spectrum.hq + unit = "Wavelength / nm (air)" if calibration is not None else "Sensor column / px" + fig, axes = plt.subplots(2, 1, figsize=(15, 8.5), constrained_layout=True, sharex=True) + for channel in hq.CHANNELS: + x = spectrum.x_px[channel] + x_plot = calibration(x) if calibration is not None else x + axes[0].plot(x_plot, spectrum.raw[channel], color=hq.CHANNEL_COLORS[channel], + lw=0.8, label=channel) + axes[0].plot(x_plot, spectrum.baseline[channel], color=hq.CHANNEL_COLORS[channel], + lw=0.6, ls=":", alpha=0.7) + peak = float(np.max(spectrum.net[channel])) if spectrum.net[channel].size else 0.0 + norm = spectrum.net[channel] / peak if peak > 0 else np.zeros_like(x) + axes[1].plot(x_plot, norm, color=hq.CHANNEL_COLORS[channel], lw=0.8, label=channel) + for line in lines: + color = "#c0392b" if line["id"] in selected else "#7f8c8d" + axes[1].axvline(line["x_plot"], color=color, lw=0.8, + alpha=0.9 if line["id"] in selected else 0.35) + if line["id"] in selected: + axes[1].annotate(line["id"], (line["x_plot"], 1.0), xytext=(-2, 6), + textcoords="offset points", rotation=90, fontsize=7, + color=color, ha="right") + if calibration is not None: + low, high = calibration.nm_span + for axis in axes: + axis.axvspan(low, high, color="#2ecc71", alpha=0.08, zorder=0) + axes[0].set_ylabel("Black-corrected ADU") + raw_top = max(float(np.max(values)) for values in spectrum.raw.values() if values.size) + raw_bottom = min(float(np.min(values)) for values in spectrum.raw.values() if values.size) + linthresh = max(1.0, 0.002 * raw_top) + # linscale keeps the empty linear band around zero thin: without it roughly + # half of the panel is spent below the data. + axes[0].set_yscale("symlog", linthresh=linthresh, linscale=0.15) + axes[0].set_ylim(min(raw_bottom * 1.2, -linthresh), raw_top * 1.15) + axes[0].legend(loc="upper right", fontsize=8) + span = (f"frame {spectrum.indices[0]}" if len(spectrum.indices) == 1 else + f"frames {spectrum.indices[0]}-{spectrum.indices[-1]} (mean {len(spectrum.indices)})") + note = ("" if calibration is None else + f" | green band = calibrated {calibration.nm_span[0]:.0f}-" + f"{calibration.nm_span[1]:.0f} nm, outside = extrapolated") + axes[0].set_title(f"{meta['recording']} {span} ROI {spectrum.roi.text()} " + f"black={spectrum.black:g} ADU{note}", fontsize=9) + axes[1].set_ylabel("Net / channel max") + axes[1].set_xlabel(unit) + axes[1].set_ylim(-0.05, 1.35) + fig.savefig(path, dpi=140) + plt.close(fig) + + +def peak_patch_box(spectrum: SpectrumData, line: dict, half_width_px: float, + min_rows: int = 24) -> tuple[int, int, int, int]: + """Full-sensor pixel box (x0, y0, x1, y1) of the raw patch for one peak.""" + + roi = spectrum.roi + width, height = spectrum.full_size + x0 = int(math.floor(max(0.0, line["sensor_x_px"] - half_width_px))) + x1 = int(math.ceil(min(float(width), line["sensor_x_px"] + half_width_px))) + x0 -= x0 % 2 + x1 = min(width, x1 + (x1 % 2)) + x0 = max(0, min(x0, width - 2)) + x1 = max(x0 + 2, x1) + + block = spectrum.image[roi.y0:roi.y1, max(x0, roi.x0):min(x1, roi.x1)] + if block.size == 0: + return x0, roi.y0, x1, roi.y1 + rows = np.maximum(block.astype(np.float64) - spectrum.black, 0.0).sum(axis=1) + if rows.max() <= 0: + return x0, roi.y0, x1, roi.y1 + active = np.where(rows >= 0.2 * rows.max())[0] + y0 = int(active.min()) + roi.y0 + y1 = int(active.max()) + 1 + roi.y0 + pad = max(4, int(round(0.12 * (y1 - y0)))) + y0 = max(roi.y0, y0 - pad) + y1 = min(roi.y1, y1 + pad) + if y1 - y0 < min_rows: + center = (y0 + y1) // 2 + y0 = max(roi.y0, center - min_rows // 2) + y1 = min(roi.y1, y0 + min_rows) + y0 += y0 % 2 + y1 -= y1 % 2 + return x0, int(y0), x1, int(y1) + + +def slit_row_centroid(spectrum: SpectrumData, line: dict) -> float: + """Intensity-weighted row centroid of the line inside the ROI.""" + + roi = spectrum.roi + center = int(round(line["sensor_x_px"])) + x0 = max(roi.x0, center - 2) + x1 = min(roi.x1, center + 3) + block = spectrum.image[roi.y0:roi.y1, x0:x1] + if block.size == 0: + return float("nan") + weight = np.maximum(block.astype(np.float64) - spectrum.black, 0.0).sum(axis=1) + total = float(weight.sum()) + if total <= 0: + return float("nan") + rows = np.arange(roi.y0, roi.y0 + weight.size, dtype=float) + return float(np.dot(rows, weight) / total) + + +def zoom_range(spectrum: SpectrumData, line: dict) -> tuple[float, float]: + """Sensor-column window shown around one peak.""" + + fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0 + half = max(3.0, 2.5 * fwhm_px) + low = max(float(spectrum.roi.x0), line["sensor_x_px"] - half) + high = min(float(spectrum.roi.x1), line["sensor_x_px"] + half) + return low, high + + +def frame_preview(spectrum: SpectrumData) -> np.ndarray: + """Auto-stretched RGB preview of the whole frame, as the player shows it.""" + + hq = spectrum.hq + source = spectrum.original + if source is None: + source = np.clip(np.rint(spectrum.image), 0.0, 65535.0).astype(np.uint16) + return hq.make_preview(source, spectrum.pattern) + + +def draw_preview_panel(ax, spectrum: SpectrumData, line: dict, preview: np.ndarray) -> None: + """The whole-frame RAW preview the player shows, with the peak located on it. + + Full sensor height and width (not just the ROI), so the spectrum band keeps + its real size and position in the frame; the ROI box and the peak column are + overlaid. + """ + + from matplotlib.patches import Rectangle + + width, height = spectrum.full_size # full_size is (width, height) + roi = spectrum.roi + fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0 + if preview.size: + ax.imshow(preview, extent=(0.0, float(width), float(height), 0.0), + interpolation="nearest", aspect="auto") + ax.set_xlim(0.0, float(width)) + ax.set_ylim(float(height), 0.0) + ax.add_patch(Rectangle((roi.x0, roi.y0), roi.width, roi.height, + fill=False, edgecolor="#f39c12", lw=0.9)) + ax.axvline(line["sensor_x_px"], color="#00e5ff", lw=1.1) + ax.axvspan(line["sensor_x_px"] - 0.5 * fwhm_px, line["sensor_x_px"] + 0.5 * fwhm_px, + color="#00e5ff", alpha=0.20) + row = slit_row_centroid(spectrum, line) + if np.isfinite(row): + ax.plot([line["sensor_x_px"]], [row], marker="o", ms=5, mfc="none", + mec="#00e5ff", mew=1.2) + ax.set_title(f"RAW preview (auto-stretched) ROI {roi.text()} " + f"peak column x = {line['sensor_x_px']:.1f} px", fontsize=8) + ax.set_xlabel("Sensor column / px") + ax.set_ylabel("Sensor row / px") + ax.tick_params(labelsize=7) + + +def draw_raw_patch(ax, spectrum: SpectrumData, line: dict, + box: tuple[int, int, int, int]) -> None: + """RAW Bayer crop around one peak, axes in full-sensor pixel coordinates.""" + + from matplotlib.colors import PowerNorm + + x0, y0, x1, y1 = box + patch = spectrum.image[y0:y1, x0:x1] + fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0 + if patch.size: + stretch = np.percentile(patch.astype(np.float64), [2, 99.95]) + low_level = max(stretch[0], spectrum.black) + high_level = max(stretch[1], low_level + 1.0) + ax.imshow(patch, cmap="inferno", aspect="auto", interpolation="nearest", + extent=(x0, x1, y1, y0), + norm=PowerNorm(gamma=0.45, vmin=low_level, vmax=high_level)) + ax.axvline(line["sensor_x_px"], color="#00e5ff", lw=1.0) + ax.axvspan(line["sensor_x_px"] - 0.5 * fwhm_px, line["sensor_x_px"] + 0.5 * fwhm_px, + color="#00e5ff", alpha=0.10) + ax.set_title(f"RAW Bayer patch (undemosaiced) x {x0}-{x1} y {y0}-{y1} (sensor px)", + fontsize=8) + ax.set_xlabel("Sensor column / px") + ax.set_ylabel("Sensor row / px") + ax.tick_params(labelsize=7) + + +def draw_zoom(ax, spectrum: SpectrumData, line: dict, calibration, args, + low: float, high: float, wavelength_axis: bool = True) -> None: + """Net profiles of the channels forming this line, with fits and FWHM marks.""" + + hq = spectrum.hq + for channel in hq.CHANNELS: + peak = line["channels"].get(channel) + if peak is None: + continue + x = spectrum.x_px[channel] + mask = (x >= low) & (x <= high) + error = peak.get("center_err_px", float("nan")) + ax.plot(x[mask], spectrum.net[channel][mask], color=hq.CHANNEL_COLORS[channel], + lw=1.1, marker=".", ms=3.0, + label=f"{channel} {peak['height_net_adu']:.0f} ADU " + f"FWHM {peak['fwhm_px']:.2f} px" + + (f" x±{error:.2f}" if np.isfinite(error) else "")) + if peak.get("fit_params"): + window = (x >= peak["window_lo_px"]) & (x <= peak["window_hi_px"]) + values = model_values(x[window], peak["fit_params"], peak["fit_xref"], args.shape) + ax.plot(x[window], values, color=hq.CHANNEL_COLORS[channel], lw=0.9, + ls="--", alpha=0.9) + if np.isfinite(peak["fwhm_px"]): + half = 0.5 * peak["height_net_adu"] + ax.plot([peak["center_px"] - 0.5 * peak["fwhm_px"], + peak["center_px"] + 0.5 * peak["fwhm_px"]], [half, half], + color=hq.CHANNEL_COLORS[channel], lw=0.8, marker="|", ms=6) + ax.axvline(line["sensor_x_px"], color="#34495e", lw=0.9, ls=":") + ax.set_xlim(low, high) + ax.set_xlabel("Sensor column / px") + ax.set_ylabel("Net ADU (baseline removed)") + ax.legend(fontsize=7, loc="upper right") + ax.grid(alpha=0.25) + ax.tick_params(labelsize=8) + if wavelength_axis: + _axis_wavelength(ax, calibration) + + +def plot_peak(path: Path, spectrum: SpectrumData, line: dict, calibration, + args, box: tuple[int, int, int, int]) -> None: + import matplotlib.pyplot as plt + + hq = spectrum.hq + fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0 + low, high = zoom_range(spectrum, line) + + fig = plt.figure(figsize=(13.5, 9.0), constrained_layout=True) + grid = fig.add_gridspec(2, 2, height_ratios=[0.72, 1.0], width_ratios=[1.0, 1.25]) + ax_context = fig.add_subplot(grid[0, :]) + ax_raw = fig.add_subplot(grid[1, 0]) + ax_zoom = fig.add_subplot(grid[1, 1]) + + for channel in hq.CHANNELS: + x = spectrum.x_px[channel] + x_plot = calibration(x) if calibration is not None else x + ax_context.plot(x_plot, spectrum.net[channel], color=hq.CHANNEL_COLORS[channel], + lw=0.7, alpha=0.85) + half_nm = (0.5 * line["fwhm_nm"] if np.isfinite(line["fwhm_nm"]) + else 0.5 * fwhm_px * (line["dispersion_nm_per_px"] + if np.isfinite(line["dispersion_nm_per_px"]) else 1.0)) + ax_context.axvspan(line["x_plot"] - 1.5 * half_nm, line["x_plot"] + 1.5 * half_nm, + color="#f1c40f", alpha=0.3, zorder=0) + frame_text = (f"frame {spectrum.indices[0]}" + if len(spectrum.indices) == 1 else + f"frames {spectrum.indices[0]}-{spectrum.indices[-1]}" + f" (mean {len(spectrum.indices)})") + title = (f"[{frame_text}] {line['id']} {line['wavelength_nm']:.2f} nm | sensor x = " + f"{line['sensor_x_px']:.3f} px | {line['channel_names']} " + f"SNR {line['snr']:.0f} FWHM {fwhm_px:.2f} px") + if np.isfinite(line["fwhm_nm"]): + title += f" ({line['fwhm_nm']:.2f} nm)" + if line["saturated"]: + title += " [SATURATED]" + if line["extrapolated"]: + title += " [extrapolated]" + if any(peak.get("deblended") for peak in line["channels"].values()): + title += " [pair fit]" + ax_context.set_title(title, fontsize=9) + ax_context.set_ylabel("Net ADU") + ax_context.set_xlabel("Wavelength / nm (air)" if calibration is not None + else "Sensor column / px") + ax_context.tick_params(labelsize=8) + ax_context.grid(alpha=0.25) + + draw_raw_patch(ax_raw, spectrum, line, box) + draw_zoom(ax_zoom, spectrum, line, calibration, args, low, high, wavelength_axis=True) + fig.savefig(path, dpi=150) + plt.close(fig) + + +# --------------------------------------------------------------------------- +# Export +# --------------------------------------------------------------------------- + + +ALL_PEAK_FIELDS = [ + "group_id", "channel", "sample_index", "sensor_x_px", "wavelength_nm", "extrapolated", + "height_net_adu", "prominence_adu", "raw_adu", "baseline_adu", "snr", + "fwhm_px", "fwhm_nm", "fwhm_halfmax_px", "center_px", "center_source", "center_parabola_px", + "center_fit_px", "center_err_px", "sigma_px", "area_net_adu_px", + "fit_ok", "fit_rms_adu", "deblended", "saturated", +] + +SELECTED_FIELDS = [ + "id", "wavelength_nm", "extrapolated", "channels", "primary_channel", "n_channels", + "sensor_x_px", "center_spread_px", "sample_index", "sample_step_px", + "dispersion_nm_per_px", "height_net_adu", "snr", "fwhm_px", "fwhm_nm", + "area_net_adu_px", "baseline_adu", "raw_adu", "saturated", "slit_y_px", + "roi_x0", "roi_y0", "roi_x1", "roi_y1", + "patch_x0", "patch_y0", "patch_x1", "patch_y1", + "frame", "frames_averaged", "image_png", "shape_csv", "raw_npy", "raw_csv", + "match_species", "match_nm", "match_delta_nm", +] + +PROFILE_FIELDS = ["channel", "sample_index", "sensor_x_px", "wavelength_nm", + "raw_adu", "baseline_adu", "net_adu", "fit_adu", "residual_adu"] + + +def _clean_output(directory: Path) -> None: + for pattern in ("peak_*", "peaks*", "selected_peaks*"): + for stale in directory.glob(pattern): + if stale.is_file(): + stale.unlink() + sub = directory / "peaks" + if sub.is_dir(): + for stale in sub.glob("*"): + if stale.is_file(): + stale.unlink() + + +def export_all(hq, spectrum: SpectrumData, lines: list[dict], calibration, + selected: set[str], args, meta: dict) -> dict: + import matplotlib + if not args.pick: + matplotlib.use("Agg") + import matplotlib.pyplot as plt # noqa: F401 + + out = Path(args.output).expanduser() + out.mkdir(parents=True, exist_ok=True) + _clean_output(out) + (out / "peaks").mkdir(exist_ok=True) + + plot_overview(out / "peaks_overview.png", spectrum, lines, calibration, selected, meta) + + all_rows = [] + for line in lines: + for channel in hq.CHANNELS: + peak = line["channels"].get(channel) + if peak is None: + continue + values = [] + for field in ALL_PEAK_FIELDS: + values.append(line["id"] if field == "group_id" else _num(peak.get(field))) + all_rows.append(values) + write_csv(out / "peaks_all.csv", ALL_PEAK_FIELDS, all_rows) + + records = [] + rows = [] + for line in lines: + chosen = line["id"] in selected + box = peak_patch_box(spectrum, line, args.patch_half_width) + x0, y0, x1, y1 = box + slit_y = slit_row_centroid(spectrum, line) + link = {"image_png": "", "shape_csv": "", "raw_npy": "", "raw_csv": ""} + if chosen: + if not args.no_images: + plot_peak(out / "peaks" / f"peak_{line['id']}.png", spectrum, line, + calibration, args, box) + link["image_png"] = f"peaks/peak_{line['id']}.png" + link["shape_csv"] = write_peak_shape(out, spectrum, line, calibration, args) + link.update(write_patch(out, spectrum, line, box, args)) + + record = { + "id": line["id"], + "wavelength_nm": line["wavelength_nm"], + "extrapolated": line["extrapolated"], + "sensor_x_px": line["sensor_x_px"], + "center_spread_px": line["center_spread_px"], + "sample_index": line["sample_index"], + "sample_step_px": spectrum.step_px, + "dispersion_nm_per_px": line["dispersion_nm_per_px"], + "channels": line["channel_names"], + "primary_channel": line["primary_channel"], + "height_net_adu": line["height_net_adu"], + "snr": line["snr"], + "fwhm_px": line["fwhm_px"], + "fwhm_nm": line["fwhm_nm"], + "area_net_adu_px": line["area_net_adu_px"], + "saturated": line["saturated"], + "slit_y_px": slit_y, + "roi": [spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1], + "patch": [x0, y0, x1, y1], + "selected": chosen, + "match": {"species": line["match_species"], "nm": line["match_nm"], + "delta_nm": line["match_delta_nm"]}, + "files": link, + "per_channel": { + channel: {key: peak.get(key) for key in ( + "sample_index", "sensor_x_px", "center_px", "center_source", + "center_parabola_px", "center_fit_px", + "center_err_px", "sigma_px", + "fwhm_px", "fwhm_nm", "fwhm_halfmax_px", "wavelength_nm", "extrapolated", + "height_net_adu", + "prominence_adu", "raw_adu", "baseline_adu", "snr", "area_net_adu_px", + "fit_ok", "fit_rms_adu", "deblended", "saturated", "dispersion_nm_per_px", + "window_lo_px", "window_hi_px", "fit_params", "fit_xref")} + for channel, peak in line["channels"].items() + }, + } + records.append(record) + if chosen: + primary = line["channels"][line["primary_channel"]] + rows.append([ + line["id"], _num(line["wavelength_nm"]), _num(line["extrapolated"]), + line["channel_names"], line["primary_channel"], len(line["channels"]), + _num(line["sensor_x_px"]), _num(line["center_spread_px"]), + line["sample_index"], _num(spectrum.step_px), + _num(line["dispersion_nm_per_px"]), _num(line["height_net_adu"]), + _num(line["snr"]), _num(line["fwhm_px"]), _num(line["fwhm_nm"]), + _num(line["area_net_adu_px"]), _num(primary["baseline_adu"]), + _num(primary["raw_adu"]), _num(line["saturated"]), _num(slit_y), + spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1, + x0, y0, x1, y1, spectrum.indices[0], len(spectrum.indices), + link["image_png"], link["shape_csv"], link["raw_npy"], link["raw_csv"], + line["match_species"], _num(line["match_nm"]), _num(line["match_delta_nm"]), + ]) + write_csv(out / "selected_peaks.csv", SELECTED_FIELDS, rows) + + manifest = { + "program": "peak_finder.py", + "created": meta["created"], + "recording": meta["recording"], + "frames": spectrum.indices, + "frames_averaged": len(spectrum.indices), + "full_size": list(spectrum.full_size), + "roi": [spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1], + "roi_text": spectrum.roi.text(), + "bayer_pattern": spectrum.pattern, + "raw_format": spectrum.raw_format, + "bit_depth": spectrum.bit_depth, + "black_adu": spectrum.black, + "full_scale_adu": spectrum.full_scale, + "reduce": args.reduce, + "baseline_window_px": args.baseline_window, + "noise_adu": spectrum.noise, + "shape": args.shape, + "thresholds": { + "peak_height_fraction": args.peak_height, + "peak_prominence_fraction": args.peak_prominence, + "min_snr": args.min_snr, + "peak_distance_px": args.peak_distance, + "max_peaks_per_channel": args.max_peaks, + "group_tolerance_px": args.group_tol, + }, + "calibration": None if calibration is None else { + "path": calibration.path, + "coefficients_descending": calibration.coeff, + "calibrated_pixel_range": calibration.span, + "wavelength_offset_nm": calibration.offset_nm, + "reference_recording": calibration.data.get("recording"), + }, + "coordinate_convention": { + "sensor_x_px": "0-based full-resolution sensor column; pixel x covers [x, x+1)", + "sensor_row": "0-based full-resolution sensor row; row 0 is the top of the frame", + "sample_index": "index into the reduced 1-D profile; samples are 2 px apart", + "roi": "[x0, y0, x1, y1) in sensor pixels", + "patch": "[x0, y0, x1, y1) of the exported raw patch", + "slit_y_px": "intensity-weighted row centroid of the line inside the ROI", + }, + "selected_ids": sorted(selected), + "peaks": records, + } + write_json(out / "peaks.json", manifest) + return {"output": out, "records": records} + + +def write_peak_shape(out: Path, spectrum: SpectrumData, line: dict, calibration, args) -> str: + name = f"peaks/peak_{line['id']}_shape.csv" + rows = [] + for channel in spectrum.hq.CHANNELS: + peak = line["channels"].get(channel) + if peak is None: + continue + x = spectrum.x_px[channel] + raw = spectrum.raw[channel] + baseline = spectrum.baseline[channel] + net = spectrum.net[channel] + mask = (x >= peak["window_lo_px"]) & (x <= peak["window_hi_px"]) + model = (model_values(x, peak["fit_params"], peak["fit_xref"], args.shape) + if peak.get("fit_params") else np.full(x.size, np.nan)) + for index in np.where(mask)[0]: + wavelength = float(calibration(x[index])) if calibration is not None else float("nan") + fitted = float(model[index]) + rows.append([channel, int(index), _num(x[index]), _num(wavelength), _num(raw[index]), + _num(baseline[index]), _num(net[index]), _num(fitted), + _num(net[index] - fitted) if np.isfinite(fitted) else ""]) + write_csv(out / name, PROFILE_FIELDS, rows) + return name + + +def write_patch(out: Path, spectrum: SpectrumData, line: dict, + box: tuple[int, int, int, int], args) -> dict: + x0, y0, x1, y1 = box + crop = (spectrum.original[y0:y1, x0:x1] if spectrum.original is not None + else spectrum.image[y0:y1, x0:x1].astype(np.float32)) + # The file name carries the pixel box, so the dump is self-describing. + stem = f"peak_{line['id']}_raw_y{y0}-{y1}_x{x0}-{x1}" + links = {"raw_npy": "", "raw_csv": ""} + if args.patch_format in ("npy", "both"): + np.save(out / "peaks" / f"{stem}.npy", crop) + links["raw_npy"] = f"peaks/{stem}.npy" + if args.patch_format in ("csv", "both") or args.patch_csv: + rows = [[y0 + row, x0 + column, _num(crop[row, column])] + for row in range(crop.shape[0]) for column in range(crop.shape[1])] + write_csv(out / "peaks" / f"{stem}.csv", ["sensor_row", "sensor_col", "adu"], rows) + links["raw_csv"] = f"peaks/{stem}.csv" + return links + + +# --------------------------------------------------------------------------- +# Interactive picking +# --------------------------------------------------------------------------- + + +def fit_window_to_screen(fig, width_fraction: float = 0.94, + height_fraction: float = 0.90) -> None: + """Shrink the figure window when it would be larger than the screen. + + A figure is sized in inches x dpi, which on scaled displays can exceed the + screen; the window then opens partly off-screen and looks "cut in half". + """ + + try: + window = getattr(fig.canvas.manager, "window", None) + if window is None or not hasattr(window, "winfo_screenwidth"): + return + screen_w = float(window.winfo_screenwidth()) + screen_h = float(window.winfo_screenheight()) + width_in, height_in = fig.get_size_inches() + dpi = float(fig.dpi) + scale = min(width_fraction * screen_w / (width_in * dpi), + height_fraction * screen_h / (height_in * dpi), 1.0) + if scale < 0.995: + fig.set_size_inches(width_in * scale, height_in * scale) + except Exception: + pass + + +def pick_peaks(spectrum: SpectrumData, lines: list[dict], calibration, + selected: set[str]) -> Optional[set[str]]: + """Browse peaks with a live RAW/zoom preview. + + Hover moves the preview, click toggles selection, ``a``/``n`` select all or + none, ``e`` exports and ``q`` cancels. + """ + + import matplotlib.pyplot as plt + + hq = spectrum.hq + args = spectrum.args + fig = plt.figure(figsize=(16.5, 8.5)) + fit_window_to_screen(fig) + # Fixed axes rectangles instead of a layout engine: the three panels must be + # there at any window size, and constrained_layout can collapse a column on + # odd geometries. + ax_over = fig.add_axes([0.045, 0.085, 0.49, 0.84]) + ax_raw = fig.add_axes([0.585, 0.60, 0.39, 0.33]) + ax_zoom = fig.add_axes([0.585, 0.075, 0.39, 0.385]) + + for channel in hq.CHANNELS: + x = spectrum.x_px[channel] + x_plot = calibration(x) if calibration is not None else x + peak = float(np.max(spectrum.net[channel])) if spectrum.net[channel].size else 0.0 + ax_over.plot(x_plot, spectrum.net[channel] / peak if peak > 0 else spectrum.net[channel], + color=hq.CHANNEL_COLORS[channel], lw=0.7, label=channel) + handles = {line["id"]: ax_over.axvline(line["x_plot"], lw=1.0) for line in lines} + ax_over.set_xlabel("Wavelength / nm (air)" if calibration is not None + else "Sensor column / px") + ax_over.set_ylabel("Net / channel max") + ax_over.set_ylim(-0.05, 1.25) + ax_over.grid(alpha=0.25) + ax_over.legend(loc="upper right", fontsize=8) + + preview = frame_preview(spectrum) + by_id = {line["id"]: line for line in lines} + state = {"selected": set(selected), + "focus": lines[0]["id"] if lines else None, + "export": False} + + def nearest(x_value: float): + found, best = None, float("inf") + for line in lines: + distance = abs(line["x_plot"] - x_value) + if distance < best: + found, best = line, distance + span = abs(ax_over.get_xlim()[1] - ax_over.get_xlim()[0]) + return found if found is not None and best <= 0.02 * span else None + + def refresh() -> None: + for key, handle in handles.items(): + chosen = key in state["selected"] + handle.set_color("#c0392b" if chosen else "#bdc3c7") + handle.set_alpha(1.0 if chosen else 0.45) + handle.set_linewidth(2.2 if key == state["focus"] else 1.0) + chosen = sorted(state["selected"]) + ax_over.set_title("Selected: " + (", ".join(chosen) if chosen else "none") + + " | hover = preview, click = toggle, a = all, n = none, " + "e = export, q = cancel", fontsize=9) + ax_raw.clear() + ax_zoom.clear() + line = by_id.get(state["focus"]) + if line is not None: + draw_preview_panel(ax_raw, spectrum, line, preview) + low, high = zoom_range(spectrum, line) + draw_zoom(ax_zoom, spectrum, line, calibration, args, low, high, + wavelength_axis=False) + flags = (" [SATURATED]" if line["saturated"] else "") + \ + (" [extrapolated]" if line["extrapolated"] else "") + wave = (f"{line['wavelength_nm']:.2f} nm " + if np.isfinite(line["wavelength_nm"]) else "") + mark = "* " if line["id"] in state["selected"] else "" + ax_zoom.set_title(f"{mark}{line['id']} {wave}sensor x " + f"{line['sensor_x_px']:.2f} px {line['channel_names']} " + f"SNR {line['snr']:.0f}{flags}", fontsize=8) + fig.canvas.draw_idle() + + def on_click(event) -> None: + if event.inaxes is not ax_over or event.xdata is None: + return + line = nearest(event.xdata) + if line is None: + return + state["focus"] = line["id"] + if line["id"] in state["selected"]: + state["selected"].remove(line["id"]) + else: + state["selected"].add(line["id"]) + refresh() + + def on_move(event) -> None: + if event.inaxes is not ax_over or event.xdata is None: + return + line = nearest(event.xdata) + if line is not None and line["id"] != state["focus"]: + state["focus"] = line["id"] + refresh() + + def on_key(event) -> None: + if event.key == "a": + state["selected"] = {line["id"] for line in lines} + refresh() + elif event.key == "n": + state["selected"] = set() + refresh() + elif event.key == "e": + state["export"] = True + plt.close(fig) + elif event.key == "q": + plt.close(fig) + + fig.canvas.mpl_connect("button_press_event", on_click) + fig.canvas.mpl_connect("motion_notify_event", on_move) + fig.canvas.mpl_connect("key_press_event", on_key) + refresh() + plt.show(block=True) + if not state["export"]: + return None + return state["selected"] + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Find spectral peaks in hq_record RAW recordings, calibrate them and " + "export peak shapes and pixel coordinates.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + io_group = parser.add_argument_group("input") + io_group.add_argument("--input", required=True, metavar="REC_DIR", + help="hq_record recording directory") + io_group.add_argument("--frame", type=int, default=0, help="zero-based frame index (default 0)") + io_group.add_argument("--average", type=int, default=1, + help="mean this many consecutive frames starting at --frame (default 1)") + io_group.add_argument("--roi", default="auto", + help="'auto', x0,y0,x1,y1, or 'calibration' to reuse the saved ROI") + io_group.add_argument("--rows", default=None, metavar="Y0,Y1", + help="restrict the ROI to these sensor rows") + io_group.add_argument("--reduce", choices=("mean", "median"), default="mean", + help="row reduction inside the ROI (default mean)") + io_group.add_argument("--black", type=float, default=None, help="black level override in ADU") + io_group.add_argument("--full-scale", type=float, default=None, + help="white level override in decoded ADU") + io_group.add_argument("--calibration", default=None, + help="calibration JSON to apply to the sensor columns; omit it to export pixel coordinates only") + io_group.add_argument("--wavelength-offset", type=float, default=0.0, metavar="NM", + help="manual shift added to the calibration (default 0)") + io_group.add_argument("--module-dir", default=None, + help="directory containing hq_spectrum.py (default this folder)") + io_group.add_argument("--quiet", action="store_true", help="suppress progress output") + + detect = parser.add_argument_group("peak detection") + detect.add_argument("--baseline-window", type=float, default=100.0, metavar="PX", + help="baseline opening width in sensor px; 0 disables the baseline") + detect.add_argument("--peak-distance", type=float, default=4.0, metavar="PX", + help="minimum distance between peaks in sensor px (2 px = one sample); " + "raise it to merge barely resolved blends") + detect.add_argument("--peak-height", type=float, default=0.02, + help="minimum peak height as a fraction of the channel maximum") + detect.add_argument("--peak-prominence", type=float, default=0.01, + help="minimum peak prominence as a fraction of the channel maximum") + detect.add_argument("--min-snr", type=float, default=15.0, + help="minimum peak height in units of the channel noise") + detect.add_argument("--max-peaks", type=int, default=40, help="maximum peaks per channel") + detect.add_argument("--group-tol", type=float, default=4.0, metavar="PX", + help="channel peaks closer than this are one line") + detect.add_argument("--fwhm-span", type=float, default=60.0, metavar="PX", + help="maximum half-width search half-range in sensor px") + detect.add_argument("--fit-window", type=float, default=0.0, metavar="PX", + help="half-width of the fit window in sensor px; 0 = automatic") + detect.add_argument("--shape", choices=SHAPES, default="gauss", help="line-shape model") + + export = parser.add_argument_group("selection and export") + export.add_argument("--select", default="all", metavar="TOKENS", + help="all | none | 3 | 3-7 | nm:540-560 | top:5 (default all)") + export.add_argument("--pick", action="store_true", + help="choose peaks by clicking in a window, then export") + export.add_argument("--output", default=None, metavar="DIR", + help="output directory (default peak_exports/_f)") + export.add_argument("--patch-half-width", type=float, default=24.0, metavar="PX", + help="half-width of the exported raw patch in sensor px") + export.add_argument("--patch-format", choices=("npy", "csv", "both", "none"), default="npy", + help="raw patch dump format (default npy)") + export.add_argument("--patch-csv", action="store_true", + help="also dump the raw patch as a long CSV") + export.add_argument("--no-images", action="store_true", help="skip the per-peak PNG images") + export.add_argument("--match", default=None, metavar="TABLE", + help="CSV/JSON line table used to suggest peak identities") + export.add_argument("--match-tolerance", type=float, default=1.5, metavar="NM", + help="matching tolerance in nm (default 1.5)") + return parser + + +def validate_arguments(args) -> None: + if args.average < 1: + raise ValueError("--average must be >= 1") + if args.max_peaks < 1: + raise ValueError("--max-peaks must be >= 1") + if args.peak_distance <= 0 or args.baseline_window < 0 or args.fwhm_span <= 0: + raise ValueError("--peak-distance and --fwhm-span must be > 0; --baseline-window >= 0") + if not 0 <= args.peak_height <= 1 or not 0 <= args.peak_prominence <= 1: + raise ValueError("--peak-height and --peak-prominence are fractions between 0 and 1") + if args.min_snr < 0 or args.group_tol <= 0 or args.patch_half_width <= 0: + raise ValueError("--min-snr >= 0, --group-tol > 0 and --patch-half-width > 0 required") + if args.fit_window < 0 or args.match_tolerance <= 0: + raise ValueError("--fit-window >= 0 and --match-tolerance > 0 required") + + +def run(args) -> int: + validate_arguments(args) + hq = load_hq(args.module_dir) + tell = (lambda *parts, **kwargs: None) if args.quiet else print + + # A calibration is only applied when the caller names one: this program does + # not calibrate and must not silently attach someone else's wavelength axis. + calibration = None + if args.calibration: + calibration = Calibration(args.calibration, args.wavelength_offset) + + source = hq.RecordingSource(args.input) + try: + source.start() + if calibration is not None: + calibration.validate_geometry(source, tell) + calibration.set_width(source.full_size[0]) + image, original, indices, frame_meta = read_frames(hq, source, args, tell) + if str(args.roi).strip().lower() == "calibration": + saved = calibration.data.get("roi") if calibration is not None else None + if not saved or len(saved) != 4: + raise ValueError("--roi calibration needs a calibration JSON with a roi field") + roi = hq.parse_roi(",".join(str(value) for value in saved), *source.full_size) + else: + roi = hq.parse_roi(args.roi, *source.full_size) + if roi is None: + # hq_spectrum.make_preview() bins integer Bayer sites, so hand it + # the original uint16 frame (or a rounded copy of the mean frame). + preview_source = original if original is not None else np.clip( + np.rint(image), 0.0, 65535.0).astype(np.uint16) + roi = hq.detect_auto_roi(hq.make_preview(preview_source, source.pattern), + *source.full_size) + if roi is None: + raise ValueError("auto ROI failed; specify --roi x0,y0,x1,y1") + roi = resolve_rows(args.rows, roi, source.full_size[1], hq) + meta = dict(source.base_meta) + meta.update({key: value for key, value in frame_meta.items() + if not str(key).startswith("_")}) + black, full_scale = hq.resolve_levels(meta, source.bit_depth, args.black, args.full_scale) + spectrum = SpectrumData(hq, source, image, original, indices, roi, args, black, full_scale) + finally: + source.close() + + tell(f"recording : {Path(args.input).expanduser().resolve()}") + tell(f"frames : {indices[0]}-{indices[-1]} (mean of {len(indices)})") + tell(f"ROI : {roi.text()} ({roi.width} x {roi.height} px)") + tell(f"black level : {black:g} ADU full scale {full_scale:g} ADU") + tell("noise (ADU) : " + ", ".join(f"{c}={spectrum.noise[c]:.1f}" for c in hq.CHANNELS)) + if calibration is not None: + tell(f"calibration : {calibration.describe()}") + else: + tell("calibration : none - pixel coordinates only " + "(pass --calibration JSON to convert columns to wavelength)") + + per_channel = spectrum.detect() + lines = group_channels(per_channel, args.group_tol) + attach_wavelength(lines, calibration) + for line in lines: + line["sample_step_px"] = spectrum.step_px + if args.match: + match_lines(lines, load_line_table(args.match), args.match_tolerance) + + if args.pick: + import matplotlib + if matplotlib.get_backend().lower() in ("agg", "pdf", "svg", "ps", "cairo", "template"): + raise ValueError("--pick needs an interactive Matplotlib backend; use --select instead") + try: + chosen = pick_peaks(spectrum, lines, calibration, set()) + except Exception as exc: # no usable display, missing Tk, ... + raise ValueError(f"--pick could not open a window ({exc}); use --select instead") from exc + if chosen is None: + tell("Selection cancelled; nothing exported.") + return 1 + selected = chosen + elif not lines: + selected = set() + else: + selected = parse_selection(args.select, lines) + + if args.output is None: + args.output = str(default_output_directory(args.input, indices)) + meta_out = {"created": _timestamp(), + "recording": str(Path(args.input).expanduser().resolve())} + result = export_all(hq, spectrum, lines, calibration, selected, args, meta_out) + + tell("") + tell(f"{'':2s}{'id':4s}{'nm':>10s}{'x_px':>10s}{'ch':>4s}{'SNR':>9s}{'height':>9s}" + f"{'FWHM px':>9s}{'FWHM nm':>9s}{'slit y':>9s} flags") + for line in lines: + flags = [] + if line["extrapolated"]: + flags.append("extrapolated") + if line["saturated"]: + flags.append("saturated") + if line["match_species"]: + flags.append(f"{line['match_species']} {line['match_delta_nm']:+.2f} nm") + mark = "*" if line["id"] in selected else " " + tell(f"{mark:2s}{line['id']:4s}{_num(line['wavelength_nm'], 7):>10s}" + f"{line['sensor_x_px']:10.3f}{line['channel_names']:>4s}{line['snr']:9.0f}" + f"{line['height_net_adu']:9.0f}{_num(line['fwhm_px'], 4):>9s}" + f"{_num(line['fwhm_nm'], 4):>9s}{_num(slit_row_centroid(spectrum, line), 6):>9s}" + f" {', '.join(flags)}") + total = sum(len(items) for items in per_channel.values()) + tell("") + tell(f"{len(lines)} line(s) from {total} channel peak(s); {len(selected)} selected.") + tell(f"exported : {result['output'].expanduser().resolve()}") + if calibration is not None: + outside = [line["id"] for line in lines if line["extrapolated"]] + if outside: + low, high = calibration.nm_span + tell(f"warning : {len(outside)} line(s) outside the calibrated range " + f"{low:.1f}-{high:.1f} nm ({', '.join(outside[:8])}" + f"{'...' if len(outside) > 8 else ''})") + return 0 + + +def default_output_directory(recording, indices) -> Path: + """Default export folder that names the analysed frame(s). + + ``peak_exports/_f0003`` for one frame and + ``peak_exports/_f0003-0010_avg8`` for an averaged stack, so that + exports of different frames never overwrite each other. + """ + + stem = Path(recording).expanduser().resolve().name + if len(indices) > 1: + return Path("peak_exports") / ( + f"{stem}_f{indices[0]:04d}-{indices[-1]:04d}_avg{len(indices)}") + return Path("peak_exports") / f"{stem}_f{indices[0]:04d}" + + +def _timestamp() -> str: + from datetime import datetime + return datetime.now().astimezone().isoformat(timespec="seconds") + + +def main(argv: Optional[list[str]] = None) -> int: + args = build_parser().parse_args(argv) + try: + return run(args) + except (ValueError, RuntimeError, OSError, IndexError, StopIteration, KeyError) as exc: + print(f"Peak finder error: {exc}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + print("Interrupted.", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + raise SystemExit(main()) + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d76ae3f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +# 分析端(PC)依赖:读录制、寻峰、画图、标定 +numpy>=1.24 +matplotlib>=3.6 + +# 树莓派采集端还需要(系统包,不走 pip;按需安装): +# sudo apt install python3-picamera2 python3-numpy python3-matplotlib python3-tk diff --git a/spectro_core.py b/spectro_core.py new file mode 100644 index 0000000..45c9946 --- /dev/null +++ b/spectro_core.py @@ -0,0 +1,860 @@ +#!/usr/bin/env python3 +"""Shared core for the spectro tools: loading, calibration, peak detection, output. + +Extracted from peak_finder.py so that wavelength_calibration.py, wavelength_player.py +and hq_average.py can call the same code instead of keeping their own copies. + +No command line and no matplotlib at import time; plotting helpers stay in the +scripts that need them. Everything here is pure NumPy. +""" +from __future__ import annotations + +import csv +import importlib.util +import json +import math +import sys +from pathlib import Path +from typing import Optional + +import numpy as np + +CORE_DIR = Path(__file__).resolve().parent +GAUSS_TO_FWHM = 2.3548200450309493 + + +def load_hq(module_dir: Optional[str] = None) -> object: + """Import hq_spectrum.py from this folder (or ``module_dir``) by path.""" + + candidates = [Path(module_dir).expanduser()] if module_dir else [CORE_DIR, Path.cwd()] + for directory in candidates: + path = directory / "hq_spectrum.py" + if path.is_file(): + spec = importlib.util.spec_from_file_location("hq_spectrum_shared", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + try: # last resort: a normal import from sys.path + import hq_spectrum as module # type: ignore + except ImportError as exc: + raise RuntimeError("cannot find hq_spectrum.py; put it beside spectro_core.py " + "or use --module-dir") from exc + return module + + +def is_saturated(image: np.ndarray, roi, x_center: float, full_scale: float, + band: tuple[int, int] = (-2, 4), threshold: float = 0.01) -> bool: + """True when a meaningful fraction of the pixels near ``x_center`` is clipped.""" + + x0 = max(0, min(int(x_center) + band[0], image.shape[1])) + x1 = max(0, min(int(x_center) + band[1], image.shape[1])) + block = image[roi.y0:roi.y1, x0:x1] + if block.size == 0: + return False + return float(np.mean(block >= 0.995 * full_scale)) > threshold + + +def _num(value, digits: int = 6) -> str: + """Format a value for CSV output; non-finite values become empty cells.""" + + if value is None: + return "" + if isinstance(value, (bool, np.bool_)): + return "1" if value else "0" + if isinstance(value, str): + return value + try: + number = float(value) + except (TypeError, ValueError): + return str(value) + if not np.isfinite(number): + return "" + return f"{number:.{digits}g}" + + +def write_csv(path: Path, fields: list[str], rows) -> None: + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(fields) + for row in rows: + writer.writerow(row) + + +def _jsonable(value): + """Convert NumPy scalars/arrays and Paths into plain JSON values.""" + + if isinstance(value, dict): + return {str(k): _jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(v) for v in value] + if isinstance(value, np.ndarray): + return [_jsonable(v) for v in value.tolist()] + if isinstance(value, (np.floating, np.integer, np.bool_)): + return value.item() + if isinstance(value, Path): + return str(value) + return value + + +def write_json(path, value) -> None: + """Write JSON with NumPy/Path values converted and a trailing newline.""" + + Path(path).write_text(json.dumps(_jsonable(value), indent=2) + "\n", encoding="utf-8") + + +class Calibration: + """Polynomial sensor-pixel -> wavelength saved by wavelength_calibration.py.""" + + def __init__(self, path, offset_nm: float = 0.0): + self.path = Path(path).expanduser().resolve() + try: + self.data = json.loads(self.path.read_text(encoding="utf-8-sig")) + except OSError as exc: + raise ValueError(f"cannot read calibration {self.path}: {exc}") from exc + except json.JSONDecodeError as exc: + raise ValueError(f"calibration {self.path} is not valid JSON: {exc}") from exc + if not isinstance(self.data, dict): + raise ValueError("calibration JSON must contain an object") + coeff = np.asarray(self.data.get("coefficients_descending"), dtype=float) + if coeff.ndim != 1 or len(coeff) not in (2, 3): + raise ValueError("calibration needs 2 or 3 descending coefficients") + if not np.all(np.isfinite(coeff)) or coeff[0] == 0: + raise ValueError("calibration coefficients must be finite and the leading one nonzero") + if self.data.get("order", len(coeff) - 1) != len(coeff) - 1: + raise ValueError("calibration order does not match coefficient count") + self.coeff = coeff + span = np.asarray(self.data.get("calibrated_pixel_range"), dtype=float) + if span.shape != (2,) or not np.all(np.isfinite(span)) or span[0] >= span[1]: + raise ValueError("calibration has an invalid calibrated_pixel_range") + self.span = span + self.offset_nm = float(offset_nm) + if not np.isfinite(self.offset_nm): + raise ValueError("wavelength offset must be finite") + self.width: Optional[int] = None + self._grid_px = None + self._grid_nm = None + + def __call__(self, x): + return np.polyval(self.coeff, np.asarray(x, dtype=float)) + self.offset_nm + + def dispersion(self, x) -> np.ndarray: + """Local nm per sensor pixel (never zero over a validated sensor).""" + + return np.polyval(np.polyder(self.coeff), np.asarray(x, dtype=float)) + + def set_width(self, width: int) -> None: + self.width = int(width) + grid = np.arange(self.width, dtype=float) + values = np.asarray(self(grid), dtype=float) + order = np.argsort(values) + self._grid_px = grid[order] + self._grid_nm = values[order] + + def inverse(self, wavelength): + """Sensor pixel of a wavelength; used for the secondary plot axis only.""" + + if self._grid_nm is None: + raise ValueError("calibration width is unknown") + return np.interp(np.asarray(wavelength, dtype=float), self._grid_nm, self._grid_px) + + @property + def nm_span(self) -> tuple[float, float]: + values = sorted(np.asarray(self(self.span), dtype=float).tolist()) + return float(values[0]), float(values[1]) + + def describe(self) -> str: + low, high = self.nm_span + text = (f"JSON calibration ({self.data.get('wavelength_medium', 'air')}); " + f"confirmed {low:.2f}-{high:.2f} nm; outside = extrapolation") + if self.offset_nm: + text += f"; manual offset {self.offset_nm:+.3f} nm" + return text + + def validate_geometry(self, source, tell=None) -> None: + """Refuse a recording whose sensor geometry differs from the calibration.""" + + tell = tell or (lambda *args, **kwargs: None) + derivative = self.dispersion([0.0, float(source.full_size[0] - 1)]) + if np.any(np.abs(derivative) < 1e-12) or derivative[0] * derivative[1] <= 0: + raise ValueError("calibration is not monotonic over this recording width") + reference = self.data.get("recording") + metadata = Path(reference) / "meta.json" if reference else None + if metadata is not None and metadata.is_file(): + reference_meta = json.loads(metadata.read_text(encoding="utf-8-sig")) + size = (int(reference_meta["width"]), int(reference_meta["height"])) + if size != tuple(source.full_size): + raise ValueError("recording dimensions differ from the calibration recording; " + "recalibrate this sensor mode") + else: + tell("note : calibration reference recording unavailable; sensor size not checked") + if self.data.get("pattern") and self.data["pattern"] != source.pattern: + tell(f"note : calibration pattern {self.data['pattern']} differs from " + f"recording pattern {source.pattern}") + + +def _rank_filter(y: np.ndarray, width: int, reducer) -> np.ndarray: + """Running min/max over the window [i - width, i + width] with edge padding.""" + + width = max(0, int(width)) + values = np.asarray(y, dtype=float) + if width == 0 or values.size <= 1: + return values.copy() + n = values.size + padded = np.pad(values, (width, width), mode="edge") + out = padded[width:width + n].copy() + for shift in range(-width, width + 1): + start = width + shift + reducer(out, padded[start:start + n], out=out) + return out + + +def _min_filter(y: np.ndarray, width: int) -> np.ndarray: + return _rank_filter(y, width, np.minimum) + + +def _max_filter(y: np.ndarray, width: int) -> np.ndarray: + return _rank_filter(y, width, np.maximum) + + +def estimate_baseline(y: np.ndarray, window_samples: int, smooth) -> np.ndarray: + """Morphological opening baseline; removes peaks narrower than the window.""" + + width = max(0, int(window_samples)) + if width == 0 or np.size(y) <= 2: + return np.zeros_like(np.asarray(y, dtype=float)) + opened = _max_filter(_min_filter(np.asarray(y, dtype=float), width), width) + return smooth(opened.astype(np.float32), max(1, width // 2)).astype(float) + + +def estimate_noise(net: np.ndarray) -> float: + """Robust noise proxy from the median absolute first difference.""" + + if np.size(net) < 3: + return 0.0 + diff = np.diff(np.asarray(net, dtype=float)) + mad = float(np.median(np.abs(diff - np.median(diff)))) + return mad / (0.6745 * math.sqrt(2.0)) + + +def parabola_center(x: np.ndarray, y: np.ndarray, index: int) -> float: + """Sub-sample vertex of the parabola through three samples (NaN if unusable).""" + + if index <= 0 or index >= y.size - 1: + return float("nan") + y0, y1, y2 = float(y[index - 1]), float(y[index]), float(y[index + 1]) + denominator = y0 - 2.0 * y1 + y2 + if not np.isfinite(denominator) or denominator == 0.0: + return float("nan") + delta = 0.5 * (y0 - y2) / denominator + if not np.isfinite(delta) or abs(delta) > 1.0: + return float("nan") + step = float(x[index + 1] - x[index]) + return float(x[index] + delta * step) + + +def half_max_crossings(x: np.ndarray, net: np.ndarray, index: int, + max_span: int) -> tuple[float, float]: + """Linearly interpolated x positions where the net signal falls to half max.""" + + peak = float(net[index]) + if not np.isfinite(peak) or peak <= 0.0: + return float("nan"), float("nan") + half = 0.5 * peak + span = max(1, int(max_span)) + left = index + floor_index = max(0, index - span) + while left > floor_index and net[left] > half: + left -= 1 + right = index + ceiling = min(net.size - 1, index + span) + while right < ceiling and net[right] > half: + right += 1 + + left_x = right_x = float("nan") + if left < index and net[left] <= half: + rise = float(net[left + 1] - net[left]) + left_x = float(x[left] + (half - net[left]) / rise * (x[left + 1] - x[left])) \ + if rise > 0 else float(x[left]) + if right > index and net[right] <= half: + fall = float(net[right - 1] - net[right]) + right_x = float(x[right] - (net[right] - half) / fall * (x[right] - x[right - 1])) \ + if fall > 0 else float(x[right]) + return left_x, right_x + + +def _core(x: np.ndarray, amplitude: float, center: float, width: float, + shape: str) -> np.ndarray: + offset = x - center + if shape == "lorentz": + return amplitude / (1.0 + (offset / width) ** 2) + return amplitude * np.exp(-0.5 * (offset / width) ** 2) + + +def _shape_model(x: np.ndarray, params: np.ndarray, xref: float, shape: str) -> np.ndarray: + amplitude, center, width, c0, c1 = params + return _core(x, amplitude, center, width, shape) + c0 + c1 * (x - xref) + + +def _multi_model(x: np.ndarray, params: np.ndarray, xref: float, shape: str, + peaks: int) -> np.ndarray: + """1..N shape components plus a linear background; params = 3*N + 2.""" + + total = np.zeros_like(x, dtype=float) + for index in range(peaks): + amplitude, center, width = params[3 * index:3 * index + 3] + total = total + _core(x, amplitude, center, width, shape) + return total + params[3 * peaks] + params[3 * peaks + 1] * (x - xref) + + +def fit_profile(x: np.ndarray, y: np.ndarray, start: list[float], shape: str, + bounds: list[tuple[float, float]], max_iter: int = 200, + peaks: int = 1, shared_width: bool = False) -> dict: + """Fit 1..N peak components + linear background; returns parameters and flags. + + ``shared_width`` fits one common line width for all components, which is the + physically sensible and numerically stable model for an unresolved doublet. + Parameter layout: per component ``A, mu(, sigma)`` then ``c0, c1``; with a + shared width the single ``sigma`` is appended before ``c0, c1``. + """ + + x = np.asarray(x, dtype=float) + y = np.asarray(y, dtype=float) + peaks = int(peaks) + per_peak = 2 if shared_width else 3 + background = per_peak * peaks + if x.size < background + 3 or not np.all(np.isfinite(y)): + return {"ok": False} + xref = float(np.mean(x)) + lower = np.array([bound[0] for bound in bounds], dtype=float) + upper = np.array([bound[1] for bound in bounds], dtype=float) + params = np.clip(np.asarray(start, dtype=float), lower, upper) + + def model(values, parameters): + shared = parameters[2 * peaks] if shared_width else None + total = np.zeros_like(values, dtype=float) + for index in range(peaks): + if shared_width: + amplitude, center = parameters[2 * index], parameters[2 * index + 1] + width = shared + else: + amplitude, center, width = parameters[3 * index:3 * index + 3] + total = total + _core(values, amplitude, center, width, shape) + return total + parameters[background] + parameters[background + 1] * (values - xref) + + residual = y - model(x, params) + cost = float(np.sum(residual * residual)) + if not np.isfinite(cost): + return {"ok": False} + lam = 1e-3 + converged = False + for _ in range(max_iter): + base = model(x, params) + jacobian = np.empty((x.size, params.size)) + for j in range(params.size): + step = 1e-6 * max(1.0, abs(params[j])) + probe = params.copy() + probe[j] += step + jacobian[:, j] = (model(x, probe) - base) / step + normal = jacobian.T @ jacobian + gradient = jacobian.T @ residual + diagonal = np.diag(np.diag(normal)) + 1e-12 * np.eye(params.size) + relative = 1.0 + stepped = False + for _ in range(40): + try: + delta = np.linalg.solve(normal + lam * diagonal, gradient) + except np.linalg.LinAlgError: + lam *= 10.0 + continue + trial = np.clip(params + delta, lower, upper) + trial_residual = y - model(x, trial) + trial_cost = float(np.sum(trial_residual * trial_residual)) + if np.isfinite(trial_cost) and trial_cost < cost: + relative = (cost - trial_cost) / max(cost, 1e-30) + params, residual, cost = trial, trial_residual, trial_cost + lam = max(lam / 3.0, 1e-15) + stepped = True + break + lam *= 10.0 + if not stepped or relative < 1e-12: + converged = True + break + + rms = float(np.sqrt(cost / max(1, x.size))) + # Parameter errors from the normal matrix at the final parameters; scale the + # columns first because amplitude, centre and width differ by orders of magnitude. + try: + base = model(x, params) + jacobian = np.empty((x.size, params.size)) + for j in range(params.size): + step = 1e-6 * max(1.0, abs(params[j])) + probe = params.copy() + probe[j] += step + jacobian[:, j] = (model(x, probe) - base) / step + normal = jacobian.T @ jacobian + scale = np.sqrt(np.maximum(np.diag(normal), 1e-300)) + correlation = normal / np.outer(scale, scale) + try: + inverse = np.linalg.inv(correlation) + except np.linalg.LinAlgError: + inverse = np.linalg.pinv(correlation) + variance = cost / max(1, x.size - params.size) + errors = np.sqrt(np.clip(np.diag(inverse) * variance, 0.0, None)) / scale + except (np.linalg.LinAlgError, ValueError, FloatingPointError): + errors = np.full(params.size, float("nan")) + components = [] + usable = bool(converged and np.isfinite(params).all()) + for index in range(peaks): + if shared_width: + amplitude = float(params[2 * index]) + center = float(params[2 * index + 1]) + width = float(params[2 * peaks]) + low, high = lower[2 * peaks], upper[2 * peaks] + amplitude_err = float(errors[2 * index]) + center_err = float(errors[2 * index + 1]) + width_err = float(errors[2 * peaks]) + else: + amplitude = float(params[3 * index]) + center = float(params[3 * index + 1]) + width = float(params[3 * index + 2]) + low, high = lower[3 * index + 2], upper[3 * index + 2] + amplitude_err = float(errors[3 * index]) + center_err = float(errors[3 * index + 1]) + width_err = float(errors[3 * index + 2]) + width_at_bound = bool(width <= low * (1 + 1e-6) or width >= high * (1 - 1e-6)) + if amplitude <= 0 or width <= 0 or width_at_bound: + usable = False + if not x[0] - 1.0 <= center <= x[-1] + 1.0: + usable = False + fwhm = 2.0 * width if shape == "lorentz" else GAUSS_TO_FWHM * width + components.append({"amplitude": amplitude, "center": center, + "width": width, "fwhm": float(fwhm), + "amplitude_err": amplitude_err, "center_err": center_err, + "width_err": width_err, + "width_at_bound": width_at_bound}) + first = components[0] + return { + "ok": usable, + "peaks": peaks, + "params": params.tolist(), + "xref": xref, + "components": components, + "center": first["center"], + "width": first["width"], + "amplitude": first["amplitude"], + "fwhm": first["fwhm"], + "c0": float(params[background]), + "c1": float(params[background + 1]), + "rms": rms, + "converged": converged, + "at_bound": any(item["width_at_bound"] for item in components), + "n_points": int(x.size), + } + + +def model_values(x, params, xref: float, shape: str) -> np.ndarray: + if not params: + return np.full(np.shape(x), np.nan, dtype=float) + return _shape_model(np.asarray(x, dtype=float), np.asarray(params, dtype=float), xref, shape) + + +def peak_prominence(net: np.ndarray, index: int) -> tuple[float, float]: + """True topographic prominence of a local maximum and the level it sits on. + + Walk outwards until the profile rises above the peak or the array ends; the + prominence base is the higher of the two minima on the way. Unlike a fixed + window this keeps resolved doublets apart and rejects shoulders. + """ + + height = float(net[index]) + left = index + left_min = height + while left > 0: + left -= 1 + value = float(net[left]) + if value > height: + break + if value < left_min: + left_min = value + right = index + right_min = height + while right < net.size - 1: + right += 1 + value = float(net[right]) + if value > height: + break + if value < right_min: + right_min = value + base = max(left_min, right_min) + return height - base, base + + +def find_peak_candidates(net: np.ndarray, min_distance: int, min_height: float, + min_prominence: float, max_peaks: int) -> list[dict]: + """Local maxima with plateau centring, true prominence and neighbour suppression.""" + + n = net.size + if n < 3 or max_peaks <= 0: + return [] + candidates = [] + index = 1 + while index < n - 1: + if net[index] > net[index - 1]: + end = index + while end + 1 < n and net[end + 1] == net[index]: + end += 1 + if end + 1 < n and net[end] > net[end + 1]: + candidates.append((index + end) // 2) + index = end + 1 + else: + index += 1 + if not candidates: + return [] + + candidates.sort(key=lambda i: float(net[i]), reverse=True) + kept: list[dict] = [] + for index in candidates: + height = float(net[index]) + if height < min_height: + continue + if any(abs(index - other["sample"]) < max(1, int(min_distance)) for other in kept): + continue + prominence, base = peak_prominence(net, index) + if prominence < min_prominence: + continue + kept.append({"sample": int(index), "height_adu": height, + "base_adu": base, "prominence_adu": float(prominence)}) + if len(kept) >= max_peaks: + break + return kept + + +def refine_peak(x: np.ndarray, net: np.ndarray, raw: np.ndarray, candidate: dict, + noise: float, step_px: float, fwhm_span: int, fit_window: int, + shape: str, saturated: bool, baseline_value: float = 0.0) -> dict: + """Measure one detected peak: centre, width, height, area and fit quality.""" + + index = candidate["sample"] + n = net.size + left_x, right_x = half_max_crossings(x, net, index, fwhm_span) + fwhm_measured = (right_x - left_x + if np.isfinite(left_x) and np.isfinite(right_x) else float("nan")) + + if fit_window > 0: + window_px = float(fit_window) * step_px + elif np.isfinite(fwhm_measured): + window_px = float(np.clip(2.0 * fwhm_measured, 8.0 * step_px, 90.0 * step_px)) + else: + window_px = 15.0 * step_px + lower = int(index) + while lower > 0 and x[index] - x[lower] < window_px: + lower -= 1 + upper = int(index) + while upper < n - 1 and x[upper] - x[index] < window_px: + upper += 1 + + center_parabola = parabola_center(x, net, index) + + start_center = center_parabola if np.isfinite(center_parabola) else float(x[index]) + start_width = fwhm_measured / (2.0 if shape == "lorentz" else GAUSS_TO_FWHM) + if not np.isfinite(start_width) or start_width <= 0: + start_width = 2.0 * step_px + span_px = float(x[upper] - x[lower]) + bounds = [ + (0.0, 10.0 * candidate["height_adu"] + 1.0), + (x[lower] - span_px, x[upper] + span_px), + (0.4 * step_px, max(4.0 * start_width, 0.5 * span_px)), + (-5.0 * candidate["height_adu"] - 1.0, 5.0 * candidate["height_adu"] + 1.0), + (-1e4, 1e4), + ] + fit = fit_profile(x[lower:upper + 1], net[lower:upper + 1], + [candidate["height_adu"], start_center, start_width, 0.0, 0.0], + shape, bounds) + if fit.get("ok") and (fit["amplitude"] < 0.25 * candidate["height_adu"] + or fit["rms"] > 0.2 * fit["amplitude"]): + # Reject a fit that misses the measured peak: keep the robust estimators. + fit["ok"] = False + + if fit.get("ok"): + center, fwhm, source = fit["center"], fit["fwhm"], "fit" + elif np.isfinite(center_parabola): + # Model-free vertex of the measured maximum: still a peak position. + center, fwhm, source = center_parabola, fwhm_measured, "apex" + else: + # Flat or saturated top makes the parabola unusable; report the sample + # where the maximum was detected rather than a window-dependent moment. + center, fwhm, source = float(x[index]), fwhm_measured, "sample" + + height = float(net[index]) + if fit.get("ok") and fit["amplitude"] > height: + height = float(fit["amplitude"]) + return { + "sample_index": int(index), + "sensor_x_px": float(x[index]), + "center_px": float(center), + "center_source": source, + "center_parabola_px": float(center_parabola) if np.isfinite(center_parabola) else float("nan"), + "center_fit_px": float(fit["center"]) if fit.get("ok") else float("nan"), + "center_err_px": float(fit["components"][0]["center_err"]) if fit.get("ok") else float("nan"), + "sigma_px": float(fit["width"]) if fit.get("ok") else float("nan"), + "fwhm_px": float(fwhm) if np.isfinite(fwhm) else float("nan"), + "fwhm_halfmax_px": float(fwhm_measured) if np.isfinite(fwhm_measured) else float("nan"), + "height_net_adu": height, + "prominence_adu": float(candidate["prominence_adu"]), + "raw_adu": float(raw[index]), + "baseline_adu": float(baseline_value), + "snr": float(height / noise) if noise > 0 else float("inf"), + "area_net_adu_px": float(np.sum(np.maximum(net[lower:upper + 1], 0.0)) * step_px), + "saturated": bool(saturated), + "fit_ok": bool(fit.get("ok")), + "fit_rms_adu": float(fit["rms"]) if fit.get("ok") else float("nan"), + "fit_shape": shape, + "fit_params": fit.get("params") if fit.get("ok") else None, + "fit_xref": fit.get("xref") if fit.get("ok") else None, + "deblended": False, + "window_lo_px": float(x[lower]), + "window_hi_px": float(x[upper]), + "window_lo_sample": int(lower), + "window_hi_sample": int(upper), + } + + +def _fit_pair(x: np.ndarray, net: np.ndarray, first: dict, second: dict, + shape: str, step_px: float, noise: float) -> Optional[list[dict]]: + """Joint two-component fit of a blended pair; None if it is not trustworthy.""" + + lower = min(first["window_lo_sample"], second["window_lo_sample"]) + upper = max(first["window_hi_sample"], second["window_hi_sample"]) + xx = x[lower:upper + 1] + yy = net[lower:upper + 1] + if xx.size < 11: + return None + starts: list[float] = [] + bounds: list[tuple[float, float]] = [] + separation = abs(float(second["sensor_x_px"]) - float(first["sensor_x_px"])) + widths = [] + for peak in (first, second): + height = max(float(peak["height_net_adu"]), 1.0) + # Start from the detected maximum, not from the biased single-peak fit. + center = float(peak["sensor_x_px"]) + width = float(peak["sigma_px"]) if np.isfinite(peak["sigma_px"]) and peak["sigma_px"] > 0 \ + else 2.0 * step_px + widths.append(width) + starts.extend([height, center]) + bounds.extend([(0.0, 10.0 * height + 1.0), + (center - 3.5 * step_px, center + 3.5 * step_px)]) + # One shared line width: two components of the same blend come from the same + # instrument, and independent widths make the fit ill-conditioned. + shared = max(0.4 * step_px, min(max(widths), 0.5 * max(separation, step_px))) + starts.append(shared) + bounds.append((0.4 * step_px, max(8.0 * max(widths), 4.0 * step_px))) + scale = max(float(first["height_net_adu"]), float(second["height_net_adu"]), 1.0) + starts.extend([0.0, 0.0]) + bounds.extend([(-3.0 * scale, 3.0 * scale), (-1e4, 1e4)]) + + fit = fit_profile(xx, yy, starts, shape, bounds, peaks=2, shared_width=True) + if not fit.get("ok"): + return None + components = fit["components"] + if components[1]["center"] - components[0]["center"] < 0.5 * step_px: + # The two components collapsed onto each other: this is one feature. + return None + if abs(components[0]["center"] - first["sensor_x_px"]) > 3.5 * step_px: + return None + if abs(components[1]["center"] - second["sensor_x_px"]) > 3.5 * step_px: + return None + + updated = [] + for peak, component in zip((first, second), components): + merged = dict(peak) + merged.update({ + "center_px": float(component["center"]), + "center_fit_px": float(component["center"]), + "center_err_px": float(component["center_err"]), + "center_source": "pair", + "sigma_px": float(component["width"]), + "fwhm_px": float(component["fwhm"]), + "height_net_adu": float(component["amplitude"]), + "snr": float(component["amplitude"] / noise) if noise > 0 else float("inf"), + "fit_ok": True, + "fit_rms_adu": float(fit["rms"]), + "fit_params": [component["amplitude"], component["center"], + component["width"], 0.0, 0.0], + "fit_xref": float(fit["xref"]), + "deblended": True, + }) + updated.append(merged) + return updated + + +def deblend_close_peaks(x: np.ndarray, net: np.ndarray, peaks: list[dict], shape: str, + step_px: float, noise: float, blend_factor: float = 1.2) -> list[dict]: + """Refit unresolved pairs jointly instead of pulling their centres together.""" + + ordered = sorted(peaks, key=lambda item: item["sensor_x_px"]) + result: list[dict] = [] + index = 0 + while index < len(ordered): + current = ordered[index] + if index + 1 < len(ordered): + neighbour = ordered[index + 1] + width_a = current["fwhm_px"] if np.isfinite(current["fwhm_px"]) else 4.0 * step_px + width_b = neighbour["fwhm_px"] if np.isfinite(neighbour["fwhm_px"]) else 4.0 * step_px + # Use the detected maxima: the fitted centres of a blend can cross. + separation = float(neighbour["sensor_x_px"] - current["sensor_x_px"]) + if 0.0 < separation <= blend_factor * 0.5 * (width_a + width_b): + pair = _fit_pair(x, net, current, neighbour, shape, step_px, noise) + if pair is not None: + result.extend(pair) + index += 2 + continue + result.append(current) + index += 1 + return result + + +def dedupe_channel_peaks(peaks: list[dict], min_distance_px: float) -> list[dict]: + """Drop repeated maxima closer than the peak distance, using detection positions. + + The fitted centre of a blended line is pulled towards its neighbour, so the + distance is measured between the raw local maxima (``sensor_x_px``) and not + between the refined centres; otherwise a resolved doublet can collapse. + """ + + kept: list[dict] = [] + for peak in sorted(peaks, key=lambda item: item["height_net_adu"], reverse=True): + if any(abs(peak["sensor_x_px"] - other["sensor_x_px"]) < min_distance_px + for other in kept): + continue + kept.append(peak) + kept.sort(key=lambda item: item["sensor_x_px"]) + return kept + + +def summarise_line(members: dict[str, dict]) -> dict: + """Combine the per-channel peaks of one line into a line record (no id).""" + + for channel, item in members.items(): + item["channel"] = channel + items = list(members.values()) + strongest = max((item["height_net_adu"] for item in items), default=0.0) + weights = np.array([ + max(0.0, item["height_net_adu"]) if item["height_net_adu"] >= 0.25 * strongest else 0.0 + for item in items + ], dtype=float) + positions = np.array([item["center_px"] for item in items], dtype=float) + center = (float(np.dot(weights, positions) / weights.sum()) + if weights.sum() > 0 else float(np.mean(positions))) + primary = max(items, key=lambda item: item["snr"]) + return { + "sensor_x_px": center, + "center_spread_px": float(positions.max() - positions.min()), + "channels": members, + "channel_names": "".join(c for c in ("R", "G", "B") if c in members), + "n_channels": len(members), + "primary_channel": primary["channel"], + "height_net_adu": primary["height_net_adu"], + "max_height_net_adu": float(strongest), + "snr": primary["snr"], + "fwhm_px": primary["fwhm_px"], + "area_net_adu_px": primary["area_net_adu_px"], + "saturated": any(item["saturated"] for item in items), + "sample_index": primary["sample_index"], + "sample_step_px": None, + } + + +def merge_coincident_lines(lines: list[dict], tolerance_px: float = 1.5) -> list[dict]: + """Join lines that ended up at the same place in complementary channels. + + Weak lines can be detected a few pixels apart in different CFA channels; if + the refined centres agree to better than one sample they are one line. + """ + + merged: list[dict] = [] + for line in sorted(lines, key=lambda item: item["sensor_x_px"]): + target = None + for other in merged: + if abs(line["sensor_x_px"] - other["sensor_x_px"]) < tolerance_px: + target = other + break + if target is None: + merged.append({"sensor_x_px": line["sensor_x_px"], + "channels": dict(line["channels"])}) + continue + combined = dict(target["channels"]) + for channel, peak in line["channels"].items(): + other = combined.get(channel) + if other is None or peak["snr"] > other["snr"]: + combined[channel] = peak + target["channels"] = combined + + summaries = [summarise_line(item["channels"]) for item in merged] + summaries.sort(key=lambda entry: entry["sensor_x_px"]) + for index, summary in enumerate(summaries, 1): + summary["id"] = f"P{index:03d}" + return summaries + + +def group_channels(peaks: dict[str, list[dict]], tolerance_px: float, + merge_tolerance_px: float = 1.5) -> list[dict]: + """Merge per-channel peaks that belong to the same spectral line.""" + + # Cluster on the detected local maxima, not on the fitted centres: a single + # peak fit inside a blend is pulled towards its neighbour and would wrongly + # merge two resolved lines. + flat = [(item["sensor_x_px"], channel, item) + for channel, items in peaks.items() for item in items] + flat.sort(key=lambda row: row[0]) + groups: list[list[tuple[float, str, dict]]] = [] + for entry in flat: + if groups and entry[0] - groups[-1][-1][0] <= tolerance_px: + groups[-1].append(entry) + else: + groups.append([entry]) + + lines = [] + for group in groups: + members: dict[str, dict] = {} + for _, channel, item in group: + other = members.get(channel) + # One line cannot have two centres in the same CFA channel. + if other is None or item["snr"] > other["snr"]: + members[channel] = item + summary = summarise_line(dict(members)) + summary["id"] = "" + lines.append(summary) + return merge_coincident_lines(lines, merge_tolerance_px) + + +def attach_wavelength(lines: list[dict], calibration: Optional[Calibration]) -> None: + """Add wavelength, extrapolation flag and per-channel wavelength data.""" + + for line in lines: + for item in line["channels"].values(): + if calibration is not None: + item["wavelength_nm"] = float(calibration(item["center_px"])) + item["extrapolated"] = bool( + item["center_px"] < calibration.span[0] or item["center_px"] > calibration.span[1]) + item["dispersion_nm_per_px"] = float(calibration.dispersion(item["center_px"])) + item["fwhm_nm"] = float(abs(item["fwhm_px"] * item["dispersion_nm_per_px"])) + else: + item["wavelength_nm"] = float("nan") + item["extrapolated"] = False + item["dispersion_nm_per_px"] = float("nan") + item["fwhm_nm"] = float("nan") + if calibration is not None: + line["wavelength_nm"] = float(calibration(line["sensor_x_px"])) + line["extrapolated"] = bool( + line["sensor_x_px"] < calibration.span[0] or line["sensor_x_px"] > calibration.span[1]) + line["dispersion_nm_per_px"] = float(calibration.dispersion(line["sensor_x_px"])) + line["fwhm_nm"] = float(abs(line["fwhm_px"] * line["dispersion_nm_per_px"])) + else: + line["wavelength_nm"] = float("nan") + line["extrapolated"] = False + line["dispersion_nm_per_px"] = float("nan") + line["fwhm_nm"] = float("nan") + line.update({"match_species": "", "match_nm": float("nan"), + "match_delta_nm": float("nan"), "match_label": ""}) + line["x_plot"] = line["wavelength_nm"] if calibration is not None else line["sensor_x_px"] diff --git a/test_peak_finder.py b/test_peak_finder.py new file mode 100644 index 0000000..490d2b2 --- /dev/null +++ b/test_peak_finder.py @@ -0,0 +1,551 @@ +import csv +import json +import os +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +import numpy as np + +import peak_finder as pf +import spectro_core as core + +HERE = Path(__file__).resolve().parent +TEST_TMP = os.environ.get("PEAK_FINDER_TEST_TMP") + + +def _find_recording(name: str) -> Path: + """Look for recordings/ here and one or two levels up. + + The recordings often live beside the program folder (Light/recordings vs + Light/Programs), so a fixed path would silently skip the end-to-end tests. + """ + + for base in (HERE, HERE.parent, HERE.parent.parent): + candidate = base / "recordings" / name + if candidate.is_dir(): + return candidate + return HERE / "recordings" / name + + +# Optional overrides let the end-to-end tests run against a recording kept elsewhere. +RECORDING = Path(os.environ.get("PEAK_FINDER_TEST_RECORDING", + _find_recording("hq_20261101_114417"))) +CALIBRATION = Path(os.environ.get("PEAK_FINDER_TEST_CALIBRATION", + HERE / "wavelength_calibration" / "calibration.json")) +MODULE_DIR = Path(os.environ.get("PEAK_FINDER_TEST_MODULE_DIR", HERE)) +AVAILABLE = RECORDING.is_dir() and CALIBRATION.is_file() + + +def read_rows(path) -> list[dict]: + with open(path, newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +class temp_directory: + """Scratch directory context manager; PEAK_FINDER_TEST_TMP overrides the location. + + A plain ``mkdir`` is used instead of ``tempfile.mkdtemp`` so the directory + keeps the parent's permissions, which matters inside restrictive sandboxes. + """ + + def __init__(self): + import shutil + import uuid + + self._shutil = shutil + base = Path(TEST_TMP) if TEST_TMP else Path(tempfile.gettempdir()) + base.mkdir(parents=True, exist_ok=True) + self.path = base / f"pf_case_{uuid.uuid4().hex[:10]}" + self.path.mkdir() + + def __enter__(self) -> str: + return str(self.path) + + def __exit__(self, *exc_info) -> bool: + self._shutil.rmtree(self.path, ignore_errors=True) + return False + + +def gaussian(x, amplitude, center, sigma, base=0.0, slope=0.0, xref=None): + xref = float(np.mean(x)) if xref is None else xref + return base + slope * (x - xref) + amplitude * np.exp(-0.5 * ((x - center) / sigma) ** 2) + + +def make_peak(center, height, fwhm, snr, channel="G", **extra): + peak = { + "channel": channel, + "sample_index": int(round(center / 2.0)), + "sensor_x_px": float(round(center / 2.0) * 2.0), + "center_px": float(center), + "center_source": "fit", + "center_parabola_px": float(center) + 0.3, + "center_fit_px": float(center), + "sigma_px": fwhm / 2.3548200450309493, + "fwhm_px": fwhm, + "fwhm_halfmax_px": fwhm * 1.02, + "height_net_adu": height, + "prominence_adu": height * 0.9, + "raw_adu": height + 100.0, + "baseline_adu": 10.0, + "snr": snr, + "area_net_adu_px": height * fwhm, + "saturated": False, + "fit_ok": True, + "fit_rms_adu": height * 0.02, + "fit_shape": "gauss", + "fit_params": [height, float(center), fwhm / 2.3548200450309493, 10.0, 0.0], + "fit_xref": float(center), + "window_lo_px": float(center) - 20.0, + "window_hi_px": float(center) + 20.0, + "window_lo_sample": 0, + "window_hi_sample": 10, + } + peak.update(extra) + return peak + + +def make_lines(): + wavelengths = [404.7, 435.8, 546.1, 577.0, 579.1] + return [ + {"id": f"P{index:03d}", "wavelength_nm": value, "sensor_x_px": 200.0 + 100.0 * index, + "snr": float(10 * index), "center_spread_px": 0.5, "extrapolated": False} + for index, value in enumerate(wavelengths, 1) + ] + + +class EstimatorTests(unittest.TestCase): + def test_baseline_follows_continuum_not_peaks(self): + x = np.arange(0, 600, 2.0) + y = 100.0 + 0.05 * x + gaussian(x, 500.0, 300.0, 5.0) + baseline = pf.estimate_baseline(y, 50, lambda values, width: values) + error = np.abs(baseline - (100.0 + 0.05 * x))[60:-60] + self.assertLess(float(np.max(error)), 5.0) + + def test_baseline_disabled_returns_zero(self): + y = np.arange(10.0) + np.testing.assert_array_equal(pf.estimate_baseline(y, 0, lambda v, w: v), np.zeros(10)) + + def test_noise_proxy_of_white_noise(self): + rng = np.random.default_rng(7) + y = rng.normal(0.0, 7.0, 4000) + self.assertAlmostEqual(pf.estimate_noise(y), 7.0, delta=0.5) + + def test_parabola_and_half_max_width(self): + x = np.arange(0, 200, 2.0) + y = gaussian(x, 1000.0, 101.0, 4.0) + index = int(np.argmax(y)) + self.assertAlmostEqual(pf.parabola_center(x, y, index), 101.0, delta=0.5) + left, right = pf.half_max_crossings(x, y, index, 40) + self.assertAlmostEqual(right - left, 2.3548200450309493 * 4.0, delta=0.6) + + +class FitTests(unittest.TestCase): + def test_gaussian_fit_recovers_parameters(self): + rng = np.random.default_rng(11) + x = np.arange(600.0, 700.0, 2.0) + truth = gaussian(x, 1200.0, 651.7, 3.4, base=40.0, slope=0.8, xref=650.0) + y = truth + rng.normal(0.0, 8.0, x.size) + bounds = [(0.0, 20000.0), (580.0, 720.0), (0.8, 40.0), (-2000.0, 2000.0), (-100.0, 100.0)] + fit = pf.fit_profile(x, y, [1000.0, 650.0, 4.0, 0.0, 0.0], "gauss", bounds) + self.assertTrue(fit["ok"]) + self.assertAlmostEqual(fit["center"], 651.7, delta=0.4) + self.assertAlmostEqual(fit["fwhm"], 2.3548200450309493 * 3.4, delta=0.6) + self.assertAlmostEqual(fit["amplitude"], 1200.0, delta=120.0) + + def test_lorentzian_fit_recovers_width(self): + rng = np.random.default_rng(3) + x = np.arange(0.0, 120.0, 2.0) + y = 900.0 / (1.0 + ((x - 61.2) / 3.0) ** 2) + rng.normal(0.0, 5.0, x.size) + bounds = [(0.0, 10000.0), (20.0, 100.0), (1.0, 20.0), (-500.0, 500.0), (-50.0, 50.0)] + fit = pf.fit_profile(x, y, [900.0, 60.0, 3.0, 0.0, 0.0], "lorentz", bounds) + self.assertTrue(fit["ok"]) + self.assertAlmostEqual(fit["fwhm"], 6.0, delta=0.6) + + def test_model_values_round_trip(self): + x = np.arange(0.0, 10.0) + params = [5.0, 4.0, 1.5, 1.0, 0.0] + model = pf.model_values(x, params, 4.0, "gauss") + self.assertAlmostEqual(float(np.max(model)), 6.0, delta=1e-6) + self.assertTrue(np.all(np.isfinite(model))) + + +class DetectionTests(unittest.TestCase): + def test_three_peaks_found_in_continuum(self): + rng = np.random.default_rng(5) + x = np.arange(0.0, 800.0, 2.0) + y = (0.02 * x + gaussian(x, 1000.0, 200.0, 4.0) + gaussian(x, 400.0, 400.0, 6.0) + + gaussian(x, 120.0, 600.0, 3.0) + rng.normal(0.0, 2.0, x.size)) + baseline = pf.estimate_baseline(y, 50, lambda values, width: values) + net = y - baseline + noise = pf.estimate_noise(net) + candidates = pf.find_peak_candidates(net, 4, 20.0, 20.0, 10) + centers = sorted(x[item["sample"]] for item in candidates) + self.assertEqual(len(centers), 3) + for found, expected in zip(centers, (200.0, 400.0, 600.0)): + self.assertAlmostEqual(found, expected, delta=4.0) + self.assertLess(noise, 5.0) + + def test_refined_centres_are_sub_sample(self): + rng = np.random.default_rng(9) + x = np.arange(0.0, 200.0, 2.0) + y = gaussian(x, 800.0, 97.3, 2.6) + rng.normal(0.0, 4.0, x.size) + baseline = pf.estimate_baseline(y, 40, lambda values, width: values) + net = y - baseline + noise = pf.estimate_noise(net) + candidate = pf.find_peak_candidates(net, 4, 20.0, 20.0, 4)[0] + peak = pf.refine_peak(x, net, y, candidate, noise, 2.0, 30, 0, "gauss", False) + self.assertAlmostEqual(peak["center_px"], 97.3, delta=0.6) + self.assertAlmostEqual(peak["fwhm_px"], 2.3548200450309493 * 2.6, delta=0.8) + self.assertTrue(peak["fit_ok"]) + + def test_silent_profile_has_no_candidates(self): + net = np.zeros(50) + self.assertEqual(pf.find_peak_candidates(net, 4, 1.0, 1.0, 5), []) + + +class BlendTests(unittest.TestCase): + def test_failed_fit_falls_back_to_the_measured_maximum(self): + from unittest.mock import patch + + x = np.arange(400.0, 700.0, 2.0) + net = (gaussian(x, 10000.0, 542.0, 15.0) + + 4000.0 * np.exp(-np.clip(x - 542.0, 0.0, None) / 120.0)) + candidate = {"sample": int(np.argmax(net)), "height_adu": float(net.max()), + "base_adu": 0.0, "prominence_adu": float(net.max())} + with patch.object(core, "fit_profile", return_value={"ok": False}): + peak = pf.refine_peak(x, net, net, candidate, 1.0, 2.0, 30, 0, "gauss", False) + self.assertEqual(peak["center_source"], "apex") + self.assertAlmostEqual(peak["center_px"], 542.0, delta=1.0) + + def test_flat_top_falls_back_to_the_sampled_maximum(self): + from unittest.mock import patch + + x = np.arange(0.0, 60.0, 2.0) + net = np.full(x.size, 100.0) + net[10:16] = 1000.0 # saturated flat top + candidate = {"sample": 12, "height_adu": 1000.0, + "base_adu": 100.0, "prominence_adu": 900.0} + with patch.object(core, "fit_profile", return_value={"ok": False}): + peak = pf.refine_peak(x, net, net, candidate, 1.0, 2.0, 30, 0, "gauss", False) + self.assertEqual(peak["center_source"], "sample") + self.assertAlmostEqual(peak["center_px"], 24.0) # x[12] + + def test_topographic_prominence_rejects_a_shoulder(self): + net = np.array([0.0, 100.0, 200.0, 500.0, 300.0, 400.0, 200.0, 0.0]) + prominence, base = pf.peak_prominence(net, 5) + self.assertAlmostEqual(prominence, 100.0) + self.assertAlmostEqual(base, 300.0) + prominence, _ = pf.peak_prominence(net, 3) + self.assertAlmostEqual(prominence, 500.0) + + def test_resolved_doublet_survives_detection(self): + rng = np.random.default_rng(21) + x = np.arange(780.0, 830.0, 2.0) + net = (gaussian(x, 3600.0, 800.0, 2.5) + gaussian(x, 4300.0, 808.0, 2.5) + + rng.normal(0.0, 20.0, x.size)) + candidates = pf.find_peak_candidates(net, 2, 200.0, 100.0, 10) + self.assertEqual(len(candidates), 2) + self.assertEqual(sorted(round(float(x[item["sample"]]), 1) for item in candidates), + [800.0, 808.0]) + + def test_doublet_is_deblended_with_one_shared_width(self): + rng = np.random.default_rng(22) + x = np.arange(780.0, 830.0, 2.0) + net = (gaussian(x, 3600.0, 800.0, 2.5) + gaussian(x, 4300.0, 808.0, 2.5) + + rng.normal(0.0, 20.0, x.size)) + noise = pf.estimate_noise(net) + candidates = pf.find_peak_candidates(net, 2, 200.0, 100.0, 10) + peaks = [pf.refine_peak(x, net, net, item, noise, 2.0, 30, 0, "gauss", False) + for item in candidates] + deduped = pf.dedupe_channel_peaks(peaks, 4.0) + self.assertEqual(len(deduped), 2) + merged = pf.deblend_close_peaks(x, net, deduped, "gauss", 2.0, noise) + self.assertEqual(len(merged), 2) + self.assertTrue(all(peak["deblended"] for peak in merged)) + centres = sorted(peak["center_px"] for peak in merged) + self.assertAlmostEqual(centres[0], 800.0, delta=0.8) + self.assertAlmostEqual(centres[1], 808.0, delta=0.8) + self.assertEqual(len({round(peak["fwhm_px"], 6) for peak in merged}), 1) + self.assertAlmostEqual(merged[0]["fwhm_px"], 2.3548200450309493 * 2.5, delta=0.8) + for peak in merged: + self.assertTrue(np.isfinite(peak["center_err_px"])) + + def test_coincident_weak_lines_are_merged(self): + first = pf.summarise_line({"B": make_peak(912.0, 430.0, 9.2, 75.0, "B")}) + second = pf.summarise_line({"R": make_peak(917.0, 70.0, 8.0, 15.0, "R")}) + first["id"] = second["id"] = "" + lines = [dict(first, sensor_x_px=912.0), dict(second, sensor_x_px=917.0)] + self.assertEqual(len(pf.merge_coincident_lines(lines, 1.5)), 2) + lines[1]["sensor_x_px"] = 912.4 + combined = pf.merge_coincident_lines(lines, 1.5) + self.assertEqual(len(combined), 1) + self.assertEqual(combined[0]["channel_names"], "RB") + + +class GroupingTests(unittest.TestCase): + def test_same_line_in_three_channels_merges(self): + peaks = { + "R": [make_peak(500.0, 300.0, 9.0, 40.0, "R")], + "G": [make_peak(501.0, 1200.0, 10.0, 200.0, "G")], + "B": [make_peak(499.5, 800.0, 11.0, 90.0, "B")], + } + lines = pf.group_channels(peaks, 4.0) + self.assertEqual(len(lines), 1) + line = lines[0] + self.assertEqual(line["channel_names"], "RGB") + self.assertEqual(line["primary_channel"], "G") + self.assertEqual(line["id"], "P001") + self.assertAlmostEqual(line["sensor_x_px"], (300 * 500 + 1200 * 501 + 800 * 499.5) / 2300, + places=6) + self.assertAlmostEqual(line["center_spread_px"], 1.5, places=6) + + def test_distinct_lines_stay_separate(self): + peaks = {"G": [make_peak(500.0, 100.0, 8.0, 20.0), make_peak(520.0, 90.0, 8.0, 18.0)]} + lines = pf.group_channels(peaks, 4.0) + self.assertEqual([line["id"] for line in lines], ["P001", "P002"]) + + def test_channel_duplicate_is_dropped(self): + peaks = {"G": [make_peak(500.0, 300.0, 9.0, 50.0), make_peak(500.4, 900.0, 9.0, 90.0)]} + lines = pf.group_channels(peaks, 4.0) + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]["height_net_adu"], 900.0) + + def test_dedupe_keeps_neighbours_further_than_distance(self): + peaks = [make_peak(100.0, 500.0, 8.0, 40.0), make_peak(112.0, 400.0, 8.0, 30.0), + make_peak(103.0, 300.0, 8.0, 20.0)] + kept = pf.dedupe_channel_peaks(peaks, 12.0) + self.assertEqual([peak["center_px"] for peak in kept], [100.0, 112.0]) + + +class OutputNamingTests(unittest.TestCase): + def test_default_output_names_the_frame(self): + single = pf.default_output_directory(r"C:\raw\hq_20261112_200710", [7]) + self.assertEqual(single.name, "hq_20261112_200710_f0007") + stacked = pf.default_output_directory(r"C:\raw\hq_20261112_200710", [5, 6, 7]) + self.assertEqual(stacked.name, "hq_20261112_200710_f0005-0007_avg3") + self.assertEqual(stacked.parent.name, "peak_exports") + + +class SelectionTests(unittest.TestCase): + def setUp(self): + self.lines = make_lines() + + def test_all_and_none(self): + self.assertEqual(len(pf.parse_selection("all", self.lines)), 5) + self.assertEqual(pf.parse_selection("none", self.lines), set()) + self.assertEqual(pf.parse_selection("ALL", self.lines), pf.parse_selection("*", self.lines)) + + def test_ids_and_ranges(self): + self.assertEqual(pf.parse_selection("2", self.lines), {"P002"}) + self.assertEqual(pf.parse_selection("1-2", self.lines), {"P001", "P002"}) + self.assertEqual(pf.parse_selection("id:4", self.lines), {"P004"}) + + def test_wavelength_ranges(self): + self.assertEqual(pf.parse_selection("nm:540-560", self.lines), {"P003"}) + self.assertEqual(pf.parse_selection("540-560", self.lines), {"P003"}) + self.assertEqual(pf.parse_selection("546.0", self.lines), {"P003"}) + self.assertEqual(pf.parse_selection("nm:430-440", self.lines), {"P002"}) + + def test_top_n_and_union(self): + self.assertEqual(pf.parse_selection("top:2", self.lines), {"P004", "P005"}) + self.assertEqual(pf.parse_selection("1,nm:540-560", self.lines), {"P001", "P003"}) + + def test_bad_tokens_raise(self): + for token in ("9", "0", "abc", "id:3-9", "nm:"): + with self.assertRaises(ValueError): + pf.parse_selection(token, self.lines) + + def test_wavelength_selection_needs_calibration(self): + lines = [dict(line, wavelength_nm=float("nan")) for line in self.lines] + with self.assertRaises(ValueError): + pf.parse_selection("nm:500-600", lines) + self.assertEqual(pf.parse_selection("2", lines), {"P002"}) + + +class CalibrationTests(unittest.TestCase): + def write(self, payload): + handle = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False, encoding="utf-8", dir=TEST_TMP) + handle.write(json.dumps(payload)) + handle.close() + return handle.name + + def test_scale_offset_and_inverse(self): + path = self.write({"coefficients_descending": [0.285, 349.45], + "calibrated_pixel_range": [304.0, 808.5]}) + calibration = pf.Calibration(path, offset_nm=0.5) + calibration.set_width(2028) + self.assertAlmostEqual(float(calibration(600.0)), 0.285 * 600 + 349.45 + 0.5, places=6) + self.assertAlmostEqual(float(calibration.dispersion(600.0)), 0.285, places=9) + self.assertAlmostEqual(float(calibration.inverse(calibration(600.0))), 600.0, delta=1.0) + self.assertAlmostEqual(calibration.nm_span[0], 0.285 * 304 + 349.95, places=6) + + def test_invalid_calibrations(self): + for payload in ({"coefficients_descending": [0.0, 1.0], "calibrated_pixel_range": [1, 2]}, + {"coefficients_descending": [1.0], "calibrated_pixel_range": [1, 2]}, + {"coefficients_descending": [1.0, 2.0], "calibrated_pixel_range": [3, 2]}, + {"coefficients_descending": [1.0, 2.0]}): + with self.assertRaises(ValueError): + pf.Calibration(self.write(payload)) + + def test_geometry_checks(self): + path = self.write({"coefficients_descending": [0.285, 349.45], + "calibrated_pixel_range": [304.0, 808.5]}) + calibration = pf.Calibration(path) + source = SimpleNamespace(full_size=(2028, 1520), pattern="BGGR") + calibration.validate_geometry(source, lambda *a, **k: None) + bad = pf.Calibration(self.write({"coefficients_descending": [0.001, -2.0, 900.0], + "calibrated_pixel_range": [304.0, 808.5]})) + with self.assertRaises(ValueError): + bad.validate_geometry(source, lambda *a, **k: None) + + +class LineTableTests(unittest.TestCase): + def test_match_nearest_within_tolerance(self): + table = [{"wavelength_nm": 546.0, "species": "Hg I", "label": "546"}, + {"wavelength_nm": 579.1, "species": "Hg I", "label": "579"}] + lines = [{"id": "P001", "wavelength_nm": 546.4, "sensor_x_px": 1.0}, + {"id": "P002", "wavelength_nm": 600.0, "sensor_x_px": 2.0}] + pf.match_lines(lines, table, 1.5) + self.assertEqual(lines[0]["match_species"], "Hg I") + self.assertAlmostEqual(lines[0]["match_delta_nm"], 0.4, places=6) + self.assertEqual(lines[1]["match_species"], "") + self.assertTrue(np.isnan(lines[1]["match_delta_nm"])) + + def test_csv_table_with_flexible_columns(self): + with temp_directory() as folder: + path = Path(folder) / "lines.csv" + path.write_text("nm,element\n546.0735,Hg I\n435.8328,Hg I\n", encoding="utf-8") + table = pf.load_line_table(path) + self.assertEqual([entry["wavelength_nm"] for entry in table], [435.8328, 546.0735]) + + +def build_spectrum_and_lines(): + """Run the pipeline up to detection, for tests that need real peaks.""" + + hq = pf.load_hq(str(MODULE_DIR)) + args = pf.build_parser().parse_args(["--input", str(RECORDING), "--output", "unused", + "--quiet"]) + source = hq.RecordingSource(str(RECORDING)) + source.start() + image, original, indices, frame_meta = pf.read_frames(hq, source, args, lambda *a, **k: None) + roi = hq.detect_auto_roi(hq.make_preview(original, source.pattern), *source.full_size) + source.close() + meta = dict(source.base_meta) + meta.update({key: value for key, value in frame_meta.items() + if not str(key).startswith("_")}) + black, full = hq.resolve_levels(meta, source.bit_depth, args.black, args.full_scale) + spectrum = pf.SpectrumData(hq, source, image, original, indices, roi, args, black, full) + lines = pf.group_channels(spectrum.detect(), args.group_tol) + pf.attach_wavelength(lines, None) + return spectrum, lines + + +@unittest.skipUnless(AVAILABLE, "recordings or calibration not available") +class PickerTests(unittest.TestCase): + def test_hover_previews_and_click_selects(self): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.backend_bases import KeyEvent, MouseEvent + + spectrum, lines = build_spectrum_and_lines() + self.assertGreaterEqual(len(lines), 3) + target = lines[2] + seen = {} + + def fake_show(block=True): + figure = plt.gcf() + canvas = figure.canvas + axes = figure.axes[0] + x_display = float(axes.transData.transform((target["x_plot"], 0.0))[0]) + y_display = float(axes.transAxes.transform((0.0, 0.5))[1]) + canvas.callbacks.process( + "motion_notify_event", MouseEvent("motion_notify_event", canvas, + x_display, y_display)) + seen["axes"] = len(figure.axes) + seen["images"] = len(figure.axes[1].images) + seen["curves"] = len(figure.axes[2].lines) + seen["title"] = figure.axes[2].get_title() + canvas.callbacks.process( + "button_press_event", MouseEvent("button_press_event", canvas, + x_display, y_display, button=1)) + canvas.callbacks.process( + "key_press_event", KeyEvent("key_press_event", canvas, key="e")) + + real_show = plt.show + plt.show = fake_show + try: + chosen = pf.pick_peaks(spectrum, lines, None, set()) + finally: + plt.show = real_show + self.assertEqual(chosen, {target["id"]}) + self.assertEqual(seen["axes"], 3) # spectrum + RAW patch + zoom + self.assertGreaterEqual(seen["images"], 1) # RAW patch rendered + self.assertGreaterEqual(seen["curves"], 3) # R/G/B profiles drawn + self.assertIn(target["id"], seen["title"]) + + +@unittest.skipUnless(AVAILABLE, "recordings or calibration not available") +class EndToEndTests(unittest.TestCase): + def test_export_selected_peaks(self): + with temp_directory() as folder: + code = pf.main(["--input", str(RECORDING), "--calibration", str(CALIBRATION), + "--output", folder, "--select", "nm:530-560", + "--no-images", "--quiet", "--module-dir", str(MODULE_DIR)]) + self.assertEqual(code, 0) + output = Path(folder) + rows = read_rows(output / "selected_peaks.csv") + self.assertTrue(rows) + for row in rows: + self.assertGreaterEqual(float(row["wavelength_nm"]), 530.0) + self.assertLessEqual(float(row["wavelength_nm"]), 560.0) + self.assertTrue(0 <= float(row["patch_x0"]) < float(row["patch_x1"]) <= 2028) + self.assertTrue(0 <= float(row["patch_y0"]) < float(row["patch_y1"]) <= 1520) + self.assertTrue(0 <= float(row["sensor_x_px"])) + self.assertTrue((output / row["shape_csv"]).is_file()) + manifest = json.loads((output / "peaks.json").read_text(encoding="utf-8")) + all_rows = read_rows(output / "peaks_all.csv") + self.assertGreaterEqual(len(all_rows), len(manifest["peaks"])) + self.assertEqual(manifest["calibration"]["wavelength_offset_nm"], 0.0) + self.assertTrue(manifest["coordinate_convention"]["sensor_x_px"]) + + def test_without_calibration_only_pixels_are_exported(self): + with temp_directory() as folder: + code = pf.main(["--input", str(RECORDING), "--output", folder, "--select", "top:1", + "--no-images", "--quiet", "--module-dir", str(MODULE_DIR)]) + self.assertEqual(code, 0) + row = read_rows(Path(folder) / "selected_peaks.csv")[0] + self.assertEqual(row["wavelength_nm"], "") + self.assertGreater(float(row["sensor_x_px"]), 0.0) + manifest = json.loads((Path(folder) / "peaks.json").read_text(encoding="utf-8")) + self.assertIsNone(manifest["calibration"]) + + def test_bad_frame_is_rejected(self): + code = pf.main(["--input", str(RECORDING), "--frame", "100000", "--quiet", + "--output", str(Path(tempfile.gettempdir()) / "pf_unused"), + "--no-images", "--module-dir", str(MODULE_DIR)]) + self.assertEqual(code, 2) + + def test_wavelength_offset_shifts_the_scale(self): + with temp_directory() as folder: + code = pf.main(["--input", str(RECORDING), "--calibration", str(CALIBRATION), + "--wavelength-offset", "-10", "--output", folder, + "--select", "top:1", "--no-images", "--quiet", + "--module-dir", str(MODULE_DIR)]) + self.assertEqual(code, 0) + row = read_rows(Path(folder) / "selected_peaks.csv")[0] + self.assertAlmostEqual(float(row["wavelength_nm"]), 545.8237 - 10.0, delta=0.2) + + +if __name__ == "__main__": + unittest.main() + + + + + + + + diff --git a/test_wavelength_player.py b/test_wavelength_player.py new file mode 100644 index 0000000..dac3931 --- /dev/null +++ b/test_wavelength_player.py @@ -0,0 +1,65 @@ +import json +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import numpy as np +import wavelength_player as wp + + +def calibration(coeff=(.6, -8), span=(690, 927)): + data = dict(coefficients_descending=list(coeff), calibrated_pixel_range=list(span), + roi=[620, 620, 1034, 848]) + with patch.object(Path, 'read_text', return_value=json.dumps(data)): + return wp.SavedCalibration('dummy.json') + + +class PlayerTests(unittest.TestCase): + def test_exact_saved_coefficients(self): + model = calibration() + np.testing.assert_array_equal(model.coeff, [.6, -8]) + np.testing.assert_allclose(model([690, 742, 926.5]), [406, 437.2, 547.9]) + + def test_invalid_models(self): + for coeff, span in (([0, 3], [1, 2]), ([float('nan'), 1], [1, 2]), + ([1], [1, 2]), ([1, 2], [4, 3])): + with self.assertRaises(ValueError): + calibration(coeff, span) + + def test_quadratic_and_reverse_geometry(self): + source = SimpleNamespace(full_size=(2028, 1520), pattern='BGGR') + calibration([.00001, .5, 5]).validate_geometry(source) + calibration([-.5, 1000]).validate_geometry(source) + with self.assertRaises(ValueError): + calibration([.001, -1, 500]).validate_geometry(source) + + def test_geometry_mismatch(self): + model = calibration() + model.data['recording'] = 'reference' + with patch.object(Path, 'is_file', return_value=True), patch.object(Path, 'read_text', + return_value=json.dumps(dict(width=4056, height=3040))): + with self.assertRaises(ValueError): + model.validate_geometry(SimpleNamespace(full_size=(2028, 1520), pattern='BGGR')) + + def test_processing_signed_signal_and_sensor_coordinates(self): + hq = wp.load_player() + model = calibration() + wp.install_hooks(hq, model) + args = hq.parse_args(['--smooth', '1', '--black', '10']) + raw = np.full((16, 64), 8, dtype=np.uint16) + raw[:, 30:32] = 110 + roi = hq.make_roi(10, 0, 60, 16, 64, 16) + result = hq.process_frame(raw, {}, roi, 'BGGR', args, model, (64, 16), 'SBGGR12', 12) + for c in hq.CHANNELS: + self.assertEqual(result.signal_raw[c][0], -2) + np.testing.assert_array_equal(result.signal[c], result.signal_raw[c]) + np.testing.assert_allclose(result.x_plot[c], model(result.x_px[c])) + self.assertEqual(result.normalized[c].max(), 1) + self.assertGreaterEqual(result.x_px[c][0], 10) + + def test_help(self): + self.assertEqual(wp.main(['--help']), 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/wavelength_calibration.py b/wavelength_calibration.py new file mode 100644 index 0000000..90d5724 --- /dev/null +++ b/wavelength_calibration.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Independent Hg wavelength calibration; uses hq_spectrum recording/CFA APIs. + +Peak finding uses the same core as peak_finder.py: topographic prominence, +Gaussian centre with a peak-top fallback, and a joint two-line fit for blended +doublets (the Hg 576.96/579.07 pair is only ~6 px apart). Baseline subtraction +is used for detection and peak heights only; `signal_adu` still reports the +black-corrected value at the peak sample. + +Sensor coordinates are full-resolution columns. +""" +from __future__ import annotations +import argparse +import csv +import json +from pathlib import Path +import sys +import numpy as np +import hq_spectrum as hq +try: + import spectro_core as core +except ImportError as exc: # pragma: no cover - placement error + raise SystemExit('wavelength_calibration.py needs spectro_core.py in the same directory ' + '(or on sys.path)') from exc + +HG = (404.6563, 435.8328, 546.0735, 576.9598, 579.0663) +SOURCE = 'https://physics.nist.gov/PhysRefData/Handbook/Tables/mercurytable2.htm' + + +def parser(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--input', required=True, help='hq_record recording directory') + p.add_argument('--frame', type=int, default=0, help='zero-based frame index (default 0)') + p.add_argument('--roi', default='auto', help='auto or x0,y0,x1,y1; same as hq_spectrum') + p.add_argument('--black', type=float, help='black level in decoded ADU') + p.add_argument('--full-scale', type=float) + p.add_argument('--reduce', choices=['mean', 'median'], default='mean') + p.add_argument('--baseline-window', type=float, default=100, + help='baseline opening width in sensor px; 0 disables the baseline') + p.add_argument('--peak-distance', type=float, default=4, + help='minimum distance in sensor pixels per channel (2 px = one sample)') + p.add_argument('--peak-prominence', type=float, default=0.01, + help='topographic prominence as a fraction of the channel maximum') + p.add_argument('--peak-height', type=float, default=0.02, + help='peak height as a fraction of the channel maximum') + p.add_argument('--min-snr', type=float, default=15, help='peak height in units of the noise') + p.add_argument('--max-peaks', type=int, default=40, help='per channel') + p.add_argument('--fwhm-span', type=float, default=60, help='half-width search range in px') + p.add_argument('--fit-window', type=float, default=0, help='fit window half width in px; 0 auto') + p.add_argument('--shape', choices=['gauss', 'lorentz'], default='gauss') + p.add_argument('--assign', help='confirmed candidate identities, e.g. B1:404.6563,B2:435.8328,G3:546.0735') + p.add_argument('--order', type=int, choices=[1, 2], default=1) + p.add_argument('--headless', action='store_true', help='save candidates; fit only with explicit --assign') + p.add_argument('--output', default='wavelength_calibration', help='output directory') + return p + + +def extract(raw, meta, roi, pattern, depth, args): + planes = hq.extract_planes(raw, pattern, roi) + counts = {c: planes[c].shape[1] for c in hq.CHANNELS} + xs = hq.channel_x_positions(roi, pattern, counts['G'], min(counts['R'], counts['B'])) + black, full = hq.resolve_levels(meta, depth, args.black, args.full_scale) + window = 0 if args.baseline_window <= 0 else max(3, int(round(args.baseline_window / 2.0))) + curves, candidates = {}, [] + for c in hq.CHANNELS: + y = (np.median(planes[c], axis=0) if args.reduce == 'median' + else np.mean(planes[c], axis=0, dtype=np.float64)) - black + x = xs[c][:len(y)] + baseline = core.estimate_baseline(y, window, hq.smooth_vector) + net = y - baseline + noise = core.estimate_noise(net) + top = float(np.max(net)) if net.size else 0.0 + positive = np.maximum(net, 0.0) + norm = positive / positive.max() if positive.max() > 0 else np.zeros(net.size) + peaks = [] + if top > 0 and norm.size: + min_height = max(args.peak_height * top, args.min_snr * noise) + min_prominence = max(args.peak_prominence * top, 0.4 * args.min_snr * noise) + min_distance = max(1, int(round(args.peak_distance / 2.0))) + found = core.find_peak_candidates(net, min_distance, min_height, + min_prominence, args.max_peaks) + for candidate in found: + index = candidate['sample'] + peaks.append(core.refine_peak( + x, net, y, candidate, noise, 2.0, + max(1, int(round(args.fwhm_span / 2.0))), + max(0, int(round(args.fit_window / 2.0))), + args.shape, core.is_saturated(raw, roi, x[index], full), + baseline_value=float(baseline[index]))) + # Two lines closer than ~1.2 FWHM are fitted jointly, otherwise a + # single-peak fit collapses them onto one centre. + peaks = core.deblend_close_peaks( + x, net, core.dedupe_channel_peaks(peaks, args.peak_distance), + args.shape, 2.0, noise) + peaks.sort(key=lambda item: item['center_px']) + curves[c] = (x, y, baseline, net, norm) + for j, peak in enumerate(peaks, 1): + index = peak['sample_index'] + candidates.append(dict( + id=f'{c}{j}', channel=c, sample=int(index), + pixel=float(peak['center_px']), + signal_adu=float(y[index]), + height_net_adu=float(peak['height_net_adu']), + normalized=float(norm[index]), + fwhm_px=float(peak['fwhm_px']), + snr=float(peak['snr']), + center_source=peak['center_source'], + deblended=bool(peak.get('deblended')), + saturated=bool(peak['saturated']))) + return curves, candidates, black, full + + +def assignments(text, candidates, order): + lookup = {v['id']: v for v in candidates} + result, used = [], set() + for token in text.split(','): + name, wavelength = token.strip().split(':') + name = name.upper().strip() + if name not in lookup or name in used: + raise ValueError(f'unknown or duplicate candidate: {name}') + nm = float(wavelength) + if not np.isfinite(nm) or nm <= 0: + raise ValueError('wavelength must be finite and positive') + used.add(name) + result.append({**lookup[name], 'wavelength_nm': nm}) + if len(result) < order + 1: + raise ValueError(f'order {order} needs at least {order + 1} confirmed lines') + x = np.array([r['pixel'] for r in result]) + y = np.array([r['wavelength_nm'] for r in result]) + if len(np.unique(x)) != len(x) or len(np.unique(y)) != len(y): + raise ValueError('select one channel candidate per distinct spectral line') + sort = np.argsort(x) + if not (np.all(np.diff(y[sort]) > 0) or np.all(np.diff(y[sort]) < 0)): + raise ValueError('confirmed wavelength order must be monotonic in sensor x') + return result + + +def fit(confirmed, order, width): + x = np.array([r['pixel'] for r in confirmed]) + y = np.array([r['wavelength_nm'] for r in confirmed]) + model = np.polynomial.Polynomial.fit(x, y, order).convert() + coeff = model.coef[::-1] + derivative = np.polyval(np.polyder(coeff), [0, width - 1]) + if np.any(np.abs(derivative) < 1e-12) or derivative[0] * derivative[1] <= 0: + raise ValueError('mapping is not monotonic across sensor; revise identities/order/ROI') + residual = np.polyval(coeff, x) - y + return coeff, residual + + +def write_csv(path, fields, rows): + with path.open('w', newline='', encoding='utf-8') as f: + w = csv.writer(f) + w.writerow(fields) + w.writerows(rows) + + +def run(args): + if args.frame < 0 or args.peak_distance <= 0 or args.max_peaks < 1: + raise ValueError('frame >= 0, peak-distance > 0 and max-peaks >= 1 required') + for v in (args.peak_height, args.peak_prominence): + if not np.isfinite(v) or not 0 <= v <= 1: + raise ValueError('normalized peak thresholds must be between 0 and 1') + import matplotlib + if args.headless: + matplotlib.use('Agg') + import matplotlib.pyplot as plt + out = Path(args.output) + out.mkdir(parents=True, exist_ok=True) + source = hq.RecordingSource(args.input) + try: + source.start() + raw, meta = source.read_at(args.frame) + roi = hq.parse_roi(args.roi, *source.full_size) + preview = hq.make_preview(raw, source.pattern) + if roi is None: + roi = hq.detect_auto_roi(preview, *source.full_size) + if roi is None: + raise ValueError('auto ROI failed; specify --roi x0,y0,x1,y1') + curves, candidates, black, full = extract(raw, meta, roi, source.pattern, + source.bit_depth, args) + finally: + source.close() + fields = ['id', 'channel', 'sample', 'pixel', 'signal_adu', 'height_net_adu', + 'normalized', 'fwhm_px', 'snr', 'center_source', 'deblended', 'saturated'] + write_csv(out / 'candidates.csv', fields, ([v[k] for k in fields] for v in candidates)) + write_csv(out / 'profiles.csv', + ['channel', 'pixel', 'black_corrected_adu', 'baseline_adu', 'net_adu', + 'normalized_for_detection'], + ([c, x, y, b, n, q] for c, arrays in curves.items() + for x, y, b, n, q in zip(*arrays))) + fig, axes = plt.subplots(3, 1, figsize=(13, 10), constrained_layout=True) + axes[0].imshow(preview, extent=(0, source.full_size[0], source.full_size[1], 0)) + from matplotlib.patches import Rectangle + axes[0].add_patch(Rectangle((roi.x0, roi.y0), roi.width, roi.height, fill=False, edgecolor='orange')) + axes[0].set_title(f'Frame {args.frame} (zero based); ROI {roi.text()}; black={black:g} ADU') + for c, (x, y, baseline, net, norm) in curves.items(): + axes[1].plot(x, y, color=hq.CHANNEL_COLORS[c], label=c) + axes[1].plot(x, baseline, color=hq.CHANNEL_COLORS[c], lw=0.6, ls=':', alpha=0.7) + axes[2].plot(x, norm, color=hq.CHANNEL_COLORS[c], label=c) + for v in candidates: + marker = 's' if v['deblended'] else 'o' + axes[2].plot(v['pixel'], v['normalized'], marker, color=hq.CHANNEL_COLORS[v['channel']], + mfc='none' if v['deblended'] else hq.CHANNEL_COLORS[v['channel']]) + axes[2].annotate(v['id'], (v['pixel'], v['normalized']), xytext=(0, 8), + textcoords='offset points') + print(f"{v['id']:5s} x={v['pixel']:9.3f} FWHM={v['fwhm_px']:7.2f} px " + f"SNR={v['snr']:8.0f} normalized={v['normalized']:.4f}" + f"{' [pair]' if v['deblended'] else ''}{' [sat]' if v['saturated'] else ''}") + axes[1].set_ylabel('Black-corrected ADU'); axes[1].legend() + axes[2].set_ylabel('Normalized (detection only)'); axes[2].set_xlabel('Sensor column / px') + axes[2].set_ylim(-0.05, 1.18) + fig.savefig(out / 'candidates.png', dpi=160) + print('Hg I air wavelengths (nm):', HG) + print('Blended pairs such as 576.960/579.066 nm (about 7 px apart) are split by a joint ' + 'two-line fit and marked [pair]; use one channel per line.') + print('Select ONE channel candidate per line. Review candidates.png before confirming identities.') + text = args.assign + if not args.headless: + print('Inspect/zoom the plot, then CLOSE its window to enter confirmed identities in the terminal.') + plt.show(block=True) + if text is None: + text = input('Confirmed IDs:wavelength (comma separated), blank to export candidates only: ').strip() + plt.close(fig) + if not text: + print(f'Candidates exported to {out.resolve()}; no calibration fitted.') + return 0 + confirmed = assignments(text, candidates, args.order) + coeff, residual = fit(confirmed, args.order, source.full_size[0]) + x = np.array([r['pixel'] for r in confirmed]) + lo, hi = float(x.min()), float(x.max()) + info = dict(recording=str(Path(args.input).resolve()), frame_zero_based=args.frame, + roi=[roi.x0, roi.y0, roi.x1, roi.y1], pattern=source.pattern, + raw_format=source.raw_format, black_adu=black, full_scale_adu=full, + reduce=args.reduce, order=args.order, coefficients_descending=coeff.tolist(), + formula='wavelength_nm = numpy.polyval(coefficients_descending, sensor_x_px)', + wavelength_medium='air', reference=SOURCE, confirmed=confirmed, + residual_definition='fitted - reference (nm)', residuals_nm=residual.tolist(), + rms_nm=float(np.sqrt(np.mean(residual ** 2))), max_abs_nm=float(np.max(np.abs(residual))), + residual_degrees_of_freedom=len(x)-args.order-1, calibrated_pixel_range=[lo, hi], + processing='spatial reduction, black subtraction, morphological baseline for ' + 'peak detection, topographic prominence, Gaussian centres with ' + 'peak-top fallback and joint fits for blended doublets') + (out / 'calibration.json').write_text(json.dumps(info, indent=2), encoding='utf-8') + write_csv(out / 'residuals.csv', ['id', 'pixel', 'reference_nm', 'fitted_nm', 'residual_nm'], + ([r['id'], r['pixel'], r['wavelength_nm'], r['wavelength_nm']+e, e] for r, e in zip(confirmed, residual))) + grid = np.arange(source.full_size[0]) + write_csv(out / 'pixel_wavelength.csv', ['sensor_x_px', 'wavelength_nm', 'extrapolated'], + ([int(p), float(w), int(p < lo or p > hi)] for p, w in zip(grid, np.polyval(coeff, grid)))) + fig, ax = plt.subplots(2, 1, figsize=(10, 7), constrained_layout=True) + ax[0].plot(grid, np.polyval(coeff, grid)); ax[0].scatter(x, [r['wavelength_nm'] for r in confirmed]) + ax[0].axvspan(lo, hi, alpha=.1, color='green', label='Confirmed span'); ax[0].legend() + ax[0].set_ylabel('Wavelength / nm (air)') + ax[1].axhline(0, color='gray'); ax[1].scatter(x, residual) + for r, e in zip(confirmed, residual): + ax[1].annotate(r['id'], (r['pixel'], e)) + ax[1].set_ylabel('Fit - reference / nm'); ax[1].set_xlabel('Sensor column / px') + fig.savefig(out / 'calibration.png', dpi=160); plt.close(fig) + print('Coefficients (descending):', coeff) + print(f"RMS={info['rms_nm']:.6g} nm; max |residual|={info['max_abs_nm']:.6g} nm") + if len(x) == args.order + 1: + print('Exact interpolation: zero residual does not establish calibration accuracy; add independent lines.') + print(f'Calibration exported to {out.resolve()}') + return 0 + + +def main(argv=None): + args = parser().parse_args(argv) + try: + return run(args) + except (ValueError, RuntimeError, OSError, IndexError, StopIteration, EOFError) as e: + print(f'Calibration error: {e or "frame outside recording"}', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/wavelength_calibration/calibration.json b/wavelength_calibration/calibration.json new file mode 100644 index 0000000..5016dc0 --- /dev/null +++ b/wavelength_calibration/calibration.json @@ -0,0 +1,66 @@ +{ + "recording": "D:\\Dev\\Light\\Programs\\recordings\\hq_20261101_114417", + "frame_zero_based": 0, + "roi": [ + 98, + 654, + 848, + 856 + ], + "pattern": "BGGR", + "raw_format": "BGGR_PISP_COMP1", + "black_adu": 4096.0, + "full_scale_adu": 65535.0, + "reduce": "mean", + "order": 1, + "coefficients_descending": [ + 0.2849422617422453, + 349.4538237085669 + ], + "formula": "wavelength_nm = numpy.polyval(coefficients_descending, sensor_x_px)", + "wavelength_medium": "air", + "reference": "https://physics.nist.gov/PhysRefData/Handbook/Tables/mercurytable2.htm", + "confirmed": [ + { + "id": "B1", + "channel": "B", + "sample": 103, + "pixel": 304.0, + "signal_adu": 9736.237623762376, + "normalized": 1.0, + "wavelength_nm": 435.8328 + }, + { + "id": "G1", + "channel": "G", + "sample": 294, + "pixel": 686.5, + "signal_adu": 54219.9801980198, + "normalized": 1.0, + "wavelength_nm": 546.0735 + }, + { + "id": "G2", + "channel": "G", + "sample": 355, + "pixel": 808.5, + "signal_adu": 17836.9900990099, + "normalized": 0.3289744856760633, + "wavelength_nm": 579.0663 + } + ], + "residual_definition": "fitted - reference (nm)", + "residuals_nm": [ + 0.24347127820948344, + -1.0068136053816943, + 0.7633423271722677 + ], + "rms_nm": 0.7428870276938522, + "max_abs_nm": 1.0068136053816943, + "residual_degrees_of_freedom": 1, + "calibrated_pixel_range": [ + 304.0, + 808.5 + ], + "processing": "spatial reduction, black subtraction; no smoothing; max normalization only for detection" +} \ No newline at end of file diff --git a/wavelength_calibration/calibration.png b/wavelength_calibration/calibration.png new file mode 100644 index 0000000..6577493 Binary files /dev/null and b/wavelength_calibration/calibration.png differ diff --git a/wavelength_calibration/candidates.csv b/wavelength_calibration/candidates.csv new file mode 100644 index 0000000..e8dac66 --- /dev/null +++ b/wavelength_calibration/candidates.csv @@ -0,0 +1,9 @@ +id,channel,sample,pixel,signal_adu,normalized +R1,R,36,301.0,1066.666666666667,0.2308984047019312 +R2,R,89,407.0,1468.606060606061,0.31790512174643165 +R3,R,285,799.0,4619.636363636364,1.0 +R4,R,345,919.0,1852.121212121212,0.4009235936188077 +G1,G,285,798.5,61439.0,1.0 +B1,B,35,298.0,8451.878787878788,0.1652635570724308 +B2,B,89,406.0,51141.818181818184,1.0 +B3,B,285,798.0,9526.30303030303,0.1862722791050436 diff --git a/wavelength_calibration/candidates.png b/wavelength_calibration/candidates.png new file mode 100644 index 0000000..82ceffb Binary files /dev/null and b/wavelength_calibration/candidates.png differ diff --git a/wavelength_calibration/pixel_wavelength.csv b/wavelength_calibration/pixel_wavelength.csv new file mode 100644 index 0000000..4cfb2b1 --- /dev/null +++ b/wavelength_calibration/pixel_wavelength.csv @@ -0,0 +1,2029 @@ +sensor_x_px,wavelength_nm,extrapolated +0,349.4538237085669,1 +1,349.73876597030915,1 +2,350.02370823205143,1 +3,350.30865049379366,1 +4,350.5935927555359,1 +5,350.8785350172781,1 +6,351.1634772790204,1 +7,351.44841954076264,1 +8,351.7333618025049,1 +9,352.0183040642471,1 +10,352.3032463259894,1 +11,352.5881885877316,1 +12,352.87313084947385,1 +13,353.1580731112161,1 +14,353.44301537295837,1 +15,353.7279576347006,1 +16,354.01289989644283,1 +17,354.29784215818506,1 +18,354.58278441992735,1 +19,354.8677266816696,1 +20,355.1526689434118,1 +21,355.43761120515404,1 +22,355.72255346689633,1 +23,356.00749572863856,1 +24,356.2924379903808,1 +25,356.577380252123,1 +26,356.8623225138653,1 +27,357.14726477560754,1 +28,357.43220703734977,1 +29,357.71714929909206,1 +30,358.0020915608343,1 +31,358.2870338225765,1 +32,358.57197608431875,1 +33,358.85691834606104,1 +34,359.14186060780327,1 +35,359.4268028695455,1 +36,359.7117451312877,1 +37,359.99668739303,1 +38,360.28162965477225,1 +39,360.5665719165145,1 +40,360.8515141782567,1 +41,361.136456439999,1 +42,361.4213987017412,1 +43,361.70634096348346,1 +44,361.9912832252257,1 +45,362.276225486968,1 +46,362.5611677487102,1 +47,362.84611001045243,1 +48,363.13105227219467,1 +49,363.41599453393695,1 +50,363.7009367956792,1 +51,363.9858790574214,1 +52,364.27082131916364,1 +53,364.55576358090593,1 +54,364.84070584264816,1 +55,365.1256481043904,1 +56,365.4105903661326,1 +57,365.6955326278749,1 +58,365.98047488961714,1 +59,366.2654171513594,1 +60,366.55035941310166,1 +61,366.8353016748439,1 +62,367.1202439365861,1 +63,367.40518619832835,1 +64,367.69012846007064,1 +65,367.97507072181287,1 +66,368.2600129835551,1 +67,368.54495524529733,1 +68,368.8298975070396,1 +69,369.11483976878185,1 +70,369.3997820305241,1 +71,369.6847242922663,1 +72,369.9696665540086,1 +73,370.2546088157508,1 +74,370.53955107749306,1 +75,370.8244933392353,1 +76,371.1094356009776,1 +77,371.3943778627198,1 +78,371.67932012446204,1 +79,371.96426238620427,1 +80,372.24920464794656,1 +81,372.5341469096888,1 +82,372.819089171431,1 +83,373.1040314331733,1 +84,373.38897369491553,1 +85,373.67391595665777,1 +86,373.9588582184,1 +87,374.2438004801422,1 +88,374.5287427418845,1 +89,374.81368500362674,1 +90,375.098627265369,1 +91,375.38356952711126,1 +92,375.6685117888535,1 +93,375.9534540505957,1 +94,376.23839631233795,1 +95,376.52333857408024,1 +96,376.8082808358225,1 +97,377.0932230975647,1 +98,377.37816535930693,1 +99,377.6631076210492,1 +100,377.94804988279145,1 +101,378.2329921445337,1 +102,378.5179344062759,1 +103,378.8028766680182,1 +104,379.08781892976043,1 +105,379.37276119150266,1 +106,379.6577034532449,1 +107,379.9426457149872,1 +108,380.2275879767294,1 +109,380.51253023847164,1 +110,380.79747250021387,1 +111,381.08241476195616,1 +112,381.3673570236984,1 +113,381.6522992854406,1 +114,381.9372415471829,1 +115,382.22218380892514,1 +116,382.50712607066737,1 +117,382.7920683324096,1 +118,383.07701059415183,1 +119,383.3619528558941,1 +120,383.64689511763635,1 +121,383.9318373793786,1 +122,384.21677964112087,1 +123,384.5017219028631,1 +124,384.7866641646053,1 +125,385.07160642634756,1 +126,385.35654868808984,1 +127,385.6414909498321,1 +128,385.9264332115743,1 +129,386.21137547331654,1 +130,386.4963177350588,1 +131,386.78125999680105,1 +132,387.0662022585433,1 +133,387.3511445202855,1 +134,387.6360867820278,1 +135,387.92102904377003,1 +136,388.20597130551226,1 +137,388.49091356725455,1 +138,388.7758558289968,1 +139,389.060798090739,1 +140,389.34574035248124,1 +141,389.6306826142235,1 +142,389.91562487596576,1 +143,390.200567137708,1 +144,390.4855093994502,1 +145,390.7704516611925,1 +146,391.05539392293474,1 +147,391.34033618467697,1 +148,391.6252784464192,1 +149,391.91022070816143,1 +150,392.1951629699037,1 +151,392.48010523164595,1 +152,392.7650474933882,1 +153,393.04998975513047,1 +154,393.3349320168727,1 +155,393.61987427861493,1 +156,393.90481654035716,1 +157,394.18975880209945,1 +158,394.4747010638417,1 +159,394.7596433255839,1 +160,395.04458558732614,1 +161,395.3295278490684,1 +162,395.61447011081066,1 +163,395.8994123725529,1 +164,396.1843546342951,1 +165,396.4692968960374,1 +166,396.75423915777964,1 +167,397.03918141952187,1 +168,397.32412368126415,1 +169,397.6090659430064,1 +170,397.8940082047486,1 +171,398.17895046649085,1 +172,398.4638927282331,1 +173,398.74883498997536,1 +174,399.0337772517176,1 +175,399.3187195134598,1 +176,399.6036617752021,1 +177,399.88860403694434,1 +178,400.1735462986866,1 +179,400.4584885604288,1 +180,400.7434308221711,1 +181,401.0283730839133,1 +182,401.31331534565555,1 +183,401.5982576073978,1 +184,401.88319986914007,1 +185,402.1681421308823,1 +186,402.45308439262453,1 +187,402.73802665436676,1 +188,403.02296891610905,1 +189,403.3079111778513,1 +190,403.5928534395935,1 +191,403.87779570133574,1 +192,404.16273796307803,1 +193,404.44768022482026,1 +194,404.7326224865625,1 +195,405.0175647483047,1 +196,405.302507010047,1 +197,405.58744927178924,1 +198,405.87239153353147,1 +199,406.15733379527376,1 +200,406.442276057016,1 +201,406.7272183187582,1 +202,407.01216058050045,1 +203,407.2971028422427,1 +204,407.58204510398497,1 +205,407.8669873657272,1 +206,408.1519296274694,1 +207,408.4368718892117,1 +208,408.72181415095395,1 +209,409.0067564126962,1 +210,409.2916986744384,1 +211,409.5766409361807,1 +212,409.8615831979229,1 +213,410.14652545966516,1 +214,410.4314677214074,1 +215,410.7164099831497,1 +216,411.0013522448919,1 +217,411.28629450663414,1 +218,411.57123676837637,1 +219,411.85617903011865,1 +220,412.1411212918609,1 +221,412.4260635536031,1 +222,412.7110058153454,1 +223,412.99594807708763,1 +224,413.28089033882986,1 +225,413.5658326005721,1 +226,413.8507748623143,1 +227,414.1357171240566,1 +228,414.42065938579884,1 +229,414.7056016475411,1 +230,414.99054390928336,1 +231,415.2754861710256,1 +232,415.5604284327678,1 +233,415.84537069451005,1 +234,416.1303129562523,1 +235,416.41525521799457,1 +236,416.7001974797368,1 +237,416.98513974147903,1 +238,417.2700820032213,1 +239,417.55502426496355,1 +240,417.8399665267058,1 +241,418.124908788448,1 +242,418.4098510501903,1 +243,418.69479331193253,1 +244,418.97973557367476,1 +245,419.26467783541705,1 +246,419.5496200971593,1 +247,419.8345623589015,1 +248,420.11950462064374,1 +249,420.40444688238597,1 +250,420.68938914412826,1 +251,420.9743314058705,1 +252,421.2592736676127,1 +253,421.544215929355,1 +254,421.82915819109724,1 +255,422.11410045283947,1 +256,422.3990427145817,1 +257,422.6839849763239,1 +258,422.9689272380662,1 +259,423.25386949980845,1 +260,423.5388117615507,1 +261,423.82375402329296,1 +262,424.1086962850352,1 +263,424.3936385467774,1 +264,424.67858080851965,1 +265,424.9635230702619,1 +266,425.2484653320042,1 +267,425.5334075937464,1 +268,425.81834985548863,1 +269,426.1032921172309,1 +270,426.38823437897315,1 +271,426.6731766407154,1 +272,426.9581189024576,1 +273,427.2430611641999,1 +274,427.52800342594213,1 +275,427.81294568768436,1 +276,428.09788794942665,1 +277,428.3828302111689,1 +278,428.6677724729111,1 +279,428.95271473465334,1 +280,429.2376569963956,1 +281,429.52259925813786,1 +282,429.8075415198801,1 +283,430.0924837816223,1 +284,430.3774260433646,1 +285,430.66236830510684,1 +286,430.94731056684907,1 +287,431.2322528285913,1 +288,431.51719509033353,1 +289,431.8021373520758,1 +290,432.08707961381805,1 +291,432.3720218755603,1 +292,432.65696413730257,1 +293,432.9419063990448,1 +294,433.226848660787,1 +295,433.51179092252926,1 +296,433.7967331842715,1 +297,434.0816754460138,1 +298,434.366617707756,1 +299,434.6515599694983,1 +300,434.9365022312405,1 +301,435.22144449298276,1 +302,435.506386754725,1 +303,435.7913290164672,1 +304,436.0762712782095,0 +305,436.36121353995173,0 +306,436.64615580169396,0 +307,436.93109806343625,0 +308,437.2160403251785,0 +309,437.5009825869207,0 +310,437.78592484866294,0 +311,438.0708671104052,0 +312,438.35580937214746,0 +313,438.6407516338897,0 +314,438.9256938956319,0 +315,439.2106361573742,0 +316,439.49557841911644,0 +317,439.7805206808587,0 +318,440.0654629426009,0 +319,440.35040520434313,0 +320,440.6353474660854,0 +321,440.92028972782765,0 +322,441.2052319895699,0 +323,441.49017425131217,0 +324,441.7751165130544,0 +325,442.06005877479663,0 +326,442.34500103653886,0 +327,442.6299432982811,0 +328,442.9148855600234,0 +329,443.1998278217656,0 +330,443.4847700835079,0 +331,443.7697123452501,0 +332,444.05465460699236,0 +333,444.3395968687346,0 +334,444.6245391304768,0 +335,444.9094813922191,0 +336,445.19442365396134,0 +337,445.47936591570357,0 +338,445.76430817744586,0 +339,446.0492504391881,0 +340,446.3341927009303,0 +341,446.61913496267255,0 +342,446.9040772244148,0 +343,447.18901948615706,0 +344,447.4739617478993,0 +345,447.7589040096415,0 +346,448.0438462713838,0 +347,448.32878853312604,0 +348,448.6137307948683,0 +349,448.8986730566105,0 +350,449.18361531835274,0 +351,449.468557580095,0 +352,449.75349984183725,0 +353,450.0384421035795,0 +354,450.3233843653218,0 +355,450.608326627064,0 +356,450.89326888880623,0 +357,451.17821115054846,0 +358,451.46315341229075,0 +359,451.748095674033,0 +360,452.0330379357752,0 +361,452.3179801975175,0 +362,452.60292245925973,0 +363,452.88786472100196,0 +364,453.1728069827442,0 +365,453.4577492444864,0 +366,453.7426915062287,0 +367,454.02763376797094,0 +368,454.31257602971317,0 +369,454.59751829145546,0 +370,454.8824605531977,0 +371,455.1674028149399,0 +372,455.45234507668215,0 +373,455.7372873384244,0 +374,456.02222960016667,0 +375,456.3071718619089,0 +376,456.59211412365113,0 +377,456.8770563853934,0 +378,457.16199864713565,0 +379,457.4469409088779,0 +380,457.7318831706201,0 +381,458.01682543236234,0 +382,458.3017676941046,0 +383,458.58670995584686,0 +384,458.87165221758914,0 +385,459.1565944793314,0 +386,459.4415367410736,0 +387,459.72647900281584,0 +388,460.01142126455807,0 +389,460.29636352630035,0 +390,460.5813057880426,0 +391,460.8662480497848,0 +392,461.1511903115271,0 +393,461.43613257326933,0 +394,461.72107483501156,0 +395,462.0060170967538,0 +396,462.290959358496,0 +397,462.5759016202383,0 +398,462.86084388198054,0 +399,463.1457861437228,0 +400,463.43072840546506,0 +401,463.7156706672073,0 +402,464.0006129289495,0 +403,464.28555519069175,0 +404,464.570497452434,0 +405,464.85543971417627,0 +406,465.1403819759185,0 +407,465.42532423766073,0 +408,465.710266499403,0 +409,465.99520876114525,0 +410,466.2801510228875,0 +411,466.5650932846297,0 +412,466.85003554637194,0 +413,467.13497780811423,0 +414,467.41992006985646,0 +415,467.70486233159875,0 +416,467.989804593341,0 +417,468.2747468550832,0 +418,468.55968911682544,0 +419,468.84463137856767,0 +420,469.12957364030996,0 +421,469.4145159020522,0 +422,469.6994581637944,0 +423,469.9844004255367,0 +424,470.26934268727894,0 +425,470.55428494902117,0 +426,470.8392272107634,0 +427,471.1241694725056,0 +428,471.4091117342479,0 +429,471.69405399599015,0 +430,471.9789962577324,0 +431,472.26393851947466,0 +432,472.5488807812169,0 +433,472.8338230429591,0 +434,473.11876530470136,0 +435,473.4037075664436,0 +436,473.6886498281859,0 +437,473.9735920899281,0 +438,474.25853435167033,0 +439,474.5434766134126,0 +440,474.82841887515485,0 +441,475.1133611368971,0 +442,475.3983033986393,0 +443,475.6832456603816,0 +444,475.96818792212383,0 +445,476.25313018386606,0 +446,476.53807244560835,0 +447,476.8230147073506,0 +448,477.1079569690928,0 +449,477.39289923083504,0 +450,477.6778414925773,0 +451,477.9627837543195,0 +452,478.2477260160618,0 +453,478.532668277804,0 +454,478.8176105395463,0 +455,479.10255280128854,0 +456,479.38749506303077,0 +457,479.672437324773,0 +458,479.95737958651523,0 +459,480.2423218482575,0 +460,480.52726410999975,0 +461,480.81220637174204,0 +462,481.09714863348427,0 +463,481.3820908952265,0 +464,481.6670331569687,0 +465,481.95197541871096,0 +466,482.2369176804532,0 +467,482.5218599421955,0 +468,482.8068022039377,0 +469,483.09174446568,0 +470,483.3766867274222,0 +471,483.66162898916446,0 +472,483.9465712509067,0 +473,484.2315135126489,0 +474,484.51645577439115,0 +475,484.80139803613343,0 +476,485.08634029787567,0 +477,485.37128255961795,0 +478,485.6562248213602,0 +479,485.9411670831024,0 +480,486.22610934484464,0 +481,486.5110516065869,0 +482,486.7959938683291,0 +483,487.0809361300714,0 +484,487.3658783918137,0 +485,487.6508206535559,0 +486,487.93576291529814,0 +487,488.2207051770404,0 +488,488.5056474387826,0 +489,488.79058970052483,0 +490,489.0755319622671,0 +491,489.36047422400935,0 +492,489.64541648575164,0 +493,489.93035874749387,0 +494,490.2153010092361,0 +495,490.50024327097833,0 +496,490.78518553272056,0 +497,491.0701277944628,0 +498,491.3550700562051,0 +499,491.6400123179473,0 +500,491.9249545796896,0 +501,492.2098968414318,0 +502,492.49483910317406,0 +503,492.7797813649163,0 +504,493.0647236266585,0 +505,493.34966588840075,0 +506,493.63460815014304,0 +507,493.91955041188527,0 +508,494.20449267362756,0 +509,494.4894349353698,0 +510,494.774377197112,0 +511,495.05931945885425,0 +512,495.3442617205965,0 +513,495.62920398233877,0 +514,495.914146244081,0 +515,496.1990885058233,0 +516,496.4840307675655,0 +517,496.76897302930774,0 +518,497.05391529105,0 +519,497.3388575527922,0 +520,497.62379981453444,0 +521,497.9087420762767,0 +522,498.19368433801895,0 +523,498.47862659976124,0 +524,498.7635688615035,0 +525,499.0485111232457,0 +526,499.33345338498793,0 +527,499.61839564673016,0 +528,499.9033379084724,0 +529,500.1882801702147,0 +530,500.4732224319569,0 +531,500.7581646936992,0 +532,501.04310695544143,0 +533,501.32804921718366,0 +534,501.6129914789259,0 +535,501.8979337406681,0 +536,502.18287600241035,0 +537,502.46781826415264,0 +538,502.75276052589487,0 +539,503.03770278763716,0 +540,503.3226450493794,0 +541,503.6075873111216,0 +542,503.89252957286385,0 +543,504.1774718346061,0 +544,504.46241409634837,0 +545,504.7473563580906,0 +546,505.0322986198329,0 +547,505.3172408815751,0 +548,505.60218314331735,0 +549,505.8871254050596,0 +550,506.1720676668018,0 +551,506.45700992854404,0 +552,506.7419521902863,0 +553,507.02689445202856,0 +554,507.31183671377084,0 +555,507.5967789755131,0 +556,507.8817212372553,0 +557,508.16666349899754,0 +558,508.45160576073977,0 +559,508.736548022482,0 +560,509.0214902842243,0 +561,509.3064325459665,0 +562,509.5913748077088,0 +563,509.87631706945103,0 +564,510.16125933119326,0 +565,510.4462015929355,0 +566,510.7311438546777,0 +567,511.01608611641996,0 +568,511.30102837816224,0 +569,511.58597063990453,0 +570,511.87091290164676,0 +571,512.155855163389,0 +572,512.4407974251312,0 +573,512.7257396868735,0 +574,513.0106819486157,0 +575,513.295624210358,0 +576,513.5805664721001,0 +577,513.8655087338425,0 +578,514.1504509955847,0 +579,514.435393257327,0 +580,514.7203355190692,0 +581,515.0052777808114,0 +582,515.2902200425536,0 +583,515.5751623042959,0 +584,515.8601045660382,0 +585,516.1450468277804,0 +586,516.4299890895227,0 +587,516.7149313512649,0 +588,516.9998736130071,0 +589,517.2848158747494,0 +590,517.5697581364916,0 +591,517.854700398234,0 +592,518.1396426599761,0 +593,518.4245849217184,0 +594,518.7095271834606,0 +595,518.9944694452029,0 +596,519.2794117069451,0 +597,519.5643539686873,0 +598,519.8492962304297,0 +599,520.1342384921718,0 +600,520.4191807539141,0 +601,520.7041230156564,0 +602,520.9890652773986,0 +603,521.2740075391408,0 +604,521.558949800883,0 +605,521.8438920626253,0 +606,522.1288343243675,0 +607,522.4137765861099,0 +608,522.6987188478521,0 +609,522.9836611095943,0 +610,523.2686033713366,0 +611,523.5535456330788,0 +612,523.838487894821,0 +613,524.1234301565632,0 +614,524.4083724183056,0 +615,524.6933146800477,0 +616,524.97825694179,0 +617,525.2631992035323,0 +618,525.5481414652745,0 +619,525.8330837270167,0 +620,526.118025988759,0 +621,526.4029682505012,0 +622,526.6879105122434,0 +623,526.9728527739858,0 +624,527.257795035728,0 +625,527.5427372974702,0 +626,527.8276795592125,0 +627,528.1126218209547,0 +628,528.3975640826969,0 +629,528.6825063444392,0 +630,528.9674486061815,0 +631,529.2523908679237,0 +632,529.537333129666,0 +633,529.8222753914082,0 +634,530.1072176531504,0 +635,530.3921599148927,0 +636,530.6771021766349,0 +637,530.9620444383772,0 +638,531.2469867001194,0 +639,531.5319289618617,0 +640,531.8168712236039,0 +641,532.1018134853462,0 +642,532.3867557470884,0 +643,532.6716980088306,0 +644,532.9566402705728,0 +645,533.2415825323151,0 +646,533.5265247940574,0 +647,533.8114670557997,0 +648,534.0964093175419,0 +649,534.3813515792841,0 +650,534.6662938410263,0 +651,534.9512361027686,0 +652,535.2361783645108,0 +653,535.5211206262532,0 +654,535.8060628879953,0 +655,536.0910051497376,0 +656,536.3759474114798,0 +657,536.6608896732221,0 +658,536.9458319349643,0 +659,537.2307741967065,0 +660,537.5157164584489,0 +661,537.800658720191,0 +662,538.0856009819333,0 +663,538.3705432436756,0 +664,538.6554855054178,0 +665,538.94042776716,0 +666,539.2253700289023,0 +667,539.5103122906445,0 +668,539.7952545523867,0 +669,540.0801968141291,0 +670,540.3651390758713,0 +671,540.6500813376135,0 +672,540.9350235993558,0 +673,541.219965861098,0 +674,541.5049081228402,0 +675,541.7898503845825,0 +676,542.0747926463248,0 +677,542.3597349080669,0 +678,542.6446771698093,0 +679,542.9296194315515,0 +680,543.2145616932937,0 +681,543.499503955036,0 +682,543.7844462167782,0 +683,544.0693884785205,0 +684,544.3543307402626,0 +685,544.639273002005,0 +686,544.9242152637472,0 +687,545.2091575254894,0 +688,545.4940997872317,0 +689,545.7790420489739,0 +690,546.0639843107161,0 +691,546.3489265724584,0 +692,546.6338688342007,0 +693,546.9188110959429,0 +694,547.2037533576852,0 +695,547.4886956194274,0 +696,547.7736378811696,0 +697,548.0585801429119,0 +698,548.3435224046541,0 +699,548.6284646663964,0 +700,548.9134069281386,0 +701,549.1983491898809,0 +702,549.4832914516231,0 +703,549.7682337133654,0 +704,550.0531759751076,0 +705,550.3381182368498,0 +706,550.623060498592,0 +707,550.9080027603343,0 +708,551.1929450220766,0 +709,551.4778872838189,0 +710,551.7628295455611,0 +711,552.0477718073033,0 +712,552.3327140690456,0 +713,552.6176563307878,0 +714,552.90259859253,0 +715,553.1875408542724,0 +716,553.4724831160146,0 +717,553.7574253777568,0 +718,554.042367639499,0 +719,554.3273099012413,0 +720,554.6122521629835,0 +721,554.8971944247257,0 +722,555.1821366864681,0 +723,555.4670789482102,0 +724,555.7520212099525,0 +725,556.0369634716948,0 +726,556.321905733437,0 +727,556.6068479951792,0 +728,556.8917902569215,0 +729,557.1767325186637,0 +730,557.4616747804059,0 +731,557.7466170421483,0 +732,558.0315593038905,0 +733,558.3165015656327,0 +734,558.601443827375,0 +735,558.8863860891172,0 +736,559.1713283508594,0 +737,559.4562706126017,0 +738,559.741212874344,0 +739,560.0261551360861,0 +740,560.3110973978285,0 +741,560.5960396595707,0 +742,560.8809819213129,0 +743,561.1659241830552,0 +744,561.4508664447974,0 +745,561.7358087065397,0 +746,562.0207509682818,0 +747,562.3056932300242,0 +748,562.5906354917664,0 +749,562.8755777535087,0 +750,563.1605200152509,0 +751,563.4454622769931,0 +752,563.7304045387353,0 +753,564.0153468004776,0 +754,564.3002890622199,0 +755,564.5852313239621,0 +756,564.8701735857044,0 +757,565.1551158474466,0 +758,565.4400581091888,0 +759,565.7250003709311,0 +760,566.0099426326733,0 +761,566.2948848944156,0 +762,566.5798271561578,0 +763,566.8647694179001,0 +764,567.1497116796423,0 +765,567.4346539413846,0 +766,567.7195962031268,0 +767,568.004538464869,0 +768,568.2894807266114,0 +769,568.5744229883535,0 +770,568.8593652500958,0 +771,569.1443075118381,0 +772,569.4292497735803,0 +773,569.7141920353225,0 +774,569.9991342970648,0 +775,570.284076558807,0 +776,570.5690188205492,0 +777,570.8539610822916,0 +778,571.1389033440338,0 +779,571.423845605776,0 +780,571.7087878675183,0 +781,571.9937301292605,0 +782,572.2786723910027,0 +783,572.563614652745,0 +784,572.8485569144873,0 +785,573.1334991762294,0 +786,573.4184414379718,0 +787,573.703383699714,0 +788,573.9883259614562,0 +789,574.2732682231984,0 +790,574.5582104849407,0 +791,574.8431527466829,0 +792,575.1280950084251,0 +793,575.4130372701675,0 +794,575.6979795319097,0 +795,575.9829217936519,0 +796,576.2678640553942,0 +797,576.5528063171364,0 +798,576.8377485788786,0 +799,577.1226908406209,0 +800,577.4076331023632,0 +801,577.6925753641054,0 +802,577.9775176258477,0 +803,578.2624598875899,0 +804,578.5474021493321,0 +805,578.8323444110744,0 +806,579.1172866728166,0 +807,579.4022289345589,0 +808,579.687171196301,0 +809,579.9721134580434,1 +810,580.2570557197856,1 +811,580.5419979815279,1 +812,580.8269402432701,1 +813,581.1118825050123,1 +814,581.3968247667545,1 +815,581.6817670284968,1 +816,581.9667092902391,1 +817,582.2516515519814,1 +818,582.5365938137236,1 +819,582.8215360754658,1 +820,583.106478337208,1 +821,583.3914205989503,1 +822,583.6763628606925,1 +823,583.9613051224349,1 +824,584.246247384177,1 +825,584.5311896459193,1 +826,584.8161319076615,1 +827,585.1010741694038,1 +828,585.386016431146,1 +829,585.6709586928882,1 +830,585.9559009546306,1 +831,586.2408432163727,1 +832,586.525785478115,1 +833,586.8107277398573,1 +834,587.0956700015995,1 +835,587.3806122633417,1 +836,587.665554525084,1 +837,587.9504967868262,1 +838,588.2354390485684,1 +839,588.5203813103108,1 +840,588.805323572053,1 +841,589.0902658337952,1 +842,589.3752080955375,1 +843,589.6601503572797,1 +844,589.9450926190219,1 +845,590.2300348807642,1 +846,590.5149771425065,1 +847,590.7999194042486,1 +848,591.084861665991,1 +849,591.3698039277332,1 +850,591.6547461894754,1 +851,591.9396884512176,1 +852,592.2246307129599,1 +853,592.5095729747021,1 +854,592.7945152364443,1 +855,593.0794574981867,1 +856,593.3643997599289,1 +857,593.6493420216711,1 +858,593.9342842834134,1 +859,594.2192265451556,1 +860,594.5041688068978,1 +861,594.7891110686401,1 +862,595.0740533303824,1 +863,595.3589955921246,1 +864,595.6439378538669,1 +865,595.9288801156091,1 +866,596.2138223773513,1 +867,596.4987646390936,1 +868,596.7837069008358,1 +869,597.0686491625781,1 +870,597.3535914243203,1 +871,597.6385336860626,1 +872,597.9234759478048,1 +873,598.2084182095471,1 +874,598.4933604712893,1 +875,598.7783027330315,1 +876,599.0632449947738,1 +877,599.348187256516,1 +878,599.6331295182583,1 +879,599.9180717800006,1 +880,600.2030140417428,1 +881,600.487956303485,1 +882,600.7728985652273,1 +883,601.0578408269695,1 +884,601.3427830887117,1 +885,601.627725350454,1 +886,601.9126676121963,1 +887,602.1976098739385,1 +888,602.4825521356807,1 +889,602.767494397423,1 +890,603.0524366591652,1 +891,603.3373789209074,1 +892,603.6223211826498,1 +893,603.9072634443919,1 +894,604.1922057061342,1 +895,604.4771479678765,1 +896,604.7620902296187,1 +897,605.0470324913609,1 +898,605.3319747531032,1 +899,605.6169170148455,1 +900,605.9018592765876,1 +901,606.18680153833,1 +902,606.4717438000721,1 +903,606.7566860618144,1 +904,607.0416283235567,1 +905,607.3265705852989,1 +906,607.6115128470411,1 +907,607.8964551087834,1 +908,608.1813973705257,1 +909,608.4663396322678,1 +910,608.7512818940102,1 +911,609.0362241557524,1 +912,609.3211664174946,1 +913,609.6061086792369,1 +914,609.8910509409791,1 +915,610.1759932027214,1 +916,610.4609354644635,1 +917,610.7458777262059,1 +918,611.0308199879481,1 +919,611.3157622496904,1 +920,611.6007045114326,1 +921,611.8856467731748,1 +922,612.1705890349172,1 +923,612.4555312966593,1 +924,612.7404735584016,1 +925,613.0254158201437,1 +926,613.3103580818861,1 +927,613.5953003436283,1 +928,613.8802426053705,1 +929,614.1651848671128,1 +930,614.450127128855,1 +931,614.7350693905973,1 +932,615.0200116523395,1 +933,615.3049539140818,1 +934,615.589896175824,1 +935,615.8748384375663,1 +936,616.1597806993085,1 +937,616.4447229610507,1 +938,616.7296652227931,1 +939,617.0146074845352,1 +940,617.2995497462775,1 +941,617.5844920080198,1 +942,617.869434269762,1 +943,618.1543765315042,1 +944,618.4393187932465,1 +945,618.7242610549888,1 +946,619.0092033167309,1 +947,619.2941455784733,1 +948,619.5790878402154,1 +949,619.8640301019577,1 +950,620.1489723637,1 +951,620.4339146254422,1 +952,620.7188568871844,1 +953,621.0037991489266,1 +954,621.288741410669,1 +955,621.5736836724111,1 +956,621.8586259341535,1 +957,622.1435681958957,1 +958,622.4285104576379,1 +959,622.7134527193801,1 +960,622.9983949811224,1 +961,623.2833372428647,1 +962,623.5682795046068,1 +963,623.8532217663492,1 +964,624.1381640280913,1 +965,624.4231062898336,1 +966,624.7080485515759,1 +967,624.9929908133181,1 +968,625.2779330750604,1 +969,625.5628753368026,1 +970,625.8478175985449,1 +971,626.132759860287,1 +972,626.4177021220294,1 +973,626.7026443837716,1 +974,626.9875866455138,1 +975,627.2725289072561,1 +976,627.5574711689983,1 +977,627.8424134307406,1 +978,628.1273556924828,1 +979,628.4122979542251,1 +980,628.6972402159673,1 +981,628.9821824777096,1 +982,629.2671247394518,1 +983,629.552067001194,1 +984,629.8370092629364,1 +985,630.1219515246785,1 +986,630.4068937864208,1 +987,630.6918360481629,1 +988,630.9767783099053,1 +989,631.2617205716475,1 +990,631.5466628333897,1 +991,631.831605095132,1 +992,632.1165473568742,1 +993,632.4014896186166,1 +994,632.6864318803587,1 +995,632.971374142101,1 +996,633.2563164038432,1 +997,633.5412586655855,1 +998,633.8262009273277,1 +999,634.1111431890699,1 +1000,634.3960854508123,1 +1001,634.6810277125544,1 +1002,634.9659699742967,1 +1003,635.250912236039,1 +1004,635.5358544977812,1 +1005,635.8207967595234,1 +1006,636.1057390212657,1 +1007,636.390681283008,1 +1008,636.6756235447501,1 +1009,636.9605658064925,1 +1010,637.2455080682346,1 +1011,637.5304503299769,1 +1012,637.8153925917192,1 +1013,638.1003348534614,1 +1014,638.3852771152036,1 +1015,638.6702193769459,1 +1016,638.9551616386882,1 +1017,639.2401039004303,1 +1018,639.5250461621727,1 +1019,639.8099884239149,1 +1020,640.0949306856571,1 +1021,640.3798729473993,1 +1022,640.6648152091416,1 +1023,640.9497574708839,1 +1024,641.234699732626,1 +1025,641.5196419943684,1 +1026,641.8045842561106,1 +1027,642.0895265178528,1 +1028,642.3744687795951,1 +1029,642.6594110413373,1 +1030,642.9443533030797,1 +1031,643.2292955648218,1 +1032,643.5142378265641,1 +1033,643.7991800883062,1 +1034,644.0841223500486,1 +1035,644.3690646117908,1 +1036,644.654006873533,1 +1037,644.9389491352753,1 +1038,645.2238913970175,1 +1039,645.5088336587598,1 +1040,645.793775920502,1 +1041,646.0787181822443,1 +1042,646.3636604439865,1 +1043,646.6486027057288,1 +1044,646.933544967471,1 +1045,647.2184872292132,1 +1046,647.5034294909556,1 +1047,647.7883717526977,1 +1048,648.07331401444,1 +1049,648.3582562761821,1 +1050,648.6431985379245,1 +1051,648.9281407996667,1 +1052,649.213083061409,1 +1053,649.4980253231513,1 +1054,649.7829675848934,1 +1055,650.0679098466358,1 +1056,650.3528521083779,1 +1057,650.6377943701202,1 +1058,650.9227366318624,1 +1059,651.2076788936047,1 +1060,651.4926211553469,1 +1061,651.7775634170891,1 +1062,652.0625056788315,1 +1063,652.3474479405736,1 +1064,652.632390202316,1 +1065,652.9173324640582,1 +1066,653.2022747258004,1 +1067,653.4872169875426,1 +1068,653.7721592492849,1 +1069,654.0571015110272,1 +1070,654.3420437727693,1 +1071,654.6269860345117,1 +1072,654.9119282962538,1 +1073,655.1968705579961,1 +1074,655.4818128197384,1 +1075,655.7667550814806,1 +1076,656.0516973432228,1 +1077,656.3366396049651,1 +1078,656.6215818667074,1 +1079,656.9065241284495,1 +1080,657.1914663901919,1 +1081,657.4764086519341,1 +1082,657.7613509136763,1 +1083,658.0462931754186,1 +1084,658.3312354371608,1 +1085,658.6161776989031,1 +1086,658.9011199606452,1 +1087,659.1860622223876,1 +1088,659.4710044841298,1 +1089,659.755946745872,1 +1090,660.0408890076143,1 +1091,660.3258312693565,1 +1092,660.6107735310989,1 +1093,660.895715792841,1 +1094,661.1806580545833,1 +1095,661.4656003163254,1 +1096,661.7505425780678,1 +1097,662.03548483981,1 +1098,662.3204271015522,1 +1099,662.6053693632945,1 +1100,662.8903116250367,1 +1101,663.175253886779,1 +1102,663.4601961485212,1 +1103,663.7451384102635,1 +1104,664.0300806720057,1 +1105,664.315022933748,1 +1106,664.5999651954902,1 +1107,664.8849074572324,1 +1108,665.1698497189748,1 +1109,665.4547919807169,1 +1110,665.7397342424592,1 +1111,666.0246765042015,1 +1112,666.3096187659437,1 +1113,666.5945610276859,1 +1114,666.8795032894282,1 +1115,667.1644455511705,1 +1116,667.4493878129126,1 +1117,667.734330074655,1 +1118,668.0192723363971,1 +1119,668.3042145981394,1 +1120,668.5891568598817,1 +1121,668.8740991216239,1 +1122,669.1590413833661,1 +1123,669.4439836451083,1 +1124,669.7289259068507,1 +1125,670.0138681685928,1 +1126,670.2988104303352,1 +1127,670.5837526920774,1 +1128,670.8686949538196,1 +1129,671.1536372155618,1 +1130,671.4385794773041,1 +1131,671.7235217390464,1 +1132,672.0084640007885,1 +1133,672.2934062625309,1 +1134,672.578348524273,1 +1135,672.8632907860153,1 +1136,673.1482330477576,1 +1137,673.4331753094998,1 +1138,673.7181175712421,1 +1139,674.0030598329843,1 +1140,674.2880020947266,1 +1141,674.5729443564687,1 +1142,674.8578866182111,1 +1143,675.1428288799533,1 +1144,675.4277711416955,1 +1145,675.7127134034378,1 +1146,675.99765566518,1 +1147,676.2825979269223,1 +1148,676.5675401886645,1 +1149,676.8524824504068,1 +1150,677.137424712149,1 +1151,677.4223669738913,1 +1152,677.7073092356335,1 +1153,677.9922514973757,1 +1154,678.2771937591181,1 +1155,678.5621360208602,1 +1156,678.8470782826025,1 +1157,679.1320205443446,1 +1158,679.416962806087,1 +1159,679.7019050678292,1 +1160,679.9868473295714,1 +1161,680.2717895913137,1 +1162,680.5567318530559,1 +1163,680.8416741147983,1 +1164,681.1266163765404,1 +1165,681.4115586382827,1 +1166,681.696500900025,1 +1167,681.9814431617672,1 +1168,682.2663854235094,1 +1169,682.5513276852516,1 +1170,682.836269946994,1 +1171,683.1212122087361,1 +1172,683.4061544704784,1 +1173,683.6910967322207,1 +1174,683.9760389939629,1 +1175,684.2609812557051,1 +1176,684.5459235174474,1 +1177,684.8308657791897,1 +1178,685.1158080409318,1 +1179,685.4007503026742,1 +1180,685.6856925644163,1 +1181,685.9706348261586,1 +1182,686.2555770879009,1 +1183,686.5405193496431,1 +1184,686.8254616113853,1 +1185,687.1104038731276,1 +1186,687.3953461348699,1 +1187,687.680288396612,1 +1188,687.9652306583544,1 +1189,688.2501729200966,1 +1190,688.5351151818388,1 +1191,688.820057443581,1 +1192,689.1049997053233,1 +1193,689.3899419670656,1 +1194,689.6748842288077,1 +1195,689.9598264905501,1 +1196,690.2447687522923,1 +1197,690.5297110140345,1 +1198,690.8146532757768,1 +1199,691.099595537519,1 +1200,691.3845377992614,1 +1201,691.6694800610035,1 +1202,691.9544223227458,1 +1203,692.2393645844879,1 +1204,692.5243068462303,1 +1205,692.8092491079725,1 +1206,693.0941913697147,1 +1207,693.379133631457,1 +1208,693.6640758931992,1 +1209,693.9490181549415,1 +1210,694.2339604166837,1 +1211,694.518902678426,1 +1212,694.8038449401682,1 +1213,695.0887872019105,1 +1214,695.3737294636527,1 +1215,695.6586717253949,1 +1216,695.9436139871373,1 +1217,696.2285562488794,1 +1218,696.5134985106217,1 +1219,696.7984407723638,1 +1220,697.0833830341062,1 +1221,697.3683252958484,1 +1222,697.6532675575907,1 +1223,697.9382098193329,1 +1224,698.2231520810751,1 +1225,698.5080943428175,1 +1226,698.7930366045596,1 +1227,699.0779788663019,1 +1228,699.3629211280441,1 +1229,699.6478633897864,1 +1230,699.9328056515286,1 +1231,700.2177479132708,1 +1232,700.5026901750132,1 +1233,700.7876324367553,1 +1234,701.0725746984976,1 +1235,701.3575169602399,1 +1236,701.6424592219821,1 +1237,701.9274014837243,1 +1238,702.2123437454666,1 +1239,702.4972860072089,1 +1240,702.782228268951,1 +1241,703.0671705306934,1 +1242,703.3521127924355,1 +1243,703.6370550541778,1 +1244,703.9219973159201,1 +1245,704.2069395776623,1 +1246,704.4918818394045,1 +1247,704.7768241011468,1 +1248,705.0617663628891,1 +1249,705.3467086246312,1 +1250,705.6316508863736,1 +1251,705.9165931481158,1 +1252,706.201535409858,1 +1253,706.4864776716003,1 +1254,706.7714199333425,1 +1255,707.0563621950848,1 +1256,707.341304456827,1 +1257,707.6262467185693,1 +1258,707.9111889803115,1 +1259,708.1961312420538,1 +1260,708.481073503796,1 +1261,708.7660157655382,1 +1262,709.0509580272806,1 +1263,709.3359002890227,1 +1264,709.620842550765,1 +1265,709.9057848125071,1 +1266,710.1907270742495,1 +1267,710.4756693359917,1 +1268,710.7606115977339,1 +1269,711.0455538594762,1 +1270,711.3304961212184,1 +1271,711.6154383829607,1 +1272,711.9003806447029,1 +1273,712.1853229064452,1 +1274,712.4702651681874,1 +1275,712.7552074299297,1 +1276,713.0401496916719,1 +1277,713.3250919534141,1 +1278,713.6100342151565,1 +1279,713.8949764768986,1 +1280,714.1799187386409,1 +1281,714.4648610003832,1 +1282,714.7498032621254,1 +1283,715.0347455238676,1 +1284,715.3196877856099,1 +1285,715.6046300473522,1 +1286,715.8895723090943,1 +1287,716.1745145708367,1 +1288,716.4594568325788,1 +1289,716.7443990943211,1 +1290,717.0293413560634,1 +1291,717.3142836178056,1 +1292,717.5992258795478,1 +1293,717.88416814129,1 +1294,718.1691104030324,1 +1295,718.4540526647745,1 +1296,718.7389949265169,1 +1297,719.0239371882591,1 +1298,719.3088794500013,1 +1299,719.5938217117435,1 +1300,719.8787639734858,1 +1301,720.1637062352281,1 +1302,720.4486484969702,1 +1303,720.7335907587126,1 +1304,721.0185330204547,1 +1305,721.303475282197,1 +1306,721.5884175439393,1 +1307,721.8733598056815,1 +1308,722.1583020674237,1 +1309,722.443244329166,1 +1310,722.7281865909083,1 +1311,723.0131288526504,1 +1312,723.2980711143928,1 +1313,723.583013376135,1 +1314,723.8679556378772,1 +1315,724.1528978996195,1 +1316,724.4378401613617,1 +1317,724.722782423104,1 +1318,725.0077246848462,1 +1319,725.2926669465885,1 +1320,725.5776092083307,1 +1321,725.862551470073,1 +1322,726.1474937318152,1 +1323,726.4324359935574,1 +1324,726.7173782552998,1 +1325,727.0023205170419,1 +1326,727.2872627787842,1 +1327,727.5722050405263,1 +1328,727.8571473022687,1 +1329,728.1420895640109,1 +1330,728.4270318257531,1 +1331,728.7119740874954,1 +1332,728.9969163492376,1 +1333,729.28185861098,1 +1334,729.5668008727221,1 +1335,729.8517431344644,1 +1336,730.1366853962066,1 +1337,730.4216276579489,1 +1338,730.7065699196911,1 +1339,730.9915121814333,1 +1340,731.2764544431757,1 +1341,731.5613967049178,1 +1342,731.8463389666601,1 +1343,732.1312812284024,1 +1344,732.4162234901446,1 +1345,732.7011657518868,1 +1346,732.9861080136291,1 +1347,733.2710502753714,1 +1348,733.5559925371135,1 +1349,733.8409347988559,1 +1350,734.125877060598,1 +1351,734.4108193223403,1 +1352,734.6957615840826,1 +1353,734.9807038458248,1 +1354,735.265646107567,1 +1355,735.5505883693093,1 +1356,735.8355306310516,1 +1357,736.1204728927937,1 +1358,736.4054151545361,1 +1359,736.6903574162783,1 +1360,736.9752996780205,1 +1361,737.2602419397628,1 +1362,737.545184201505,1 +1363,737.8301264632473,1 +1364,738.1150687249894,1 +1365,738.4000109867318,1 +1366,738.684953248474,1 +1367,738.9698955102162,1 +1368,739.2548377719585,1 +1369,739.5397800337007,1 +1370,739.824722295443,1 +1371,740.1096645571852,1 +1372,740.3946068189275,1 +1373,740.6795490806696,1 +1374,740.964491342412,1 +1375,741.2494336041542,1 +1376,741.5343758658964,1 +1377,741.8193181276387,1 +1378,742.1042603893809,1 +1379,742.3892026511232,1 +1380,742.6741449128654,1 +1381,742.9590871746077,1 +1382,743.2440294363499,1 +1383,743.5289716980922,1 +1384,743.8139139598344,1 +1385,744.0988562215766,1 +1386,744.383798483319,1 +1387,744.6687407450611,1 +1388,744.9536830068034,1 +1389,745.2386252685455,1 +1390,745.5235675302879,1 +1391,745.8085097920301,1 +1392,746.0934520537724,1 +1393,746.3783943155146,1 +1394,746.6633365772568,1 +1395,746.9482788389992,1 +1396,747.2332211007413,1 +1397,747.5181633624836,1 +1398,747.8031056242259,1 +1399,748.0880478859681,1 +1400,748.3729901477103,1 +1401,748.6579324094525,1 +1402,748.9428746711949,1 +1403,749.227816932937,1 +1404,749.5127591946793,1 +1405,749.7977014564216,1 +1406,750.0826437181638,1 +1407,750.367585979906,1 +1408,750.6525282416483,1 +1409,750.9374705033906,1 +1410,751.2224127651327,1 +1411,751.5073550268751,1 +1412,751.7922972886172,1 +1413,752.0772395503595,1 +1414,752.3621818121018,1 +1415,752.647124073844,1 +1416,752.9320663355862,1 +1417,753.2170085973285,1 +1418,753.5019508590708,1 +1419,753.7868931208129,1 +1420,754.0718353825553,1 +1421,754.3567776442975,1 +1422,754.6417199060397,1 +1423,754.926662167782,1 +1424,755.2116044295242,1 +1425,755.4965466912665,1 +1426,755.7814889530086,1 +1427,756.066431214751,1 +1428,756.3513734764932,1 +1429,756.6363157382355,1 +1430,756.9212579999777,1 +1431,757.2062002617199,1 +1432,757.4911425234623,1 +1433,757.7760847852044,1 +1434,758.0610270469467,1 +1435,758.3459693086888,1 +1436,758.6309115704312,1 +1437,758.9158538321734,1 +1438,759.2007960939156,1 +1439,759.4857383556579,1 +1440,759.7706806174001,1 +1441,760.0556228791424,1 +1442,760.3405651408846,1 +1443,760.6255074026269,1 +1444,760.9104496643691,1 +1445,761.1953919261114,1 +1446,761.4803341878536,1 +1447,761.7652764495958,1 +1448,762.0502187113382,1 +1449,762.3351609730803,1 +1450,762.6201032348226,1 +1451,762.9050454965649,1 +1452,763.1899877583071,1 +1453,763.4749300200493,1 +1454,763.7598722817916,1 +1455,764.0448145435339,1 +1456,764.329756805276,1 +1457,764.6146990670184,1 +1458,764.8996413287605,1 +1459,765.1845835905028,1 +1460,765.469525852245,1 +1461,765.7544681139873,1 +1462,766.0394103757295,1 +1463,766.3243526374717,1 +1464,766.6092948992141,1 +1465,766.8942371609562,1 +1466,767.1791794226986,1 +1467,767.4641216844408,1 +1468,767.749063946183,1 +1469,768.0340062079252,1 +1470,768.3189484696675,1 +1471,768.6038907314098,1 +1472,768.8888329931519,1 +1473,769.1737752548943,1 +1474,769.4587175166364,1 +1475,769.7436597783787,1 +1476,770.028602040121,1 +1477,770.3135443018632,1 +1478,770.5984865636054,1 +1479,770.8834288253477,1 +1480,771.16837108709,1 +1481,771.4533133488321,1 +1482,771.7382556105745,1 +1483,772.0231978723167,1 +1484,772.3081401340589,1 +1485,772.5930823958012,1 +1486,772.8780246575434,1 +1487,773.1629669192857,1 +1488,773.4479091810279,1 +1489,773.7328514427702,1 +1490,774.0177937045124,1 +1491,774.3027359662547,1 +1492,774.5876782279969,1 +1493,774.8726204897391,1 +1494,775.1575627514815,1 +1495,775.4425050132236,1 +1496,775.7274472749659,1 +1497,776.012389536708,1 +1498,776.2973317984504,1 +1499,776.5822740601926,1 +1500,776.8672163219348,1 +1501,777.1521585836771,1 +1502,777.4371008454193,1 +1503,777.7220431071617,1 +1504,778.0069853689038,1 +1505,778.2919276306461,1 +1506,778.5768698923883,1 +1507,778.8618121541306,1 +1508,779.1467544158728,1 +1509,779.431696677615,1 +1510,779.7166389393574,1 +1511,780.0015812010995,1 +1512,780.2865234628418,1 +1513,780.5714657245841,1 +1514,780.8564079863263,1 +1515,781.1413502480685,1 +1516,781.4262925098108,1 +1517,781.7112347715531,1 +1518,781.9961770332952,1 +1519,782.2811192950376,1 +1520,782.5660615567797,1 +1521,782.851003818522,1 +1522,783.1359460802643,1 +1523,783.4208883420065,1 +1524,783.7058306037487,1 +1525,783.990772865491,1 +1526,784.2757151272333,1 +1527,784.5606573889754,1 +1528,784.8455996507178,1 +1529,785.13054191246,1 +1530,785.4154841742022,1 +1531,785.7004264359445,1 +1532,785.9853686976867,1 +1533,786.270310959429,1 +1534,786.5552532211711,1 +1535,786.8401954829135,1 +1536,787.1251377446557,1 +1537,787.410080006398,1 +1538,787.6950222681402,1 +1539,787.9799645298824,1 +1540,788.2649067916248,1 +1541,788.5498490533669,1 +1542,788.8347913151092,1 +1543,789.1197335768513,1 +1544,789.4046758385937,1 +1545,789.6896181003359,1 +1546,789.9745603620781,1 +1547,790.2595026238204,1 +1548,790.5444448855626,1 +1549,790.8293871473049,1 +1550,791.1143294090471,1 +1551,791.3992716707894,1 +1552,791.6842139325316,1 +1553,791.9691561942739,1 +1554,792.2540984560161,1 +1555,792.5390407177583,1 +1556,792.8239829795007,1 +1557,793.1089252412428,1 +1558,793.3938675029851,1 +1559,793.6788097647272,1 +1560,793.9637520264696,1 +1561,794.2486942882118,1 +1562,794.533636549954,1 +1563,794.8185788116963,1 +1564,795.1035210734385,1 +1565,795.3884633351809,1 +1566,795.673405596923,1 +1567,795.9583478586653,1 +1568,796.2432901204076,1 +1569,796.5282323821498,1 +1570,796.813174643892,1 +1571,797.0981169056342,1 +1572,797.3830591673766,1 +1573,797.6680014291187,1 +1574,797.952943690861,1 +1575,798.2378859526033,1 +1576,798.5228282143455,1 +1577,798.8077704760877,1 +1578,799.09271273783,1 +1579,799.3776549995723,1 +1580,799.6625972613144,1 +1581,799.9475395230568,1 +1582,800.2324817847989,1 +1583,800.5174240465412,1 +1584,800.8023663082835,1 +1585,801.0873085700257,1 +1586,801.3722508317679,1 +1587,801.6571930935102,1 +1588,801.9421353552525,1 +1589,802.2270776169946,1 +1590,802.512019878737,1 +1591,802.7969621404792,1 +1592,803.0819044022214,1 +1593,803.3668466639637,1 +1594,803.6517889257059,1 +1595,803.9367311874482,1 +1596,804.2216734491903,1 +1597,804.5066157109327,1 +1598,804.7915579726749,1 +1599,805.0765002344172,1 +1600,805.3614424961594,1 +1601,805.6463847579016,1 +1602,805.931327019644,1 +1603,806.2162692813861,1 +1604,806.5012115431284,1 +1605,806.7861538048705,1 +1606,807.0710960666129,1 +1607,807.3560383283551,1 +1608,807.6409805900973,1 +1609,807.9259228518396,1 +1610,808.2108651135818,1 +1611,808.4958073753241,1 +1612,808.7807496370663,1 +1613,809.0656918988086,1 +1614,809.3506341605508,1 +1615,809.6355764222931,1 +1616,809.9205186840353,1 +1617,810.2054609457775,1 +1618,810.4904032075199,1 +1619,810.775345469262,1 +1620,811.0602877310043,1 +1621,811.3452299927465,1 +1622,811.6301722544888,1 +1623,811.915114516231,1 +1624,812.2000567779733,1 +1625,812.4849990397156,1 +1626,812.7699413014577,1 +1627,813.0548835632001,1 +1628,813.3398258249422,1 +1629,813.6247680866845,1 +1630,813.9097103484268,1 +1631,814.194652610169,1 +1632,814.4795948719112,1 +1633,814.7645371336534,1 +1634,815.0494793953958,1 +1635,815.3344216571379,1 +1636,815.6193639188803,1 +1637,815.9043061806225,1 +1638,816.1892484423647,1 +1639,816.474190704107,1 +1640,816.7591329658492,1 +1641,817.0440752275915,1 +1642,817.3290174893336,1 +1643,817.613959751076,1 +1644,817.8989020128181,1 +1645,818.1838442745604,1 +1646,818.4687865363027,1 +1647,818.7537287980449,1 +1648,819.0386710597871,1 +1649,819.3236133215294,1 +1650,819.6085555832717,1 +1651,819.8934978450138,1 +1652,820.1784401067562,1 +1653,820.4633823684984,1 +1654,820.7483246302406,1 +1655,821.0332668919829,1 +1656,821.3182091537251,1 +1657,821.6031514154674,1 +1658,821.8880936772096,1 +1659,822.1730359389519,1 +1660,822.4579782006941,1 +1661,822.7429204624364,1 +1662,823.0278627241786,1 +1663,823.3128049859208,1 +1664,823.5977472476632,1 +1665,823.8826895094053,1 +1666,824.1676317711476,1 +1667,824.4525740328897,1 +1668,824.7375162946321,1 +1669,825.0224585563743,1 +1670,825.3074008181165,1 +1671,825.5923430798588,1 +1672,825.877285341601,1 +1673,826.1622276033434,1 +1674,826.4471698650855,1 +1675,826.7321121268278,1 +1676,827.01705438857,1 +1677,827.3019966503123,1 +1678,827.5869389120545,1 +1679,827.8718811737967,1 +1680,828.1568234355391,1 +1681,828.4417656972812,1 +1682,828.7267079590235,1 +1683,829.0116502207658,1 +1684,829.296592482508,1 +1685,829.5815347442502,1 +1686,829.8664770059925,1 +1687,830.1514192677348,1 +1688,830.4363615294769,1 +1689,830.7213037912193,1 +1690,831.0062460529614,1 +1691,831.2911883147037,1 +1692,831.576130576446,1 +1693,831.8610728381882,1 +1694,832.1460150999304,1 +1695,832.4309573616727,1 +1696,832.715899623415,1 +1697,833.0008418851571,1 +1698,833.2857841468995,1 +1699,833.5707264086417,1 +1700,833.8556686703839,1 +1701,834.1406109321262,1 +1702,834.4255531938684,1 +1703,834.7104954556107,1 +1704,834.9954377173528,1 +1705,835.2803799790952,1 +1706,835.5653222408373,1 +1707,835.8502645025796,1 +1708,836.1352067643219,1 +1709,836.4201490260641,1 +1710,836.7050912878065,1 +1711,836.9900335495486,1 +1712,837.2749758112909,1 +1713,837.559918073033,1 +1714,837.8448603347754,1 +1715,838.1298025965176,1 +1716,838.4147448582598,1 +1717,838.6996871200021,1 +1718,838.9846293817443,1 +1719,839.2695716434866,1 +1720,839.5545139052288,1 +1721,839.8394561669711,1 +1722,840.1243984287133,1 +1723,840.4093406904556,1 +1724,840.6942829521978,1 +1725,840.97922521394,1 +1726,841.2641674756824,1 +1727,841.5491097374245,1 +1728,841.8340519991668,1 +1729,842.118994260909,1 +1730,842.4039365226513,1 +1731,842.6888787843935,1 +1732,842.9738210461358,1 +1733,843.258763307878,1 +1734,843.5437055696202,1 +1735,843.8286478313626,1 +1736,844.1135900931047,1 +1737,844.398532354847,1 +1738,844.6834746165893,1 +1739,844.9684168783315,1 +1740,845.2533591400737,1 +1741,845.5383014018159,1 +1742,845.8232436635583,1 +1743,846.1081859253004,1 +1744,846.3931281870427,1 +1745,846.678070448785,1 +1746,846.9630127105272,1 +1747,847.2479549722694,1 +1748,847.5328972340117,1 +1749,847.817839495754,1 +1750,848.1027817574961,1 +1751,848.3877240192385,1 +1752,848.6726662809806,1 +1753,848.9576085427229,1 +1754,849.2425508044652,1 +1755,849.5274930662074,1 +1756,849.8124353279496,1 +1757,850.0973775896919,1 +1758,850.3823198514342,1 +1759,850.6672621131763,1 +1760,850.9522043749187,1 +1761,851.2371466366609,1 +1762,851.5220888984031,1 +1763,851.8070311601454,1 +1764,852.0919734218876,1 +1765,852.3769156836299,1 +1766,852.661857945372,1 +1767,852.9468002071144,1 +1768,853.2317424688566,1 +1769,853.5166847305989,1 +1770,853.8016269923411,1 +1771,854.0865692540833,1 +1772,854.3715115158257,1 +1773,854.6564537775678,1 +1774,854.9413960393101,1 +1775,855.2263383010522,1 +1776,855.5112805627946,1 +1777,855.7962228245368,1 +1778,856.081165086279,1 +1779,856.3661073480213,1 +1780,856.6510496097635,1 +1781,856.9359918715058,1 +1782,857.220934133248,1 +1783,857.5058763949903,1 +1784,857.7908186567325,1 +1785,858.0757609184748,1 +1786,858.360703180217,1 +1787,858.6456454419592,1 +1788,858.9305877037016,1 +1789,859.2155299654437,1 +1790,859.500472227186,1 +1791,859.7854144889282,1 +1792,860.0703567506705,1 +1793,860.3552990124127,1 +1794,860.640241274155,1 +1795,860.9251835358973,1 +1796,861.2101257976394,1 +1797,861.4950680593818,1 +1798,861.780010321124,1 +1799,862.0649525828662,1 +1800,862.3498948446085,1 +1801,862.6348371063507,1 +1802,862.9197793680929,1 +1803,863.2047216298351,1 +1804,863.4896638915774,1 +1805,863.7746061533197,1 +1806,864.059548415062,1 +1807,864.3444906768042,1 +1808,864.6294329385464,1 +1809,864.9143752002886,1 +1810,865.1993174620309,1 +1811,865.4842597237731,1 +1812,865.7692019855153,1 +1813,866.0541442472577,1 +1814,866.3390865089999,1 +1815,866.6240287707421,1 +1816,866.9089710324844,1 +1817,867.1939132942266,1 +1818,867.4788555559688,1 +1819,867.7637978177111,1 +1820,868.0487400794533,1 +1821,868.3336823411956,1 +1822,868.6186246029379,1 +1823,868.9035668646801,1 +1824,869.1885091264223,1 +1825,869.4734513881646,1 +1826,869.7583936499068,1 +1827,870.043335911649,1 +1828,870.3282781733914,1 +1829,870.6132204351336,1 +1830,870.8981626968758,1 +1831,871.1831049586181,1 +1832,871.4680472203603,1 +1833,871.7529894821025,1 +1834,872.0379317438448,1 +1835,872.322874005587,1 +1836,872.6078162673293,1 +1837,872.8927585290716,1 +1838,873.1777007908138,1 +1839,873.462643052556,1 +1840,873.7475853142982,1 +1841,874.0325275760405,1 +1842,874.3174698377827,1 +1843,874.6024120995249,1 +1844,874.8873543612673,1 +1845,875.1722966230095,1 +1846,875.4572388847517,1 +1847,875.742181146494,1 +1848,876.0271234082362,1 +1849,876.3120656699784,1 +1850,876.5970079317207,1 +1851,876.881950193463,1 +1852,877.1668924552052,1 +1853,877.4518347169475,1 +1854,877.7367769786897,1 +1855,878.0217192404319,1 +1856,878.3066615021742,1 +1857,878.5916037639164,1 +1858,878.8765460256586,1 +1859,879.161488287401,1 +1860,879.4464305491432,1 +1861,879.7313728108854,1 +1862,880.0163150726277,1 +1863,880.3012573343699,1 +1864,880.5861995961121,1 +1865,880.8711418578544,1 +1866,881.1560841195966,1 +1867,881.4410263813389,1 +1868,881.7259686430812,1 +1869,882.0109109048234,1 +1870,882.2958531665656,1 +1871,882.5807954283079,1 +1872,882.8657376900501,1 +1873,883.1506799517923,1 +1874,883.4356222135345,1 +1875,883.7205644752769,1 +1876,884.0055067370191,1 +1877,884.2904489987613,1 +1878,884.5753912605036,1 +1879,884.8603335222458,1 +1880,885.145275783988,1 +1881,885.4302180457303,1 +1882,885.7151603074726,1 +1883,886.0001025692148,1 +1884,886.2850448309571,1 +1885,886.5699870926993,1 +1886,886.8549293544415,1 +1887,887.1398716161838,1 +1888,887.424813877926,1 +1889,887.7097561396682,1 +1890,887.9946984014106,1 +1891,888.2796406631528,1 +1892,888.564582924895,1 +1893,888.8495251866373,1 +1894,889.1344674483795,1 +1895,889.4194097101217,1 +1896,889.704351971864,1 +1897,889.9892942336062,1 +1898,890.2742364953485,1 +1899,890.5591787570908,1 +1900,890.844121018833,1 +1901,891.1290632805752,1 +1902,891.4140055423175,1 +1903,891.6989478040597,1 +1904,891.9838900658019,1 +1905,892.2688323275441,1 +1906,892.5537745892865,1 +1907,892.8387168510287,1 +1908,893.123659112771,1 +1909,893.4086013745132,1 +1910,893.6935436362554,1 +1911,893.9784858979976,1 +1912,894.2634281597399,1 +1913,894.5483704214822,1 +1914,894.8333126832244,1 +1915,895.1182549449667,1 +1916,895.4031972067089,1 +1917,895.6881394684511,1 +1918,895.9730817301934,1 +1919,896.2580239919356,1 +1920,896.5429662536778,1 +1921,896.8279085154202,1 +1922,897.1128507771624,1 +1923,897.3977930389046,1 +1924,897.6827353006469,1 +1925,897.9676775623891,1 +1926,898.2526198241313,1 +1927,898.5375620858736,1 +1928,898.8225043476158,1 +1929,899.1074466093581,1 +1930,899.3923888711004,1 +1931,899.6773311328426,1 +1932,899.9622733945848,1 +1933,900.2472156563271,1 +1934,900.5321579180693,1 +1935,900.8171001798115,1 +1936,901.1020424415539,1 +1937,901.3869847032961,1 +1938,901.6719269650383,1 +1939,901.9568692267806,1 +1940,902.2418114885228,1 +1941,902.526753750265,1 +1942,902.8116960120072,1 +1943,903.0966382737495,1 +1944,903.3815805354918,1 +1945,903.666522797234,1 +1946,903.9514650589763,1 +1947,904.2364073207185,1 +1948,904.5213495824607,1 +1949,904.806291844203,1 +1950,905.0912341059452,1 +1951,905.3761763676874,1 +1952,905.6611186294298,1 +1953,905.946060891172,1 +1954,906.2310031529142,1 +1955,906.5159454146565,1 +1956,906.8008876763987,1 +1957,907.0858299381409,1 +1958,907.3707721998832,1 +1959,907.6557144616254,1 +1960,907.9406567233677,1 +1961,908.22559898511,1 +1962,908.5105412468522,1 +1963,908.7954835085944,1 +1964,909.0804257703367,1 +1965,909.3653680320789,1 +1966,909.6503102938211,1 +1967,909.9352525555635,1 +1968,910.2201948173057,1 +1969,910.5051370790479,1 +1970,910.7900793407902,1 +1971,911.0750216025324,1 +1972,911.3599638642746,1 +1973,911.6449061260168,1 +1974,911.9298483877591,1 +1975,912.2147906495014,1 +1976,912.4997329112437,1 +1977,912.7846751729859,1 +1978,913.0696174347281,1 +1979,913.3545596964703,1 +1980,913.6395019582126,1 +1981,913.9244442199548,1 +1982,914.209386481697,1 +1983,914.4943287434394,1 +1984,914.7792710051816,1 +1985,915.0642132669238,1 +1986,915.3491555286661,1 +1987,915.6340977904083,1 +1988,915.9190400521505,1 +1989,916.2039823138928,1 +1990,916.488924575635,1 +1991,916.7738668373773,1 +1992,917.0588090991196,1 +1993,917.3437513608618,1 +1994,917.628693622604,1 +1995,917.9136358843463,1 +1996,918.1985781460885,1 +1997,918.4835204078307,1 +1998,918.7684626695731,1 +1999,919.0534049313153,1 +2000,919.3383471930575,1 +2001,919.6232894547998,1 +2002,919.908231716542,1 +2003,920.1931739782842,1 +2004,920.4781162400265,1 +2005,920.7630585017687,1 +2006,921.048000763511,1 +2007,921.3329430252533,1 +2008,921.6178852869955,1 +2009,921.9028275487377,1 +2010,922.18776981048,1 +2011,922.4727120722222,1 +2012,922.7576543339644,1 +2013,923.0425965957066,1 +2014,923.327538857449,1 +2015,923.6124811191912,1 +2016,923.8974233809334,1 +2017,924.1823656426757,1 +2018,924.4673079044179,1 +2019,924.7522501661601,1 +2020,925.0371924279024,1 +2021,925.3221346896447,1 +2022,925.607076951387,1 +2023,925.8920192131292,1 +2024,926.1769614748714,1 +2025,926.4619037366136,1 +2026,926.7468459983559,1 +2027,927.0317882600981,1 diff --git a/wavelength_calibration/profiles.csv b/wavelength_calibration/profiles.csv new file mode 100644 index 0000000..a0ada5c --- /dev/null +++ b/wavelength_calibration/profiles.csv @@ -0,0 +1,1096 @@ +channel,pixel,black_corrected_adu,normalized_for_detection +R,229.0,45.09090909090901,0.009760705289672525 +R,231.0,68.84848484848499,0.014903442485306494 +R,233.0,76.60606060606096,0.016582703610411496 +R,235.0,58.66666666666697,0.012699412258606277 +R,237.0,67.87878787878799,0.01469353484466837 +R,239.0,50.90909090909099,0.011020151133501277 +R,241.0,82.90909090909099,0.01794710327455921 +R,243.0,63.515151515151956,0.013748950461796904 +R,245.0,48.969696969697,0.010600335852225026 +R,247.0,36.36363636363603,0.007871536523929398 +R,249.0,46.060606060606005,0.00997061293031065 +R,251.0,81.939393939394,0.017737195633921087 +R,253.0,47.030303030303,0.010180520570948776 +R,255.0,37.81818181818198,0.008186397984886686 +R,257.0,64.0,0.013853904282115867 +R,259.0,47.030303030303,0.010180520570948776 +R,261.0,58.18181818181802,0.012594458438287118 +R,263.0,53.81818181818198,0.011649874055415652 +R,265.0,30.54545454545496,0.006612090680100845 +R,267.0,20.363636363636033,0.004408060453400432 +R,269.0,61.57575757575796,0.013329135180520653 +R,271.0,42.66666666666697,0.00923593618807731 +R,273.0,45.09090909090901,0.009760705289672525 +R,275.0,38.78787878787898,0.00839630562552481 +R,277.0,57.69696969696997,0.012489504617968153 +R,279.0,156.121212121212,0.03379513014273717 +R,281.0,214.30303030303003,0.04638958858102429 +R,283.0,291.39393939393904,0.06307724601175474 +R,285.0,395.63636363636397,0.08564231738035272 +R,287.0,540.606060606061,0.11702350965575153 +R,289.0,694.787878787879,0.15039882451721245 +R,291.0,793.212121212121,0.17170445004198148 +R,293.0,897.939393939394,0.1943744752308984 +R,295.0,1002.666666666667,0.21704450041981532 +R,297.0,1025.939393939394,0.22208228379513015 +R,299.0,991.030303030303,0.21452560873215784 +R,301.0,1066.666666666667,0.2308984047019312 +R,303.0,983.757575757576,0.21295130142737198 +R,305.0,817.939393939394,0.17705709487825358 +R,307.0,686.545454545455,0.1486146095717885 +R,309.0,559.030303030303,0.12101175482787573 +R,311.0,363.63636363636397,0.07871536523929477 +R,313.0,250.66666666666697,0.05426112510495388 +R,315.0,211.878787878788,0.045864819479429074 +R,317.0,189.090909090909,0.040931989924433226 +R,319.0,121.69696969696997,0.02634340890008402 +R,321.0,137.21212121212102,0.029701931150293827 +R,323.0,67.39393939393904,0.014588581024349208 +R,325.0,109.09090909090901,0.023614609571788393 +R,327.0,119.27272727272702,0.02581863979848861 +R,329.0,72.72727272727298,0.015743073047858994 +R,331.0,100.36363636363603,0.021725440806045267 +R,333.0,82.42424242424204,0.01784214945424005 +R,335.0,49.939393939393995,0.010810243492863152 +R,337.0,50.42424242424204,0.010915197313182115 +R,339.0,36.36363636363603,0.007871536523929398 +R,341.0,31.515151515151956,0.00682199832073897 +R,343.0,48.0,0.010390428211586901 +R,345.0,76.12121212121201,0.016477749790092335 +R,347.0,39.75757575757598,0.008606213266162935 +R,349.0,26.66666666666697,0.0057724601175483435 +R,351.0,36.848484848484986,0.00797649034424856 +R,353.0,41.21212121212102,0.008921074727120025 +R,355.0,22.303030303030027,0.004827875734676682 +R,357.0,48.969696969697,0.010600335852225026 +R,359.0,30.54545454545496,0.006612090680100845 +R,361.0,43.63636363636397,0.009445843828715435 +R,363.0,36.36363636363603,0.007871536523929398 +R,365.0,48.0,0.010390428211586901 +R,367.0,76.12121212121201,0.016477749790092335 +R,369.0,44.12121212121201,0.0095507976490344 +R,371.0,44.12121212121201,0.0095507976490344 +R,373.0,48.969696969697,0.010600335852225026 +R,375.0,47.515151515151956,0.010285474391267937 +R,377.0,33.45454545454504,0.0072418136020150235 +R,379.0,21.818181818181984,0.004722921914357718 +R,381.0,59.63636363636397,0.012909319899244404 +R,383.0,21.818181818181984,0.004722921914357718 +R,385.0,68.84848484848499,0.014903442485306494 +R,387.0,74.66666666666697,0.016162888329135244 +R,389.0,108.60606060606096,0.02350965575146943 +R,391.0,152.24242424242402,0.03295549958018467 +R,393.0,291.39393939393904,0.06307724601175474 +R,395.0,387.878787878788,0.0839630562552477 +R,397.0,684.606060606061,0.14819479429051224 +R,399.0,934.787878787879,0.20235096557514695 +R,401.0,1284.848484848485,0.278127623845508 +R,403.0,1441.939393939394,0.31213266162888326 +R,405.0,1455.030303030303,0.31496641477749787 +R,407.0,1468.606060606061,0.31790512174643165 +R,409.0,1333.333333333333,0.28862300587741385 +R,411.0,1224.727272727273,0.2651133501259446 +R,413.0,1019.636363636364,0.22071788413098242 +R,415.0,844.121212121212,0.18272460117548275 +R,417.0,627.878787878788,0.13591519731318222 +R,419.0,374.30303030303,0.08102434928631395 +R,421.0,229.81818181818198,0.04974811083123429 +R,423.0,176.48484848484804,0.0382031905961376 +R,425.0,138.18181818181802,0.02991183879093195 +R,427.0,135.75757575757598,0.02938706968933674 +R,429.0,84.84848484848499,0.01836691855583546 +R,431.0,88.24242424242402,0.0191015952980688 +R,433.0,60.606060606060964,0.013119227539882528 +R,435.0,39.272727272727025,0.008501259445843774 +R,437.0,33.45454545454504,0.0072418136020150235 +R,439.0,42.18181818181802,0.00913098236775815 +R,441.0,11.151515151515014,0.002413937867338341 +R,443.0,37.81818181818198,0.008186397984886686 +R,445.0,34.90909090909099,0.007556675062972309 +R,447.0,57.69696969696997,0.012489504617968153 +R,449.0,37.33333333333303,0.008081444164567525 +R,451.0,66.90909090909099,0.014483627204030243 +R,453.0,38.78787878787898,0.00839630562552481 +R,455.0,44.606060606060964,0.009655751469353562 +R,457.0,22.78787878787898,0.004932829554995843 +R,459.0,63.515151515151956,0.013748950461796904 +R,461.0,34.90909090909099,0.007556675062972309 +R,463.0,51.87878787878799,0.011230058774139401 +R,465.0,28.12121212121201,0.006087321578505433 +R,467.0,30.54545454545496,0.006612090680100845 +R,469.0,47.030303030303,0.010180520570948776 +R,471.0,25.21212121212102,0.005457598656591058 +R,473.0,4.848484848484986,0.0010495382031906258 +R,475.0,30.060606060606005,0.0065071368597816835 +R,477.0,46.54545454545496,0.010075566750629811 +R,479.0,57.21212121212102,0.012384550797648991 +R,481.0,82.90909090909099,0.01794710327455921 +R,483.0,55.75757575757598,0.012069689336691903 +R,485.0,45.57575757575796,0.009865659109991686 +R,487.0,34.90909090909099,0.007556675062972309 +R,489.0,34.42424242424204,0.007451721242653148 +R,491.0,58.18181818181802,0.012594458438287118 +R,493.0,10.181818181818016,0.002204030226700216 +R,495.0,43.63636363636397,0.009445843828715435 +R,497.0,39.272727272727025,0.008501259445843774 +R,499.0,50.90909090909099,0.011020151133501277 +R,501.0,57.69696969696997,0.012489504617968153 +R,503.0,34.90909090909099,0.007556675062972309 +R,505.0,31.030303030303003,0.006717044500419809 +R,507.0,34.90909090909099,0.007556675062972309 +R,509.0,53.33333333333303,0.011544920235096491 +R,511.0,22.303030303030027,0.004827875734676682 +R,513.0,30.060606060606005,0.0065071368597816835 +R,515.0,41.69696969696997,0.009026028547439186 +R,517.0,34.90909090909099,0.007556675062972309 +R,519.0,32.969696969697,0.007136859781696059 +R,521.0,52.36363636363603,0.011335012594458366 +R,523.0,58.18181818181802,0.012594458438287118 +R,525.0,10.66666666666697,0.002308984047019377 +R,527.0,16.484848484848044,0.003568429890847931 +R,529.0,30.060606060606005,0.0065071368597816835 +R,531.0,28.12121212121201,0.006087321578505433 +R,533.0,49.45454545454504,0.01070528967254399 +R,535.0,43.151515151515014,0.009340890008396276 +R,537.0,17.939393939393995,0.0038832913518052174 +R,539.0,48.0,0.010390428211586901 +R,541.0,24.242424242424022,0.005247691015952932 +R,543.0,19.87878787878799,0.004303106633081468 +R,545.0,42.66666666666697,0.00923593618807731 +R,547.0,17.939393939393995,0.0038832913518052174 +R,549.0,26.181818181818016,0.005667506297229183 +R,551.0,48.969696969697,0.010600335852225026 +R,553.0,14.545454545454959,0.003148614609571878 +R,555.0,43.151515151515014,0.009340890008396276 +R,557.0,30.060606060606005,0.0065071368597816835 +R,559.0,52.848484848484986,0.011439966414777528 +R,561.0,14.545454545454959,0.003148614609571878 +R,563.0,47.515151515151956,0.010285474391267937 +R,565.0,67.87878787878799,0.01469353484466837 +R,567.0,52.848484848484986,0.011439966414777528 +R,569.0,84.36363636363603,0.0182619647355163 +R,571.0,45.09090909090901,0.009760705289672525 +R,573.0,46.54545454545496,0.010075566750629811 +R,575.0,65.939393939394,0.014273719563392118 +R,577.0,42.66666666666697,0.00923593618807731 +R,579.0,65.939393939394,0.014273719563392118 +R,581.0,53.81818181818198,0.011649874055415652 +R,583.0,71.75757575757598,0.01553316540722087 +R,585.0,51.393939393939036,0.01112510495382024 +R,587.0,64.0,0.013853904282115867 +R,589.0,70.78787878787898,0.015323257766582745 +R,591.0,51.87878787878799,0.011230058774139401 +R,593.0,64.969696969697,0.014063811922753994 +R,595.0,49.45454545454504,0.01070528967254399 +R,597.0,47.030303030303,0.010180520570948776 +R,599.0,72.24242424242402,0.015638119227539833 +R,601.0,44.606060606060964,0.009655751469353562 +R,603.0,55.75757575757598,0.012069689336691903 +R,605.0,68.36363636363603,0.014798488664987333 +R,607.0,56.24242424242402,0.012174643157010867 +R,609.0,73.69696969696997,0.01595298068849712 +R,611.0,56.727272727272975,0.012279596977330028 +R,613.0,79.51515151515196,0.01721242653232587 +R,615.0,96.969696969697,0.020990764063811927 +R,617.0,68.36363636363603,0.014798488664987333 +R,619.0,93.09090909090901,0.020151133501259428 +R,621.0,81.939393939394,0.017737195633921087 +R,623.0,67.87878787878799,0.01469353484466837 +R,625.0,83.39393939393904,0.018052057094878175 +R,627.0,32.484848484848044,0.007031905961376898 +R,629.0,66.90909090909099,0.014483627204030243 +R,631.0,49.939393939393995,0.010810243492863152 +R,633.0,51.87878787878799,0.011230058774139401 +R,635.0,65.939393939394,0.014273719563392118 +R,637.0,20.363636363636033,0.004408060453400432 +R,639.0,78.54545454545496,0.017002518891687746 +R,641.0,76.60606060606096,0.016582703610411496 +R,643.0,41.21212121212102,0.008921074727120025 +R,645.0,107.63636363636397,0.023299748110831305 +R,647.0,64.969696969697,0.014063811922753994 +R,649.0,65.939393939394,0.014273719563392118 +R,651.0,59.63636363636397,0.012909319899244404 +R,653.0,72.72727272727298,0.015743073047858994 +R,655.0,57.21212121212102,0.012384550797648991 +R,657.0,67.39393939393904,0.014588581024349208 +R,659.0,82.42424242424204,0.01784214945424005 +R,661.0,45.09090909090901,0.009760705289672525 +R,663.0,62.060606060606005,0.013434089000839618 +R,665.0,42.18181818181802,0.00913098236775815 +R,667.0,64.0,0.013853904282115867 +R,669.0,65.939393939394,0.014273719563392118 +R,671.0,52.36363636363603,0.011335012594458366 +R,673.0,48.969696969697,0.010600335852225026 +R,675.0,19.393939393939036,0.004198152812762306 +R,677.0,41.69696969696997,0.009026028547439186 +R,679.0,48.969696969697,0.010600335852225026 +R,681.0,32.969696969697,0.007136859781696059 +R,683.0,50.42424242424204,0.010915197313182115 +R,685.0,59.151515151515014,0.012804366078925242 +R,687.0,26.66666666666697,0.0057724601175483435 +R,689.0,16.969696969696997,0.0036733837111670923 +R,691.0,27.151515151515014,0.005877413937867308 +R,693.0,33.939393939393995,0.007346767422334185 +R,695.0,31.030303030303003,0.006717044500419809 +R,697.0,20.363636363636033,0.004408060453400432 +R,699.0,8.727272727272975,0.0018891687657431266 +R,701.0,52.36363636363603,0.011335012594458366 +R,703.0,27.151515151515014,0.005877413937867308 +R,705.0,-6.787878787878981,0.0 +R,707.0,16.0,0.003463476070528967 +R,709.0,15.030303030303003,0.0032535684298908418 +R,711.0,48.0,0.010390428211586901 +R,713.0,48.484848484848044,0.010495382031905864 +R,715.0,12.121212121212011,0.002623845507976466 +R,717.0,24.242424242424022,0.005247691015952932 +R,719.0,45.57575757575796,0.009865659109991686 +R,721.0,26.66666666666697,0.0057724601175483435 +R,723.0,23.272727272727025,0.0050377833753148075 +R,725.0,71.75757575757598,0.01553316540722087 +R,727.0,27.636363636363967,0.005982367758186469 +R,729.0,25.696969696969973,0.005562552476910219 +R,731.0,36.848484848484986,0.00797649034424856 +R,733.0,25.696969696969973,0.005562552476910219 +R,735.0,31.515151515151956,0.00682199832073897 +R,737.0,11.636363636363967,0.0025188916876575022 +R,739.0,20.848484848484986,0.004513014273719593 +R,741.0,22.78787878787898,0.004932829554995843 +R,743.0,37.33333333333303,0.008081444164567525 +R,745.0,-3.3939393939394904,0.0 +R,747.0,41.69696969696997,0.009026028547439186 +R,749.0,17.45454545454504,0.0037783375314860562 +R,751.0,-3.878787878787989,0.0 +R,753.0,38.30303030303003,0.00829135180520565 +R,755.0,18.90909090909099,0.004093198992443343 +R,757.0,70.30303030303003,0.015218303946263584 +R,759.0,46.54545454545496,0.010075566750629811 +R,761.0,31.515151515151956,0.00682199832073897 +R,763.0,26.66666666666697,0.0057724601175483435 +R,765.0,46.54545454545496,0.010075566750629811 +R,767.0,13.090909090909008,0.0028337531486145916 +R,769.0,33.939393939393995,0.007346767422334185 +R,771.0,13.575757575757962,0.0029387069689337524 +R,773.0,24.727272727272975,0.005352644836272093 +R,775.0,30.060606060606005,0.0065071368597816835 +R,777.0,45.09090909090901,0.009760705289672525 +R,779.0,24.727272727272975,0.005352644836272093 +R,781.0,43.151515151515014,0.009340890008396276 +R,783.0,36.36363636363603,0.007871536523929398 +R,785.0,56.727272727272975,0.012279596977330028 +R,787.0,47.030303030303,0.010180520570948776 +R,789.0,122.18181818181802,0.026448362720402983 +R,791.0,187.63636363636397,0.04061712846347614 +R,793.0,457.69696969697,0.09907640638119233 +R,795.0,1159.757575757576,0.25104953820319065 +R,797.0,3539.393939393939,0.766162888329135 +R,799.0,4619.636363636364,1.0 +R,801.0,1442.909090909091,0.3123425692695214 +R,803.0,619.636363636364,0.13413098236775825 +R,805.0,409.69696969697,0.08868597816960543 +R,807.0,434.909090909091,0.09414357682619648 +R,809.0,423.272727272727,0.09162468513853898 +R,811.0,393.212121212121,0.0851175482787573 +R,813.0,483.878787878788,0.10474391267842151 +R,815.0,442.66666666666697,0.09582283795130149 +R,817.0,441.212121212121,0.0955079764903442 +R,819.0,383.030303030303,0.08291351805205709 +R,821.0,338.909090909091,0.07336272040302268 +R,823.0,312.242424242424,0.06759026028547434 +R,825.0,292.36363636363603,0.06328715365239287 +R,827.0,282.181818181818,0.06108312342569266 +R,829.0,182.30303030303003,0.03946263643996635 +R,831.0,165.33333333333303,0.03578925272879926 +R,833.0,186.18181818181802,0.040302267002518856 +R,835.0,134.78787878787898,0.02917716204869861 +R,837.0,108.60606060606096,0.02350965575146943 +R,839.0,96.969696969697,0.020990764063811927 +R,841.0,87.75757575757598,0.018996641477749838 +R,843.0,53.81818181818198,0.011649874055415652 +R,845.0,45.09090909090901,0.009760705289672525 +R,847.0,32.0,0.006926952141057934 +R,849.0,25.21212121212102,0.005457598656591058 +R,851.0,19.87878787878799,0.004303106633081468 +R,853.0,67.87878787878799,0.01469353484466837 +R,855.0,45.57575757575796,0.009865659109991686 +R,857.0,25.696969696969973,0.005562552476910219 +R,859.0,27.636363636363967,0.005982367758186469 +R,861.0,19.393939393939036,0.004198152812762306 +R,863.0,7.272727272727025,0.0015743073047858405 +R,865.0,38.78787878787898,0.00839630562552481 +R,867.0,21.818181818181984,0.004722921914357718 +R,869.0,19.87878787878799,0.004303106633081468 +R,871.0,27.151515151515014,0.005877413937867308 +R,873.0,-1.9393939393939945,0.0 +R,875.0,3.3939393939390357,0.0007346767422333396 +R,877.0,8.727272727272975,0.0018891687657431266 +R,879.0,31.030303030303003,0.006717044500419809 +R,881.0,18.42424242424204,0.003988245172124182 +R,883.0,17.939393939393995,0.0038832913518052174 +R,885.0,24.727272727272975,0.005352644836272093 +R,887.0,63.515151515151956,0.013748950461796904 +R,889.0,37.33333333333303,0.008081444164567525 +R,891.0,34.90909090909099,0.007556675062972309 +R,893.0,19.393939393939036,0.004198152812762306 +R,895.0,33.45454545454504,0.0072418136020150235 +R,897.0,14.545454545454959,0.003148614609571878 +R,899.0,37.81818181818198,0.008186397984886686 +R,901.0,47.515151515151956,0.010285474391267937 +R,903.0,68.84848484848499,0.014903442485306494 +R,905.0,100.84848484848499,0.02183039462636443 +R,907.0,197.81818181818198,0.04282115869017636 +R,909.0,639.030303030303,0.13832913518052056 +R,911.0,1126.30303030303,0.24380772460117542 +R,913.0,743.272727272727,0.16089420654911832 +R,915.0,535.757575757576,0.11597397145256091 +R,917.0,1213.090909090909,0.2625944584382871 +R,919.0,1852.121212121212,0.4009235936188077 +R,921.0,934.30303030303,0.20224601175482781 +R,923.0,593.939393939394,0.12856842989084802 +R,925.0,511.030303030303,0.11062132661628882 +R,927.0,568.727272727273,0.12311083123425696 +R,929.0,570.666666666667,0.12353064651553322 +R,931.0,607.030303030303,0.13140218303946263 +R,933.0,668.121212121212,0.14462636439966411 +R,935.0,506.66666666666697,0.10967674223341735 +R,937.0,475.63636363636397,0.10295969773299754 +R,939.0,415.51515151515196,0.08994542401343418 +R,941.0,467.39393939393904,0.10117548278757338 +R,943.0,381.090909090909,0.08249370277078083 +R,945.0,296.242424242424,0.06412678421494537 +R,947.0,292.36363636363603,0.06328715365239287 +R,949.0,297.69696969697,0.06444164567590266 +R,951.0,243.878787878788,0.052791771620487 +R,953.0,240.48484848484804,0.05205709487825347 +R,955.0,169.69696969696997,0.03673383711167092 +R,957.0,160.48484848484804,0.03473971452560863 +G,228.5,448.0,0.007291785348068816 +G,230.5,365.090909090909,0.005942331566121014 +G,232.5,364.36363636363603,0.0059304942522442755 +G,234.5,328.969696969697,0.005354411643576507 +G,236.5,332.36363636363603,0.005409652441667931 +G,238.5,338.181818181818,0.005504350952681814 +G,240.5,307.39393939393904,0.005003237998566693 +G,242.5,292.36363636363603,0.0047586001784475015 +G,244.5,271.51515151515196,0.004419263847314441 +G,246.5,279.030303030303,0.0045415827573740295 +G,248.5,285.818181818182,0.004652064353556894 +G,250.5,294.30303030303,0.0047901663487854625 +G,252.5,289.45454545454504,0.004711250922940559 +G,254.5,283.63636363636397,0.004616552411926691 +G,256.5,258.181818181818,0.004202246426240955 +G,258.5,275.151515151515,0.004478450416698108 +G,260.5,260.60606060606096,0.004241704139163413 +G,262.5,254.060606060606,0.004135168314272791 +G,264.5,271.272727272727,0.004415318076022185 +G,266.5,269.090909090909,0.004379806134391983 +G,268.5,290.181818181818,0.004723088236817299 +G,270.5,248.24242424242402,0.004040469803258908 +G,272.5,264.969696969697,0.004312728022423819 +G,274.5,275.39393939393904,0.004482396187990349 +G,276.5,298.42424242424204,0.004857244460753626 +G,278.5,329.69696969697,0.005366248957453246 +G,280.5,388.848484848485,0.006329017152761031 +G,282.5,558.787878787879,0.009095002828624799 +G,284.5,638.545454545455,0.010393161583773416 +G,286.5,801.69696969697,0.01304866566345432 +G,288.5,996.848484848485,0.016225011553711567 +G,290.5,1148.121212121212,0.01868717284007246 +G,292.5,1330.909090909091,0.02166228439442522 +G,294.5,1392.484848484848,0.022664510302655448 +G,296.5,1414.30303030303,0.023019629718957504 +G,298.5,1493.818181818182,0.02431384270281388 +G,300.5,1409.939393939394,0.0229486058356971 +G,302.5,1349.575757575758,0.02196610878392809 +G,304.5,1189.090909090909,0.019354008188461874 +G,306.5,1048.242424242424,0.017061515067667506 +G,308.5,911.757575757576,0.014840045830133562 +G,310.5,672.0,0.010937678022103225 +G,312.5,526.060606060606,0.008562323704171716 +G,314.5,453.090909090909,0.0073746465452059606 +G,316.5,349.33333333333303,0.0056858564321250835 +G,318.5,349.818181818182,0.005693747974709582 +G,320.5,321.69696969697,0.00523603850480916 +G,322.5,301.090909090909,0.004900647944968327 +G,324.5,276.36363636363603,0.004498179273159329 +G,326.5,256.48484848484804,0.004174626027195235 +G,328.5,243.39393939393904,0.0039615543774140045 +G,330.5,237.81818181818198,0.0038708016376923777 +G,332.5,207.030303030303,0.0033696886835772556 +G,334.5,216.72727272727298,0.0035275195352670614 +G,336.5,208.72727272727298,0.003397309082622975 +G,338.5,214.54545454545496,0.0034920075936368585 +G,340.5,208.72727272727298,0.003397309082622975 +G,342.5,212.848484848485,0.003464387194591139 +G,344.5,231.27272727272702,0.003764265812801755 +G,346.5,222.060606060606,0.003614326503696447 +G,348.5,208.48484848484804,0.003393363311330719 +G,350.5,222.060606060606,0.003614326503696447 +G,352.5,216.0,0.0035156822213903223 +G,354.5,227.151515151515,0.0036971877008335915 +G,356.5,205.090909090909,0.0033381225132392946 +G,358.5,206.78787878787898,0.003365742912285014 +G,360.5,232.48484848484804,0.003783994669262977 +G,362.5,227.151515151515,0.0036971877008335915 +G,364.5,220.121212121212,0.0035827603333584857 +G,366.5,233.69696969696997,0.003803723525724214 +G,368.5,224.24242424242402,0.0036498384453266496 +G,370.5,250.909090909091,0.0040838732874736074 +G,372.5,225.939393939394,0.003677458844372369 +G,374.5,212.121212121212,0.0034525498807144 +G,376.5,242.18181818181802,0.003941825520952783 +G,378.5,206.54545454545496,0.0033617971409927727 +G,380.5,245.33333333333303,0.0039931205477519655 +G,382.5,233.45454545454504,0.0037997777544319577 +G,384.5,260.36363636363603,0.0042377583678711575 +G,386.5,305.212121212121,0.00496772605693649 +G,388.5,346.66666666666697,0.005642452947910399 +G,390.5,529.939393939394,0.008625456044847638 +G,392.5,785.939393939394,0.012792190529458389 +G,394.5,1130.424242424242,0.01839913153573857 +G,396.5,1653.575757575758,0.026914105984403358 +G,398.5,2361.69696969697,0.03843970392905109 +G,400.5,3039.757575757576,0.04947602623346044 +G,402.5,3640.0,0.05924575595305913 +G,404.5,3695.515151515152,0.060149337578983254 +G,406.5,3749.333333333333,0.06102529880586163 +G,408.5,3504.0,0.05703217825810967 +G,410.5,3252.848484848485,0.05294435919934382 +G,412.5,2783.030303030303,0.04529745443497295 +G,414.5,2302.060606060606,0.03746904419115881 +G,416.5,1743.272727272727,0.02837404136253401 +G,418.5,1171.393939393939,0.019065966884127983 +G,420.5,772.121212121212,0.01256728156580042 +G,422.5,584.484848484848,0.009513254585602761 +G,424.5,484.848484848485,0.007891542584490064 +G,426.5,434.42424242424204,0.0070708221557030885 +G,428.5,352.242424242424,0.005733205687632026 +G,430.5,327.757575757576,0.005334682787115285 +G,432.5,303.272727272727,0.004936159886598529 +G,434.5,256.727272727273,0.004178571798487491 +G,436.5,254.78787878787898,0.00414700562814953 +G,438.5,241.939393939394,0.003937879749660541 +G,440.5,240.48484848484804,0.003914205121907063 +G,442.5,198.060606060606,0.003223695145764189 +G,444.5,210.66666666666697,0.0034288752529609366 +G,446.5,217.21212121212102,0.0035354110778515442 +G,448.5,240.24242424242402,0.003910259350614821 +G,450.5,212.60606060606096,0.0034604414232988975 +G,452.5,234.42424242424204,0.003815560839600938 +G,454.5,228.36363636363603,0.0037169165572948134 +G,456.5,192.72727272727298,0.003136888177334803 +G,458.5,218.66666666666697,0.0035590857056050224 +G,460.5,216.24242424242402,0.0035196279926825638 +G,462.5,219.151515151515,0.0035669772481895052 +G,464.5,220.121212121212,0.0035827603333584857 +G,466.5,225.45454545454504,0.0036695673017878715 +G,468.5,230.060606060606,0.003744536956340533 +G,470.5,191.51515151515196,0.003117159320873581 +G,472.5,198.54545454545496,0.0032315866883486865 +G,474.5,202.909090909091,0.0033026105716090918 +G,476.5,221.33333333333303,0.0036024891898197076 +G,478.5,212.60606060606096,0.0034604414232988975 +G,480.5,214.54545454545496,0.0034920075936368585 +G,482.5,206.30303030303003,0.0033578513697005165 +G,484.5,208.72727272727298,0.003397309082622975 +G,486.5,214.30303030303003,0.0034880618223446023 +G,488.5,203.39393939393904,0.0033105021141935746 +G,490.5,192.24242424242402,0.0031289966347503055 +G,492.5,198.54545454545496,0.0032315866883486865 +G,494.5,191.75757575757598,0.0031211050921658226 +G,496.5,216.48484848484804,0.003523573763974805 +G,498.5,198.060606060606,0.003223695145764189 +G,500.5,213.33333333333303,0.003472278737175622 +G,502.5,196.36363636363603,0.0031960747467184693 +G,504.5,193.939393939394,0.003156617033796025 +G,506.5,233.45454545454504,0.0037997777544319577 +G,508.5,203.39393939393904,0.0033105021141935746 +G,510.5,176.0,0.002864629958169892 +G,512.5,188.60606060606096,0.0030698100653666397 +G,514.5,218.18181818181802,0.0035511941630205247 +G,516.5,217.939393939394,0.0035472483917282833 +G,518.5,218.18181818181802,0.0035511941630205247 +G,520.5,235.63636363636397,0.003835289696062175 +G,522.5,239.27272727272702,0.003894476265445841 +G,524.5,210.18181818181802,0.0034209837103764385 +G,526.5,240.0,0.00390631357932258 +G,528.5,213.57575757575796,0.003476224508467878 +G,530.5,228.121212121212,0.003712970786002572 +G,532.5,238.060606060606,0.003874747408984619 +G,534.5,248.969696969697,0.0040523071171356465 +G,536.5,231.030303030303,0.0037603200415095134 +G,538.5,214.54545454545496,0.0034920075936368585 +G,540.5,244.36363636363603,0.003977337462582985 +G,542.5,241.69696969696997,0.0039339339783683 +G,544.5,242.66666666666697,0.003949717063537281 +G,546.5,213.090909090909,0.0034683329658833804 +G,548.5,251.39393939393904,0.00409176483005809 +G,550.5,265.69696969697,0.0043245653363005575 +G,552.5,260.60606060606096,0.004241704139163413 +G,554.5,258.42424242424204,0.0042061921975331965 +G,556.5,264.969696969697,0.004312728022423819 +G,558.5,266.66666666666697,0.004340348421469538 +G,560.5,276.36363636363603,0.004498179273159329 +G,562.5,264.969696969697,0.004312728022423819 +G,564.5,282.66666666666697,0.0046007693267577105 +G,566.5,284.121212121212,0.004624443954511174 +G,568.5,313.69696969697,0.005105828052165074 +G,570.5,311.272727272727,0.005066370339242615 +G,572.5,312.242424242424,0.005082153424411596 +G,574.5,299.878787878788,0.0048809190885071045 +G,576.5,297.939393939394,0.0048493529181691435 +G,578.5,271.51515151515196,0.004419263847314441 +G,580.5,317.818181818182,0.005172906164133238 +G,582.5,343.272727272727,0.005587212149818959 +G,584.5,325.33333333333303,0.005295225074192826 +G,586.5,320.969696969697,0.00522420119093242 +G,588.5,342.060606060606,0.005567483293357737 +G,590.5,361.939393939394,0.005891036539321832 +G,592.5,378.181818181818,0.006155403215902245 +G,594.5,435.63636363636397,0.007090551012164325 +G,596.5,561.212121212121,0.009134460541547242 +G,598.5,823.030303030303,0.013395893537171877 +G,600.5,848.242424242424,0.013806253751565358 +G,602.5,852.606060606061,0.013877277634825777 +G,604.5,812.848484848485,0.013230171142897588 +G,606.5,697.212121212121,0.011348038236496704 +G,608.5,554.424242424242,0.009023978945364379 +G,610.5,524.848484848485,0.008542594847710493 +G,612.5,502.060606060606,0.008171692346239457 +G,614.5,510.060606060606,0.008301902798883543 +G,616.5,483.878787878788,0.007875759499321082 +G,618.5,485.57575757575796,0.007903379898366801 +G,620.5,502.30303030303,0.008175638117531699 +G,622.5,476.60606060606096,0.007757386360553736 +G,624.5,448.48484848484804,0.007299676890653299 +G,626.5,479.51515151515196,0.0078047356160606774 +G,628.5,432.48484848484804,0.0070392559853651275 +G,630.5,420.36363636363603,0.006841967420752877 +G,632.5,417.939393939394,0.006802509707830433 +G,634.5,374.54545454545496,0.006096216646518579 +G,636.5,415.030303030303,0.006755160452323492 +G,638.5,439.757575757576,0.0071576291241324886 +G,640.5,374.30303030303,0.006092270875226322 +G,642.5,375.272727272727,0.006108053960395303 +G,644.5,391.030303030303,0.006364529094391233 +G,646.5,380.848484848485,0.006198806700116945 +G,648.5,392.969696969697,0.006396095264729194 +G,650.5,406.787878787879,0.006621004228387164 +G,652.5,321.45454545454504,0.005232092733516903 +G,654.5,344.969696969697,0.005614832548864679 +G,656.5,357.33333333333303,0.00581606688476917 +G,658.5,384.0,0.006250101726916128 +G,660.5,358.060606060606,0.005827904198645909 +G,662.5,349.57575757575796,0.00568980220341734 +G,664.5,370.181818181818,0.0060251927632581585 +G,666.5,362.181818181818,0.005894982310614073 +G,668.5,339.151515151515,0.005520134037850796 +G,670.5,351.272727272727,0.005717422602463045 +G,672.5,379.63636363636397,0.006179077843655723 +G,674.5,371.878787878788,0.0060528131623038784 +G,676.5,381.818181818182,0.006214589785285925 +G,678.5,363.878787878788,0.005922602709659793 +G,680.5,310.787878787879,0.005058478796658132 +G,682.5,345.939393939394,0.005630615634033659 +G,684.5,347.878787878788,0.00566218180437162 +G,686.5,309.57575757575796,0.00503874994019691 +G,688.5,333.818181818182,0.0054333270694214095 +G,690.5,317.090909090909,0.0051610688502564984 +G,692.5,342.787878787879,0.005579320607234476 +G,694.5,322.42424242424204,0.005247875818685884 +G,696.5,306.181818181818,0.00498350914210547 +G,698.5,314.909090909091,0.005125556908626296 +G,700.5,325.818181818182,0.005303116616777324 +G,702.5,310.787878787879,0.005058478796658132 +G,704.5,310.060606060606,0.005046641482781393 +G,706.5,316.848484848485,0.005157123078964257 +G,708.5,309.818181818182,0.005042695711489151 +G,710.5,310.54545454545496,0.005054533025365891 +G,712.5,306.66666666666697,0.004991400684689969 +G,714.5,287.757575757576,0.004683630523894855 +G,716.5,309.57575757575796,0.00503874994019691 +G,718.5,275.878787878788,0.004490287730574846 +G,720.5,283.878787878788,0.004620498183218932 +G,722.5,318.30303030303,0.005180797706717721 +G,724.5,312.242424242424,0.005082153424411596 +G,726.5,285.57575757575796,0.004648118582264652 +G,728.5,314.42424242424204,0.005117665366041798 +G,730.5,313.212121212121,0.0050979365095805765 +G,732.5,327.51515151515196,0.005330737015823043 +G,734.5,307.63636363636397,0.005007183769858949 +G,736.5,308.121212121212,0.005015075312443432 +G,738.5,323.878787878788,0.005271550446439363 +G,740.5,352.727272727273,0.0057410972302165235 +G,742.5,317.090909090909,0.0051610688502564984 +G,744.5,329.45454545454504,0.00536230318616099 +G,746.5,336.727272727273,0.005480676324928351 +G,748.5,364.848484848485,0.005938385794828773 +G,750.5,306.909090909091,0.00499534645598221 +G,752.5,332.36363636363603,0.005409652441667931 +G,754.5,311.757575757576,0.005074261881827113 +G,756.5,358.060606060606,0.005827904198645909 +G,758.5,378.42424242424204,0.006159348987194486 +G,760.5,402.181818181818,0.0065460345738345025 +G,762.5,374.060606060606,0.006088325103934081 +G,764.5,370.909090909091,0.006037030077134898 +G,766.5,379.63636363636397,0.006179077843655723 +G,768.5,364.121212121212,0.005926548480952034 +G,770.5,402.66666666666697,0.006553926116419 +G,772.5,386.181818181818,0.006285613668546331 +G,774.5,397.818181818182,0.006475010690574098 +G,776.5,447.030303030303,0.007276002262899836 +G,778.5,478.787878787879,0.007792898302183938 +G,780.5,452.60606060606096,0.007366755002621478 +G,782.5,490.42424242424204,0.00798229532421169 +G,784.5,667.636363636364,0.010866654138842819 +G,786.5,1045.090909090909,0.017010220040868326 +G,788.5,1824.969696969697,0.02970376628802059 +G,790.5,3382.060606060606,0.05504745529811042 +G,792.5,6652.121212121212,0.10827196425920363 +G,794.5,16636.121212121212,0.27077460915902296 +G,796.5,51766.030303030304,0.842559779668131 +G,798.5,61439.0,1.0 +G,800.5,30124.606060606064,0.4903173238595365 +G,802.5,10786.90909090909,0.17557103941973487 +G,804.5,6976.0,0.113543514705643 +G,806.5,7138.90909090909,0.11619507301403165 +G,808.5,7285.818181818182,0.11858621041713215 +G,810.5,7111.757575757576,0.11575314662930021 +G,812.5,7888.0,0.1283875063070688 +G,814.5,7795.878787878788,0.1268881132160157 +G,816.5,7180.121212121212,0.11686585413371331 +G,818.5,6532.848484848484,0.10633064478341907 +G,820.5,5932.121212121212,0.09655302352123589 +G,822.5,5396.363636363636,0.08783286896537437 +G,824.5,4793.212121212122,0.07801578999026876 +G,826.5,4047.030303030303,0.06587070595273854 +G,828.5,3381.090909090909,0.05503167221294144 +G,830.5,2770.909090909091,0.0451001658703607 +G,832.5,2368.727272727273,0.0385541312965262 +G,834.5,2063.757575757576,0.033590351010881946 +G,836.5,1652.606060606061,0.026898322899234377 +G,838.5,1227.151515151515,0.01997349428134434 +G,840.5,919.515151515152,0.014966310511485408 +G,842.5,744.242424242424,0.01211351786719224 +G,844.5,584.727272727273,0.009517200356895017 +G,846.5,482.42424242424204,0.007852084871567605 +G,848.5,411.63636363636397,0.006699919654232067 +G,850.5,398.060606060606,0.006478956461866339 +G,852.5,352.48484848484804,0.005737151458924267 +G,854.5,316.848484848485,0.005157123078964257 +G,856.5,299.39393939393904,0.004873027545922607 +G,858.5,258.909090909091,0.004214083740117694 +G,860.5,261.090909090909,0.004249595681747896 +G,862.5,260.848484848485,0.004245649910455655 +G,864.5,246.54545454545496,0.004012849404213203 +G,866.5,231.030303030303,0.0037603200415095134 +G,868.5,212.60606060606096,0.0034604414232988975 +G,870.5,227.39393939393904,0.003701133472125833 +G,872.5,216.48484848484804,0.003523573763974805 +G,874.5,218.18181818181802,0.0035511941630205247 +G,876.5,204.60606060606096,0.0033302309706548113 +G,878.5,210.18181818181802,0.0034209837103764385 +G,880.5,195.39393939393904,0.003180291661549489 +G,882.5,185.939393939394,0.0030264065811519392 +G,884.5,177.939393939394,0.0028961961285078534 +G,886.5,168.0,0.002734419505525806 +G,888.5,176.72727272727298,0.002876467272046631 +G,890.5,185.45454545454504,0.0030185150385674416 +G,892.5,181.090909090909,0.0029474911553070363 +G,894.5,177.45454545454504,0.0028883045859233554 +G,896.5,167.75757575757598,0.0027304737342335648 +G,898.5,181.81818181818198,0.0029593284691837754 +G,900.5,243.39393939393904,0.0039615543774140045 +G,902.5,327.757575757576,0.005334682787115285 +G,904.5,508.121212121212,0.008270336628545582 +G,906.5,874.666666666667,0.014236342822420076 +G,908.5,2352.484848484848,0.038289764619945765 +G,910.5,5121.939393939394,0.083366255862553 +G,912.5,3206.545454545455,0.05219071688252502 +G,914.5,1914.424242424242,0.031159755894859 +G,916.5,3041.939393939394,0.049511538175090644 +G,918.5,5042.424242424242,0.08207204287869663 +G,920.5,3162.424242424242,0.05147258650733642 +G,922.5,2024.969696969697,0.03295902760412274 +G,924.5,1785.212121212121,0.029056659796092402 +G,926.5,1785.69696969697,0.0290645513386769 +G,928.5,1941.333333333333,0.031597736508298196 +G,930.5,1826.181818181818,0.02972349514448181 +G,932.5,1869.818181818182,0.030433733977085924 +G,934.5,1593.939393939394,0.025943446246511077 +G,936.5,1423.515151515152,0.023169569028062827 +G,938.5,1245.575757575758,0.020273372899554973 +G,940.5,1138.666666666667,0.018533287759674912 +G,942.5,1121.212121212121,0.01824919222663326 +G,944.5,962.181818181818,0.015660766258920523 +G,946.5,904.242424242424,0.01471772692007396 +G,948.5,798.30303030303,0.01299342486536288 +G,950.5,710.060606060606,0.011557164114985693 +G,952.5,600.969696969697,0.00978156703347543 +G,954.5,500.60606060606096,0.008148017718485993 +G,956.5,503.272727272727,0.00819142120270068 +B,228.0,249.21212121212102,0.004872961698900262 +B,230.0,171.63636363636397,0.003356086461888516 +B,232.0,242.909090909091,0.004749715585893061 +B,234.0,185.69696969696997,0.003631020098596896 +B,236.0,199.27272727272702,0.0038964732650739425 +B,238.0,216.72727272727298,0.004237770193401598 +B,240.0,260.848484848485,0.005100492984452031 +B,242.0,191.51515151515196,0.0037447857413727805 +B,244.0,219.151515151515,0.004285172544558207 +B,246.0,208.0,0.00406712172923777 +B,248.0,231.27272727272702,0.004522184300341292 +B,250.0,222.54545454545496,0.004351535836177482 +B,252.0,217.69696969696997,0.004256731133864245 +B,254.0,233.69696969696997,0.0045695866514979195 +B,256.0,216.24242424242402,0.004228289723170265 +B,258.0,224.48484848484804,0.004389457717102759 +B,260.0,178.42424242424204,0.0034888130451270305 +B,262.0,218.18181818181802,0.00426621160409556 +B,264.0,201.21212121212102,0.003934395145999237 +B,266.0,163.878787878788,0.003204398938187336 +B,268.0,220.60606060606096,0.004313613955252187 +B,270.0,289.939393939394,0.005669321198331438 +B,272.0,264.242424242424,0.005166856276071288 +B,274.0,328.727272727273,0.00642775881683732 +B,276.0,493.57575757575796,0.009651118695487303 +B,278.0,736.0,0.014391353811149032 +B,280.0,1190.787878787879,0.023284034888130453 +B,282.0,1910.787878787879,0.03736253318164581 +B,284.0,2775.272727272727,0.05426621160409555 +B,286.0,3888.484848484848,0.07603337125521424 +B,288.0,5083.151515151516,0.09939324990519531 +B,290.0,6237.09090909091,0.12195676905574518 +B,292.0,7194.181818181818,0.1406712172923777 +B,294.0,7937.939393939394,0.1552142586272279 +B,296.0,8319.030303030304,0.16266590822904817 +B,298.0,8451.878787878788,0.1652635570724308 +B,300.0,8341.333333333334,0.16310200985968903 +B,302.0,7911.757575757576,0.15470231323473643 +B,304.0,7019.636363636364,0.13725824800910125 +B,306.0,6161.454545454546,0.12047781569965871 +B,308.0,5056.0,0.09886234357224118 +B,310.0,3632.484848484848,0.07102768297307545 +B,312.0,2638.060606060606,0.05158323852863102 +B,314.0,1927.272727272727,0.037684869169510804 +B,316.0,1489.454545454545,0.029124004550625703 +B,318.0,1355.636363636364,0.026507394766780438 +B,320.0,1099.151515151515,0.021492226014410313 +B,322.0,903.272727272727,0.017662116040955626 +B,324.0,746.666666666667,0.014599924156238155 +B,326.0,679.757575757576,0.013291619264315513 +B,328.0,566.30303030303,0.011073189230185812 +B,330.0,491.151515151515,0.009603716344330675 +B,332.0,460.121212121212,0.008996966249525973 +B,334.0,432.0,0.008447098976109214 +B,336.0,395.151515151515,0.007726583238528628 +B,338.0,448.48484848484804,0.008769434963974204 +B,340.0,368.0,0.007195676905574516 +B,342.0,368.48484848484804,0.007205157375805831 +B,344.0,336.969696969697,0.006588926810769814 +B,346.0,320.969696969697,0.00627607129313614 +B,348.0,388.36363636363603,0.007593856655290096 +B,350.0,319.030303030303,0.006238149412210845 +B,352.0,336.0,0.006569965870307167 +B,354.0,363.63636363636397,0.007110352673492612 +B,356.0,330.181818181818,0.0064562002275312825 +B,358.0,362.181818181818,0.007081911262798632 +B,360.0,354.909090909091,0.006939704209328784 +B,362.0,396.121212121212,0.007745544178991276 +B,364.0,379.63636363636397,0.007423208191126286 +B,366.0,381.57575757575796,0.007461130072051581 +B,368.0,368.48484848484804,0.007205157375805831 +B,370.0,456.727272727273,0.008930602957906717 +B,372.0,432.0,0.008447098976109214 +B,374.0,476.121212121212,0.00930982176715965 +B,376.0,512.0,0.010011376564277587 +B,378.0,520.727272727273,0.010182025028441415 +B,380.0,572.121212121212,0.011186954872961697 +B,382.0,706.424242424242,0.013813045127038293 +B,384.0,877.090909090909,0.01715017064846416 +B,386.0,1357.575757575758,0.026545316647705732 +B,388.0,2234.666666666667,0.04369548729616989 +B,390.0,4027.636363636364,0.0787542662116041 +B,392.0,7478.30303030303,0.14622677284793326 +B,394.0,12578.909090909092,0.2459613196814562 +B,396.0,19731.39393939394,0.3858172165339401 +B,398.0,28753.454545454544,0.5622298065984073 +B,400.0,39955.393939393936,0.7812665908229047 +B,402.0,49086.06060606061,0.9598028062191885 +B,404.0,50641.454545454544,0.9902161547212741 +B,406.0,51141.818181818184,1.0 +B,408.0,49516.606060606064,0.9682214637846037 +B,410.0,45474.90909090909,0.8891922639362911 +B,412.0,39606.30303030303,0.774440652256352 +B,414.0,31662.545454545456,0.6191126279863481 +B,416.0,23431.757575757576,0.4581721653394008 +B,418.0,15301.81818181818,0.2992036405005688 +B,420.0,9328.484848484848,0.18240424725066362 +B,422.0,6058.666666666666,0.11846795601061812 +B,424.0,4272.484848484848,0.08354190367842244 +B,426.0,3398.30303030303,0.06644861585134622 +B,428.0,2499.393939393939,0.0488718240424725 +B,430.0,1833.212121212121,0.03584565794463405 +B,432.0,1389.090909090909,0.02716154721274175 +B,434.0,1164.121212121212,0.022762609025407658 +B,436.0,900.848484848485,0.017614713689799017 +B,438.0,731.636363636364,0.014306029579067127 +B,440.0,674.909090909091,0.013196814562002276 +B,442.0,655.030303030303,0.012808115282518013 +B,444.0,549.333333333333,0.010741372772089489 +B,446.0,521.212121212121,0.01019150549867273 +B,448.0,515.878787878788,0.010087220326128177 +B,450.0,455.757575757576,0.00891164201744407 +B,452.0,462.060606060606,0.00903488813045127 +B,454.0,478.060606060606,0.009347743648084944 +B,456.0,422.30303030303,0.00825748957148274 +B,458.0,463.030303030303,0.009053849070913916 +B,460.0,478.060606060606,0.009347743648084944 +B,462.0,461.090909090909,0.009015927189988622 +B,464.0,464.0,0.009072810011376563 +B,466.0,506.181818181818,0.009897610921501702 +B,468.0,511.51515151515196,0.010001896094046272 +B,470.0,525.575757575758,0.010276829730754652 +B,472.0,509.57575757575796,0.009963974213120978 +B,474.0,456.242424242424,0.008921122487675385 +B,476.0,456.242424242424,0.008921122487675385 +B,478.0,448.969696969697,0.008778915434205536 +B,480.0,443.63636363636397,0.008674630261660984 +B,482.0,420.848484848485,0.008229048160788777 +B,484.0,412.60606060606096,0.008067880166856283 +B,486.0,425.69696969697,0.008323852863102015 +B,488.0,402.42424242424204,0.007868790291998475 +B,490.0,370.42424242424204,0.007243079256731126 +B,492.0,367.51515151515196,0.007186196435343201 +B,494.0,331.151515151515,0.00647516116799393 +B,496.0,356.848484848485,0.006977626090254079 +B,498.0,356.848484848485,0.006977626090254079 +B,500.0,324.848484848485,0.00635191505498673 +B,502.0,318.060606060606,0.006219188471748197 +B,504.0,368.48484848484804,0.007205157375805831 +B,506.0,348.121212121212,0.006806977626090251 +B,508.0,362.66666666666697,0.007091391733029964 +B,510.0,308.848484848485,0.0060390595373530555 +B,512.0,283.151515151515,0.005536594615092906 +B,514.0,304.48484848484804,0.005953735305271133 +B,516.0,320.969696969697,0.00627607129313614 +B,518.0,319.030303030303,0.006238149412210845 +B,520.0,309.818181818182,0.006058020477815703 +B,522.0,292.848484848485,0.00572620401971938 +B,524.0,300.60606060606096,0.005877891543420561 +B,526.0,308.36363636363603,0.006029579067121722 +B,528.0,349.090909090909,0.006825938566552899 +B,530.0,278.30303030303,0.0054417899127796685 +B,532.0,298.181818181818,0.0058304891922639325 +B,534.0,298.66666666666697,0.005839969662495266 +B,536.0,246.30303030303003,0.004816078877512319 +B,538.0,295.272727272727,0.00577360637087599 +B,540.0,278.787878787879,0.005451270383011001 +B,542.0,251.151515151515,0.004910883579825556 +B,544.0,303.51515151515196,0.005934774364808503 +B,546.0,281.69696969697,0.005508153204398943 +B,548.0,294.787878787879,0.005764125900644675 +B,550.0,292.848484848485,0.00572620401971938 +B,552.0,267.151515151515,0.0052237390974592315 +B,554.0,276.848484848485,0.005413348502085706 +B,556.0,248.24242424242402,0.004854000758437614 +B,558.0,223.51515151515196,0.00437049677664013 +B,560.0,285.090909090909,0.0055745164960182 +B,562.0,280.242424242424,0.005479711793704963 +B,564.0,225.45454545454504,0.004408418657565407 +B,566.0,238.54545454545496,0.004664391353811157 +B,568.0,272.48484848484804,0.005328024270003783 +B,570.0,273.45454545454504,0.005346985210466431 +B,572.0,242.42424242424204,0.004740235115661729 +B,574.0,221.090909090909,0.004323094425483502 +B,576.0,245.81818181818198,0.0048065984072810045 +B,578.0,242.909090909091,0.004749715585893061 +B,580.0,230.78787878787898,0.004512703830109977 +B,582.0,260.848484848485,0.005100492984452031 +B,584.0,230.30303030303003,0.004503223359878644 +B,586.0,263.272727272727,0.005147895335608641 +B,588.0,262.787878787879,0.005138414865377326 +B,590.0,280.727272727273,0.005489192263936296 +B,592.0,251.63636363636397,0.004920364050056889 +B,594.0,326.30303030303,0.006380356465680692 +B,596.0,351.030303030303,0.0068638604474781945 +B,598.0,576.0,0.011262798634812287 +B,600.0,613.333333333333,0.011992794842624189 +B,602.0,600.727272727273,0.011746302616609788 +B,604.0,548.363636363636,0.010722411831626841 +B,606.0,501.33333333333303,0.009802806219188465 +B,608.0,403.878787878788,0.007897231702692456 +B,610.0,372.36363636363603,0.007281001137656421 +B,612.0,350.060606060606,0.0068448995070155465 +B,614.0,320.0,0.006257110352673493 +B,616.0,321.45454545454504,0.006285551763367455 +B,618.0,287.030303030303,0.0056124383769434955 +B,620.0,311.757575757576,0.006095942358740998 +B,622.0,267.63636363636397,0.005233219567690564 +B,624.0,235.151515151515,0.0045980280621918815 +B,626.0,255.030303030303,0.004986727341676146 +B,628.0,232.72727272727298,0.004550625711035272 +B,630.0,263.757575757576,0.005157375805839973 +B,632.0,246.78787878787898,0.004825559347743652 +B,634.0,213.33333333333303,0.004171406901782322 +B,636.0,168.24242424242402,0.003289723170269241 +B,638.0,207.51515151515196,0.004057641259006455 +B,640.0,196.36363636363603,0.003839590443686 +B,642.0,217.69696969696997,0.004256731133864245 +B,644.0,209.939393939394,0.004105043610163065 +B,646.0,204.121212121212,0.0039912779673871805 +B,648.0,225.45454545454504,0.004408418657565407 +B,650.0,174.060606060606,0.0034034888130451257 +B,652.0,174.060606060606,0.0034034888130451257 +B,654.0,146.42424242424204,0.0028631020098596814 +B,656.0,167.27272727272702,0.0032707622298065934 +B,658.0,176.48484848484804,0.0034508911642017357 +B,660.0,189.57575757575796,0.0037068638604474857 +B,662.0,176.0,0.003441410693970421 +B,664.0,159.51515151515196,0.0031190747061054314 +B,666.0,146.909090909091,0.002872582480091014 +B,668.0,176.48484848484804,0.0034508911642017357 +B,670.0,149.33333333333303,0.0029199848312476237 +B,672.0,146.909090909091,0.002872582480091014 +B,674.0,125.09090909090901,0.0024459613196814545 +B,676.0,150.78787878787898,0.002948426241941604 +B,678.0,124.60606060606096,0.0024364808494501396 +B,680.0,154.66666666666697,0.003024270003792194 +B,682.0,168.24242424242402,0.003289723170269241 +B,684.0,132.848484848485,0.0025976488433826343 +B,686.0,119.27272727272702,0.0023321956769055694 +B,688.0,102.30303030303003,0.0020003792188092477 +B,690.0,115.39393939393904,0.0022563519150549795 +B,692.0,115.87878787878799,0.002265832385286312 +B,694.0,120.72727272727298,0.0023606370875995496 +B,696.0,93.57575757575796,0.0018297307546454378 +B,698.0,70.78787878787898,0.0013841486537732308 +B,700.0,121.69696969696997,0.0023795980280621972 +B,702.0,100.36363636363603,0.0019624573378839525 +B,704.0,92.60606060606096,0.0018107698141827905 +B,706.0,127.030303030303,0.0024838832006067496 +B,708.0,86.30303030303003,0.001687523701175573 +B,710.0,88.72727272727298,0.0017349260523322006 +B,712.0,102.78787878787898,0.00200985968904058 +B,714.0,83.87878787878799,0.001640121350018963 +B,716.0,84.36363636363603,0.001649601820250278 +B,718.0,91.63636363636397,0.0017918088737201429 +B,720.0,101.33333333333303,0.0019814182783466 +B,722.0,126.060606060606,0.002464922260144102 +B,724.0,93.57575757575796,0.0018297307546454378 +B,726.0,93.09090909090901,0.0018202502844141052 +B,728.0,100.36363636363603,0.0019624573378839525 +B,730.0,91.15151515151501,0.0017823284034888102 +B,732.0,47.515151515151956,0.000929086082669709 +B,734.0,53.33333333333303,0.001042851725445576 +B,736.0,113.45454545454504,0.0022184300341296847 +B,738.0,91.63636363636397,0.0017918088737201429 +B,740.0,68.84848484848499,0.0013462267728479359 +B,742.0,129.45454545454504,0.0025312855517633593 +B,744.0,84.36363636363603,0.001649601820250278 +B,746.0,112.48484848484804,0.002199469093667037 +B,748.0,65.939393939394,0.0012893439514599933 +B,750.0,66.90909090909099,0.001308304891922641 +B,752.0,47.030303030303,0.0009196056124383764 +B,754.0,69.33333333333303,0.0013557072430792508 +B,756.0,46.54545454545496,0.0009101251422070615 +B,758.0,88.72727272727298,0.0017349260523322006 +B,760.0,104.24242424242402,0.0020383010997345424 +B,762.0,70.78787878787898,0.0013841486537732308 +B,764.0,69.33333333333303,0.0013557072430792508 +B,766.0,98.42424242424204,0.0019245354569586575 +B,768.0,56.727272727272975,0.0011092150170648513 +B,770.0,53.81818181818198,0.0010523321956769087 +B,772.0,73.21212121212102,0.0014315510049298407 +B,774.0,60.606060606060964,0.0011850587789154412 +B,776.0,86.30303030303003,0.001687523701175573 +B,778.0,75.63636363636397,0.0014789533560864683 +B,780.0,70.78787878787898,0.0013841486537732308 +B,782.0,68.36363636363603,0.0013367463026166032 +B,784.0,122.66666666666697,0.002398558968524845 +B,786.0,114.90909090909099,0.002246871444823665 +B,788.0,206.54545454545496,0.004038680318543808 +B,790.0,368.48484848484804,0.007205157375805831 +B,792.0,665.212121212121,0.0130072051573758 +B,794.0,1568.969696969697,0.03067880166856276 +B,796.0,5046.30303030303,0.0986727341676147 +B,798.0,9526.30303030303,0.1862722791050436 +B,800.0,4105.69696969697,0.08028062191884718 +B,802.0,1239.272727272727,0.024232081911262793 +B,804.0,806.787878787879,0.015775502464922264 +B,806.0,766.060606060606,0.014979142965491087 +B,808.0,800.484848484848,0.015652256351915046 +B,810.0,798.060606060606,0.015604854000758436 +B,812.0,949.818181818182,0.018572241183162686 +B,814.0,842.181818181818,0.01646757679180887 +B,816.0,789.333333333333,0.015434205536594608 +B,818.0,732.606060606061,0.014324990519529774 +B,820.0,647.272727272727,0.012656427758816832 +B,822.0,642.424242424242,0.012561623056503594 +B,824.0,593.454545454545,0.011604095563139924 +B,826.0,418.909090909091,0.008191126279863483 +B,828.0,393.69696969697,0.007698141827834666 +B,830.0,355.39393939393904,0.006949184679560099 +B,832.0,299.151515151515,0.0058494501327265806 +B,834.0,224.969696969697,0.0043989381873340925 +B,836.0,210.909090909091,0.004124004550625712 +B,838.0,123.63636363636397,0.002417519908987492 +B,840.0,88.72727272727298,0.0017349260523322006 +B,842.0,104.72727272727298,0.002047781569965875 +B,844.0,84.84848484848499,0.0016590822904816104 +B,846.0,42.18181818181802,0.000824800910125139 +B,848.0,102.30303030303003,0.0020003792188092477 +B,850.0,81.939393939394,0.001602199469093668 +B,852.0,63.515151515151956,0.0012419416003033837 +B,854.0,51.393939393939036,0.001004929844520281 +B,856.0,77.09090909090901,0.0015073947667804306 +B,858.0,51.87878787878799,0.0010144103147516138 +B,860.0,35.87878787878799,0.0007015547971179392 +B,862.0,48.0,0.0009385665529010238 +B,864.0,47.030303030303,0.0009196056124383764 +B,866.0,67.39393939393904,0.0013177853621539558 +B,868.0,28.606060606060964,0.000559347743648092 +B,870.0,67.39393939393904,0.0013177853621539558 +B,872.0,30.060606060606005,0.0005877891543420542 +B,874.0,44.606060606060964,0.0008722032612817665 +B,876.0,40.727272727272975,0.0007963594994311766 +B,878.0,24.242424242424022,0.0004740235115661694 +B,880.0,58.18181818181802,0.0011376564277588135 +B,882.0,53.81818181818198,0.0010523321956769087 +B,884.0,1.9393939393939945,3.792188092529497e-05 +B,886.0,40.24242424242402,0.0007868790291998439 +B,888.0,23.757575757575978,0.0004645430413348545 +B,890.0,58.18181818181802,0.0011376564277588135 +B,892.0,35.87878787878799,0.0007015547971179392 +B,894.0,48.0,0.0009385665529010238 +B,896.0,10.181818181818016,0.0001990898748577897 +B,898.0,49.939393939393995,0.0009764884338263188 +B,900.0,51.393939393939036,0.001004929844520281 +B,902.0,18.42424242424204,0.00036025786879028445 +B,904.0,63.515151515151956,0.0012419416003033837 +B,906.0,64.0,0.0012514220705346984 +B,908.0,146.909090909091,0.002872582480091014 +B,910.0,332.60606060606096,0.00650360257868791 +B,912.0,276.848484848485,0.005413348502085706 +B,914.0,129.45454545454504,0.0025312855517633593 +B,916.0,204.60606060606096,0.004000758437618513 +B,918.0,336.0,0.006569965870307167 +B,920.0,245.81818181818198,0.0048065984072810045 +B,922.0,175.51515151515196,0.003431930223739106 +B,924.0,157.090909090909,0.0030716723549488035 +B,926.0,151.75757575757598,0.0029673871824042516 +B,928.0,157.57575757575796,0.0030811528251801362 +B,930.0,140.121212121212,0.0027398558968524814 +B,932.0,159.51515151515196,0.0031190747061054314 +B,934.0,147.878787878788,0.0028915434205536617 +B,936.0,90.18181818181802,0.0017633674630261628 +B,938.0,73.69696969696997,0.0014410314751611734 +B,940.0,133.33333333333303,0.002607129313613949 +B,942.0,102.78787878787898,0.00200985968904058 +B,944.0,92.12121212121201,0.0018012893439514578 +B,946.0,64.0,0.0012514220705346984 +B,948.0,106.66666666666697,0.00208570345089117 +B,950.0,88.72727272727298,0.0017349260523322006 +B,952.0,52.36363636363603,0.0010238907849829287 +B,954.0,14.545454545454959,0.00028441410693971227 +B,956.0,68.36363636363603,0.0013367463026166032 diff --git a/wavelength_calibration/residuals.csv b/wavelength_calibration/residuals.csv new file mode 100644 index 0000000..a341dd8 --- /dev/null +++ b/wavelength_calibration/residuals.csv @@ -0,0 +1,4 @@ +id,pixel,reference_nm,fitted_nm,residual_nm +B1,304.0,435.8328,436.0762712782095,0.24347127820948344 +G1,686.5,546.0735,545.0666863946183,-1.0068136053816943 +G2,808.5,579.0663,579.8296423271722,0.7633423271722677 diff --git a/wavelength_player.py b/wavelength_player.py new file mode 100644 index 0000000..d53e80e --- /dev/null +++ b/wavelength_player.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Replay hq_record RAW recordings with saved wavelength calibration. + +Keep next to hq_spectrum.py and spectro_core.py. All original playback switches +are supported. +""" +from __future__ import annotations +import argparse +from contextlib import redirect_stdout +import io +from pathlib import Path +import sys + +import numpy as np + +try: # shared core lives beside this script + import spectro_core as core +except ImportError as exc: # pragma: no cover - placement error + raise SystemExit('wavelength_player.py needs spectro_core.py in the same directory ' + '(or on sys.path)') from exc + +# Kept as the historical name: the player validates a saved calibration exactly +# like every other tool now does. +SavedCalibration = core.Calibration + + +def load_player(): + # A private module instance provides hooks without editing the original file. + return core.load_hq() + + +def install_hooks(hq, calibration): + original_process = hq.process_frame + + def process(raw, meta, roi, pattern, args, model, full_size, raw_format, bit_depth): + result = original_process(raw, meta, roi, pattern, args, model, full_size, raw_format, bit_depth) + # Retain signed black-subtracted ADU; normalized copies are detection only. + planes = hq.extract_planes(raw, pattern, roi) + for c in hq.CHANNELS: + values = (np.median(planes[c], axis=0) if args.reduce == 'median' + else np.mean(planes[c], axis=0, dtype=np.float64)) - result.black + values = values[:len(result.x_px[c])] + positive = np.maximum(values, 0) + norm = positive / positive.max() if positive.size and positive.max() > 0 else np.zeros_like(values) + result.signal_raw[c] = values + result.signal[c] = values.copy() + result.normalized[c] = norm + result.peaks[c] = hq.find_peaks_normalized(norm, result.x_px[c], + max(1, int(np.ceil(args.peak_distance / 2))), args.peak_prominence, + args.peak_height, args.max_peaks) + return result + + class CalibratedDashboard(hq.Dashboard): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ax_norm.set_title('Normalized profiles (detection only; no smoothing)', fontsize=9) + for axis in (self.ax_spec, self.ax_norm, self.ax_water): + axis.axvspan(*calibration.nm_span, alpha=.08, color='green', zorder=0) + + def update(self, result, waterfall, fps, status_text): + # imshow needs an equally spaced nm grid, including quadratic/reverse mappings. + if waterfall.size: + gx = result.x_plot['G'] + xmin = min(float(x.min()) for x in result.x_plot.values()) + xmax = max(float(x.max()) for x in result.x_plot.values()) + grid = np.linspace(xmin, xmax, waterfall.shape[1]) + order = np.argsort(gx) + waterfall = np.asarray([np.interp(grid, gx[order], row[order]) for row in waterfall]) + super().update(result, waterfall, fps, status_text) + + def _target_y_limits(self, result): + # Signed signal (black already subtracted) can dip below zero, so the + # range is padded; rounding and hysteresis happen in Dashboard, which + # keeps the cached background valid instead of redrawing every frame. + valid = [v for v in result.signal.values() if v.size] + ymin = min(float(v.min()) for v in valid) if valid else 0.0 + ymax = max(float(v.max()) for v in valid) if valid else 1.0 + pad = max(1.0, (ymax - ymin) * 0.05) + return (hq._nice_floor(min(0.0, ymin - pad)), + hq._nice_ceiling(max(1.0, ymax + pad))) + + hq.process_frame = process + hq.Dashboard = CalibratedDashboard + hq.make_calibration = lambda args: calibration + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else list(argv) + options = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + options.add_argument('--calibration', default=str(Path(__file__).with_name('wavelength_calibration') / 'calibration.json')) + extra, rest = options.parse_known_args(argv) + try: + hq = load_player() + if '-h' in rest or '--help' in rest: + print('Wavelength recording player: wavelength_player.py --input REC_DIR --calibration calibration.json') + print('--calibration: saved JSON; --roi calibration: use saved ROI; default ROI is auto.') + print('No smoothing. Original player options follow:\n') + buffer = io.StringIO() + with redirect_stdout(buffer): + try: + hq.parse_args(['--help']) + except SystemExit: + pass + print(buffer.getvalue().replace('(default: 3)', '(default: 1)').replace( + 'Live R/G/B spectral analysis of Raspberry Pi HQ Camera RAW frames.', + 'Saved-calibration playback of HQ Camera RAW recordings.')) + print('Only recording playback is supported; --demo, --list-modes, --calibrate are unavailable.') + return 0 + args = hq.parse_args(['--smooth', '1'] + rest) + if not args.input or args.demo or args.list_modes or args.calibrate: + raise ValueError('use --input REC_DIR and --calibration JSON; live/demo/--calibrate are unsupported') + if args.smooth != 1: + raise ValueError('this player preserves unsmoothed signal; --smooth must be 1') + calibration = SavedCalibration(extra.calibration) + source = hq.RecordingSource(args.input) + calibration.validate_geometry(source) + args.width, args.height = source.full_size + if args.roi == 'calibration': + roi = calibration.data['roi'] + if len(roi) != 4 or any(not np.isfinite(float(v)) for v in roi): + raise ValueError('invalid saved ROI') + args.roi = ','.join(str(v) for v in roi) + print('Calibration:', calibration.path) + print('Descending coefficients:', calibration.coeff.tolist()) + print(calibration.describe()) + print('Use unchanged optics and sensor mode; each recording supplies its own black level.') + if args.headless and not args.frames: + print('Headless mode: processing all frames; use --frames N for a shorter run.') + install_hooks(hq, calibration) + return hq.run(args) + except (ValueError, KeyError, TypeError, OSError, RuntimeError) as exc: + print(f'Player error: {exc}', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main())