标定改为消费 peak_finder 的峰

- wavelength_calibration.py: 输入改为 peaks.json / selected_peaks.csv(--peaks),或在进程内调用 peak_finder.analyse()(--input);不再自带一套检测

- peak_finder.py: 抽出 analyse(),run() 与标定共用;--input 模式复用同一实现

- 新增 test_wavelength_calibration.py(14 项)

- 文档:PEAK_FINDER.md / README 改为 寻峰 -> 标定 两步流程
This commit is contained in:
starcells committed 2026-11-14 14:25:36 +08:00
1 parent 7fd4f56738
commit 341763190d
5 files changed
+546 -242

No files matched your search

+21
View File
@@ -13,6 +13,27 @@
支持的数据是 `hq_record.py` 录制的 RAW(`BGGR_PISP_COMP1`、2028×1520、uint8 容器)。
逐帧元数据里的黑电平(`SensorBlackLevels`,本数据集为 4096)会被自动使用。
## 标定:把峰交给 wavelength_calibration.py
标定不再是另起一套寻峰,而是**消费本程序导出的峰**:
```bash
# 1. 导出峰(不给 --calibration 就只导出像素坐标)
python peak_finder.py --input $rec --select all
# → peak_exports/<recording>_f0000/peaks.json
# 2. 看一眼峰表(编号 P001…、列坐标、FWHM、SNR、是否饱和)
python wavelength_calibration.py --peaks peak_exports/<recording>_f0000/peaks.json
# 3. 指定身份后拟合(ID:波长)
python wavelength_calibration.py --peaks peak_exports/<recording>_f0000/peaks.json ^
--assign "P001:404.6563,P002:435.8328,P003:546.0735,P004:576.9598,P005:579.0663"
```
`--peaks` 也可以指向 `selected_peaks.csv`(例如你用 `--pick` 挑过的那几条)。若改用
`--input <录制>`,标定脚本会在进程内调用本程序的 `analyse()`,得到同一套峰——两边
不存在第二套寻峰实现。
依赖:Python 3 + NumPy + Matplotlib,**不需要 SciPy**。脚本放在 `hq_spectrum.py`
同目录即可(本目录里那份 `hq_spectrum.py` 是 2026-11-14 版的副本,方便单独运行;
你自己的 `hq_spectrum.py` 更新后记得同步过来,或用 `--module-dir` 指过去)。
+12 -5
View File
@@ -54,15 +54,22 @@ python peak_finder.py --input recordings/hq_20261101_114417 \
python peak_finder.py --input recordings/hq_20261101_114417 --pick
```
重新标定:
重新标定(用寻峰结果,不重新造一套检测):
```bash
python wavelength_calibration.py --input recordings/hq_20261101_114417 --headless
# 看清 candidates.png 后指定:候选编号:波长
python wavelength_calibration.py --input recordings/hq_20261101_114417 --headless \
--assign "B1:404.6563,B2:435.8328,G1:546.0735,G2:576.9598,G3:579.0663"
# 1. 导出峰
python peak_finder.py --input recordings/hq_20261101_114417 --select all
# → peak_exports/hq_20261101_114417_f0000/peaks.json
# 2. 看峰表拿到编号,再指定身份
python wavelength_calibration.py --peaks peak_exports/hq_20261101_114417_f0000/peaks.json
python wavelength_calibration.py --peaks peak_exports/hq_20261101_114417_f0000/peaks.json ^
--assign "P001:404.6563,P002:435.8328,P003:546.0735,P004:576.9598,P005:579.0663"
```
`--peaks` 也可以指向 `selected_peaks.csv`;`--input <录制>` 则让标定脚本在内部调用
`peak_finder.analyse()`,同样是同一套峰。
## 测试
```bash
+32 -5
View File
@@ -1257,15 +1257,33 @@ def validate_arguments(args) -> None:
raise ValueError("--fit-window >= 0 and --match-tolerance > 0 required")
def run(args) -> int:
class Analysis:
"""Everything one recording yields: reduced spectrum, grouped lines, calibration."""
def __init__(self, hq, spectrum: "SpectrumData", lines: list[dict], calibration,
indices: list[int], per_channel: dict[str, list[dict]]):
self.hq = hq
self.spectrum = spectrum
self.lines = lines
self.calibration = calibration
self.indices = indices
self.per_channel = per_channel
def analyse(args, tell=print) -> "Analysis":
"""Read the frames, reduce the spectrum and detect/group lines.
Shared by ``run()`` (which then exports) and by wavelength_calibration.py
(which only needs the line positions).
"""
validate_arguments(args)
hq = load_hq(args.module_dir)
tell = (lambda *parts, **kwargs: None) if args.quiet else print
# A calibration is only applied when the caller names one: this program does
# not calibrate and must not silently attach someone else's wavelength axis.
calibration = None
if args.calibration:
if getattr(args, "calibration", None):
calibration = Calibration(args.calibration, args.wavelength_offset)
source = hq.RecordingSource(args.input)
@@ -1316,8 +1334,17 @@ def run(args) -> int:
attach_wavelength(lines, calibration)
for line in lines:
line["sample_step_px"] = spectrum.step_px
if args.match:
if getattr(args, "match", None):
match_lines(lines, load_line_table(args.match), args.match_tolerance)
return Analysis(hq, spectrum, lines, calibration, indices, per_channel)
def run(args) -> int:
analysis = analyse(args)
hq, spectrum, lines = analysis.hq, analysis.spectrum, analysis.lines
calibration, indices = analysis.calibration, analysis.indices
tell = (lambda *parts, **kwargs: None) if args.quiet else print
roi = spectrum.roi
if args.pick:
import matplotlib
@@ -1359,7 +1386,7 @@ def run(args) -> int:
f"{line['height_net_adu']:9.0f}{_num(line['fwhm_px'], 4):>9s}"
f"{_num(line['fwhm_nm'], 4):>9s}{_num(slit_row_centroid(spectrum, line), 6):>9s}"
f" {', '.join(flags)}")
total = sum(len(items) for items in per_channel.values())
total = sum(len(items) for items in analysis.per_channel.values())
tell("")
tell(f"{len(lines)} line(s) from {total} channel peak(s); {len(selected)} selected.")
tell(f"exported : {result['output'].expanduser().resolve()}")
+181
View File
@@ -0,0 +1,181 @@
"""Tests for wavelength_calibration: peaks in, calibration out."""
from __future__ import annotations
import csv
import json
import shutil
import os
import sys
import tempfile
import unittest
import uuid
from pathlib import Path
import numpy as np
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
TEST_TMP = os.environ.get('PEAK_FINDER_TEST_TMP')
def scratch():
"""Temporary directory that works inside restrictive sandboxes."""
base = Path(TEST_TMP) if TEST_TMP else Path(tempfile.gettempdir())
base.mkdir(parents=True, exist_ok=True)
path = base / f'wc_case_{uuid.uuid4().hex[:10]}'
path.mkdir()
return path
import wavelength_calibration as wc # noqa: E402
HG = {"P001": 404.6563, "P002": 435.8328, "P003": 546.0735,
"P004": 576.9598, "P005": 579.0663}
PIXELS = {"P001": 188.4, "P002": 302.5, "P003": 689.2, "P004": 802.1, "P005": 810.4}
def synthetic_peaks() -> list[dict]:
coeff = [0.2815541, 351.2738] # wavelength = coeff[0] * x + coeff[1]
peaks = []
for name, nm in HG.items():
x = (nm - coeff[1]) / coeff[0]
peaks.append({"id": name, "x": x, "fwhm_px": 8.0, "snr": 100.0,
"saturated": False, "channels": "G"})
return peaks
class LoadTests(unittest.TestCase):
def setUp(self):
self.dir = scratch()
def tearDown(self):
shutil.rmtree(self.dir, ignore_errors=True)
def test_json_roundtrip(self):
path = self.dir / "peaks.json"
path.write_text(json.dumps({
"recording": "some/recording", "frames": [0], "roi": [0, 426, 1270, 1396],
"full_size": [2028, 1520], "peaks": [
{"id": name, "sensor_x_px": x, "fwhm_px": 7.5, "snr": 42.0,
"saturated": False, "channels": "G"}
for name, x in PIXELS.items()]}), encoding="utf-8")
peaks, context = wc.read_peaks(path)
self.assertEqual([peak["id"] for peak in peaks], list(PIXELS))
self.assertAlmostEqual(peaks[2]["x"], 689.2, places=6)
self.assertEqual(context["full_size"], [2028, 1520])
self.assertEqual(context["roi"], [0, 426, 1270, 1396])
def test_csv_roundtrip_keeps_roi(self):
path = self.dir / "selected_peaks.csv"
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.writer(handle)
writer.writerow(["id", "sensor_x_px", "fwhm_px", "snr", "saturated",
"channels", "roi_x0", "roi_y0", "roi_x1", "roi_y1"])
for name, x in PIXELS.items():
writer.writerow([name, x, 7.5, 42.0, 0, "G", 0, 426, 1270, 1396])
peaks, context = wc.read_peaks(path)
self.assertEqual(len(peaks), 5)
self.assertEqual(context["roi"], [0, 426, 1270, 1396])
self.assertFalse(peaks[0]["saturated"])
def test_csv_without_id_column_is_rejected(self):
path = self.dir / "bad.csv"
path.write_text("name,x\nP001,1\n", encoding="utf-8")
with self.assertRaises(ValueError):
wc.read_peaks(path)
def test_empty_json_is_rejected(self):
path = self.dir / "empty.json"
path.write_text(json.dumps({"peaks": []}), encoding="utf-8")
with self.assertRaises(ValueError):
wc.read_peaks(path)
class AssignTests(unittest.TestCase):
def setUp(self):
self.peaks = synthetic_peaks()
def test_parses_ids_and_wavelengths(self):
confirmed = wc.parse_assign("P001:404.6563,P003:546.0735", self.peaks, 1)
self.assertEqual([peak["id"] for peak in confirmed], ["P001", "P003"])
self.assertAlmostEqual(confirmed[1]["wavelength_nm"], 546.0735)
def test_lowercase_ids_are_accepted(self):
confirmed = wc.parse_assign("p002:435.8328,p003:546.0735", self.peaks, 1)
self.assertEqual([peak["id"] for peak in confirmed], ["P002", "P003"])
def test_unknown_id_names_the_available_ones(self):
with self.assertRaises(ValueError) as raised:
wc.parse_assign("P999:500", self.peaks, 1)
self.assertIn("P001", str(raised.exception))
def test_duplicate_id_is_rejected(self):
with self.assertRaises(ValueError):
wc.parse_assign("P001:404.6,P001:435.8", self.peaks, 1)
def test_non_monotonic_wavelengths_are_rejected(self):
# x rises 188 -> 303 -> 689 but the wavelengths go 404 -> 579 -> 546
with self.assertRaises(ValueError):
wc.parse_assign("P001:404.6563,P002:579.0663,P003:546.0735", self.peaks, 1)
def test_too_few_points_for_the_order(self):
with self.assertRaises(ValueError):
wc.parse_assign("P001:404.6563,P002:435.8328", self.peaks, 2)
class FitTests(unittest.TestCase):
def test_recovers_a_linear_calibration(self):
confirmed = wc.parse_assign("P001:404.6563,P002:435.8328,P003:546.0735,"
"P004:576.9598,P005:579.0663", synthetic_peaks(), 1)
coeff, residual = wc.fit(confirmed, 1, 2028)
self.assertAlmostEqual(coeff[0], 0.2815541, places=6)
self.assertAlmostEqual(coeff[1], 351.2738, places=3)
self.assertLess(float(np.max(np.abs(residual))), 1e-6)
def test_non_monotonic_mapping_is_rejected(self):
# a quadratic through these points turns over inside the sensor width
confirmed = wc.parse_assign("P001:546.0735,P002:500.0,P003:470.0",
synthetic_peaks(), 1)
with self.assertRaises(ValueError):
wc.fit(confirmed, 2, 2028)
class OutputTests(unittest.TestCase):
def test_writes_calibration_json_and_tables(self):
tmp = scratch()
if True:
peaks = synthetic_peaks()
confirmed = wc.parse_assign("P001:404.6563,P002:435.8328,P003:546.0735,"
"P004:576.9598,P005:579.0663", peaks, 1)
coeff, residual = wc.fit(confirmed, 1, 2028)
args = wc.build_parser().parse_args(["--peaks", "x.json", "--order", "1",
"--output", str(tmp)])
context = {"source": "test", "full_size": [2028, 1520], "roi": [0, 426, 1270, 1396]}
info = wc.write_outputs(args, peaks, confirmed, coeff, residual, context)
out = Path(tmp)
self.assertTrue((out / "calibration.json").is_file())
self.assertTrue((out / "residuals.csv").is_file())
self.assertTrue((out / "pixel_wavelength.csv").is_file())
self.assertTrue((out / "calibration.png").is_file())
saved = json.loads((out / "calibration.json").read_text(encoding="utf-8"))
self.assertEqual(saved["order"], 1)
self.assertEqual(len(saved["confirmed"]), 5)
self.assertLess(saved["rms_nm"], 1e-6)
self.assertEqual(saved["calibrated_pixel_range"],
[min(peak["x"] for peak in confirmed), max(peak["x"] for peak in confirmed)])
self.assertEqual(info["roi"], [0, 426, 1270, 1396])
rows = list(csv.DictReader((out / "pixel_wavelength.csv").open(encoding="utf-8")))
self.assertEqual(len(rows), 2028)
def test_parser_requires_exactly_one_input(self):
parser = wc.build_parser()
args = parser.parse_args(["--peaks", "a.json"])
with self.assertRaises(OSError):
wc.run(args) # the file does not exist
args = parser.parse_args(["--peaks", "a.json", "--input", "rec"])
with self.assertRaises(ValueError):
wc.run(args)
if __name__ == "__main__":
unittest.main()
+300 -232
View File
@@ -1,276 +1,344 @@
#!/usr/bin/env python3
"""Independent Hg wavelength calibration; uses hq_spectrum recording/CFA APIs.
"""Fit the sensor-column -> wavelength calibration from peaks found by peak_finder.
Peak finding uses the same core as peak_finder.py: topographic prominence,
Gaussian centre with a peak-top fallback, and a joint two-line fit for blended
doublets (the Hg 576.96/579.07 pair is only ~6 px apart). Baseline subtraction
is used for detection and peak heights only; `signal_adu` still reports the
black-corrected value at the peak sample.
The peaks are the input, not the raw frames: either an exported peaks.json /
selected_peaks.csv, or a recording analysed by peak_finder in-process. Peak
identities are supplied as ID:WAVELENGTH pairs, e.g.
Sensor coordinates are full-resolution columns.
wavelength_calibration.py --peaks peak_exports/xxx/peaks.json \
--assign "P003:546.0735,P004:576.9598,P005:579.0663"
This program does not detect peaks; peak_finder.py does, and this one only fits
the polynomial and records the provenance.
"""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
import sys
from pathlib import Path
import numpy as np
import hq_spectrum as hq
try:
try: # shared core lives beside this script
import spectro_core as core
except ImportError as exc: # pragma: no cover - placement error
raise SystemExit('wavelength_calibration.py needs spectro_core.py in the same directory '
'(or on sys.path)') from exc
except ImportError as exc: # pragma: no cover - placement error
raise SystemExit('wavelength_calibration.py needs spectro_core.py in the same '
'directory (or on sys.path)') from exc
HG = (404.6563, 435.8328, 546.0735, 576.9598, 579.0663)
SOURCE = 'https://physics.nist.gov/PhysRefData/Handbook/Tables/mercurytable2.htm'
SOURCE = "https://physics.nist.gov/PhysRefData/Handbook/Tables/mercurytable2.htm"
def parser():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--input', required=True, help='hq_record recording directory')
p.add_argument('--frame', type=int, default=0, help='zero-based frame index (default 0)')
p.add_argument('--roi', default='auto', help='auto or x0,y0,x1,y1; same as hq_spectrum')
p.add_argument('--black', type=float, help='black level in decoded ADU')
p.add_argument('--full-scale', type=float)
p.add_argument('--reduce', choices=['mean', 'median'], default='mean')
p.add_argument('--baseline-window', type=float, default=100,
help='baseline opening width in sensor px; 0 disables the baseline')
p.add_argument('--peak-distance', type=float, default=4,
help='minimum distance in sensor pixels per channel (2 px = one sample)')
p.add_argument('--peak-prominence', type=float, default=0.01,
help='topographic prominence as a fraction of the channel maximum')
p.add_argument('--peak-height', type=float, default=0.02,
help='peak height as a fraction of the channel maximum')
p.add_argument('--min-snr', type=float, default=15, help='peak height in units of the noise')
p.add_argument('--max-peaks', type=int, default=40, help='per channel')
p.add_argument('--fwhm-span', type=float, default=60, help='half-width search range in px')
p.add_argument('--fit-window', type=float, default=0, help='fit window half width in px; 0 auto')
p.add_argument('--shape', choices=['gauss', 'lorentz'], default='gauss')
p.add_argument('--assign', help='confirmed candidate identities, e.g. B1:404.6563,B2:435.8328,G3:546.0735')
p.add_argument('--order', type=int, choices=[1, 2], default=1)
p.add_argument('--headless', action='store_true', help='save candidates; fit only with explicit --assign')
p.add_argument('--output', default='wavelength_calibration', help='output directory')
# --------------------------------------------------------------------------- CLI
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
source = p.add_argument_group("peaks")
source.add_argument("--peaks", metavar="PATH",
help="peaks.json or selected_peaks.csv written by peak_finder.py")
source.add_argument("--input", metavar="REC_DIR",
help="hq_record recording directory; run peak_finder's detection on it")
source.add_argument("--frame", type=int, default=0, help="zero-based frame index (default 0)")
source.add_argument("--average", type=int, default=1,
help="mean this many consecutive frames starting at --frame (default 1)")
source.add_argument("--roi", default="auto", help="'auto' or x0,y0,x1,y1")
source.add_argument("--rows", default=None, metavar="Y0,Y1",
help="restrict the ROI to these sensor rows")
source.add_argument("--reduce", choices=("mean", "median"), default="mean")
source.add_argument("--black", type=float, default=None, help="black level in ADU")
source.add_argument("--full-scale", type=float, default=None)
source.add_argument("--module-dir", default=None,
help="directory containing hq_spectrum.py (default this folder)")
fit_group = p.add_argument_group("fitting")
fit_group.add_argument("--assign", metavar="ID:NM,...",
help="confirmed peak identities, e.g. P003:546.0735,P004:576.9598")
fit_group.add_argument("--order", type=int, choices=(1, 2), default=1)
fit_group.add_argument("--output", default="wavelength_calibration", metavar="DIR")
fit_group.add_argument("--quiet", action="store_true", help="suppress the peak table")
return p
def extract(raw, meta, roi, pattern, depth, args):
planes = hq.extract_planes(raw, pattern, roi)
counts = {c: planes[c].shape[1] for c in hq.CHANNELS}
xs = hq.channel_x_positions(roi, pattern, counts['G'], min(counts['R'], counts['B']))
black, full = hq.resolve_levels(meta, depth, args.black, args.full_scale)
window = 0 if args.baseline_window <= 0 else max(3, int(round(args.baseline_window / 2.0)))
curves, candidates = {}, []
for c in hq.CHANNELS:
y = (np.median(planes[c], axis=0) if args.reduce == 'median'
else np.mean(planes[c], axis=0, dtype=np.float64)) - black
x = xs[c][:len(y)]
baseline = core.estimate_baseline(y, window, hq.smooth_vector)
net = y - baseline
noise = core.estimate_noise(net)
top = float(np.max(net)) if net.size else 0.0
positive = np.maximum(net, 0.0)
norm = positive / positive.max() if positive.max() > 0 else np.zeros(net.size)
peaks = []
if top > 0 and norm.size:
min_height = max(args.peak_height * top, args.min_snr * noise)
min_prominence = max(args.peak_prominence * top, 0.4 * args.min_snr * noise)
min_distance = max(1, int(round(args.peak_distance / 2.0)))
found = core.find_peak_candidates(net, min_distance, min_height,
min_prominence, args.max_peaks)
for candidate in found:
index = candidate['sample']
peaks.append(core.refine_peak(
x, net, y, candidate, noise, 2.0,
max(1, int(round(args.fwhm_span / 2.0))),
max(0, int(round(args.fit_window / 2.0))),
args.shape, core.is_saturated(raw, roi, x[index], full),
baseline_value=float(baseline[index])))
# Two lines closer than ~1.2 FWHM are fitted jointly, otherwise a
# single-peak fit collapses them onto one centre.
peaks = core.deblend_close_peaks(
x, net, core.dedupe_channel_peaks(peaks, args.peak_distance),
args.shape, 2.0, noise)
peaks.sort(key=lambda item: item['center_px'])
curves[c] = (x, y, baseline, net, norm)
for j, peak in enumerate(peaks, 1):
index = peak['sample_index']
candidates.append(dict(
id=f'{c}{j}', channel=c, sample=int(index),
pixel=float(peak['center_px']),
signal_adu=float(y[index]),
height_net_adu=float(peak['height_net_adu']),
normalized=float(norm[index]),
fwhm_px=float(peak['fwhm_px']),
snr=float(peak['snr']),
center_source=peak['center_source'],
deblended=bool(peak.get('deblended')),
saturated=bool(peak['saturated'])))
return curves, candidates, black, full
# ----------------------------------------------------------------- inputs
def _float(value) -> float:
try:
number = float(value)
except (TypeError, ValueError):
return float("nan")
return number if np.isfinite(number) else float("nan")
def assignments(text, candidates, order):
lookup = {v['id']: v for v in candidates}
result, used = [], set()
for token in text.split(','):
name, wavelength = token.strip().split(':')
name = name.upper().strip()
if name not in lookup or name in used:
raise ValueError(f'unknown or duplicate candidate: {name}')
nm = float(wavelength)
def _truthy(value) -> bool:
return str(value).strip().lower() in ("1", "true", "yes", "y")
def _normalise(record: dict) -> dict:
return {
"id": str(record["id"]).strip(),
"x": _float(record.get("sensor_x_px")),
"fwhm_px": _float(record.get("fwhm_px")),
"snr": _float(record.get("snr")),
"saturated": _truthy(record.get("saturated", False)),
"channels": str(record.get("channels", "") or ""),
}
def read_peaks(path: Path) -> tuple[list[dict], dict]:
"""Load peaks from peak_finder's peaks.json or selected_peaks.csv."""
context: dict = {"source": str(path.resolve())}
text = path.read_text(encoding="utf-8-sig")
if path.suffix.lower() == ".json":
data = json.loads(text)
records = data.get("peaks") or []
if not records:
raise ValueError(f"{path.name} contains no peaks")
for key in ("recording", "frames", "roi", "bayer_pattern", "raw_format",
"bit_depth", "black_adu", "full_scale_adu", "reduce", "full_size"):
if data.get(key) is not None:
context[key] = data[key]
else:
rows = list(csv.DictReader(text.splitlines()))
if not rows:
raise ValueError(f"{path.name} contains no rows")
missing = {"id", "sensor_x_px"} - set(rows[0])
if missing:
raise ValueError(f"{path.name} is missing column(s): {', '.join(sorted(missing))}")
records = rows
corners = ("roi_x0", "roi_y0", "roi_x1", "roi_y1")
if all(key in rows[0] and str(rows[0][key]).strip() for key in corners):
context["roi"] = [int(float(rows[0][key])) for key in corners]
peaks = [_normalise(record) for record in records]
bad = [peak["id"] for peak in peaks if not np.isfinite(peak["x"])]
if bad:
raise ValueError(f"peaks without a finite sensor_x_px: {', '.join(bad)}")
return peaks, context
def analyse_recording(args) -> tuple[list[dict], dict]:
"""Run peak_finder's detection on a recording and return its lines."""
import peak_finder as pf
defaults = pf.build_parser().parse_args(["--input", str(args.input)])
for key, value in vars(defaults).items():
if not hasattr(args, key) or getattr(args, key) is None:
setattr(args, key, value)
args.calibration = None # calibrating, not applying one
analysis = pf.analyse(args, tell=(lambda *a, **k: None) if args.quiet else print)
peaks = [_normalise({"id": line["id"], "sensor_x_px": line["sensor_x_px"],
"fwhm_px": line["fwhm_px"], "snr": line["snr"],
"saturated": line["saturated"],
"channels": line.get("channel_names", "")})
for line in analysis.lines]
spectrum = analysis.spectrum
context = {
"source": "peak_finder detection",
"recording": str(Path(args.input).expanduser().resolve()),
"frames": list(analysis.indices),
"roi": [spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1],
"bayer_pattern": spectrum.pattern,
"raw_format": spectrum.raw_format,
"bit_depth": spectrum.bit_depth,
"black_adu": spectrum.black,
"full_scale_adu": spectrum.full_scale,
"reduce": args.reduce,
"full_size": list(spectrum.full_size),
}
return peaks, context
# ----------------------------------------------------------------- fitting
def parse_assign(text: str, peaks: list[dict], order: int) -> list[dict]:
lookup = {peak["id"].upper(): peak for peak in peaks}
confirmed, used = [], set()
for token in text.split(","):
token = token.strip()
if not token:
continue
if ":" not in token:
raise ValueError(f"expected ID:wavelength, got {token!r}")
name, wavelength = token.split(":", 1)
name = name.strip().upper()
if name not in lookup:
raise ValueError(f"unknown peak {name}; exported ids are "
f"{', '.join(p['id'] for p in peaks[:10])}"
f"{'...' if len(peaks) > 10 else ''}")
if name in used:
raise ValueError(f"peak {name} assigned twice")
nm = _float(wavelength)
if not np.isfinite(nm) or nm <= 0:
raise ValueError('wavelength must be finite and positive')
raise ValueError(f"wavelength for {name} must be finite and positive")
used.add(name)
result.append({**lookup[name], 'wavelength_nm': nm})
if len(result) < order + 1:
raise ValueError(f'order {order} needs at least {order + 1} confirmed lines')
x = np.array([r['pixel'] for r in result])
y = np.array([r['wavelength_nm'] for r in result])
if len(np.unique(x)) != len(x) or len(np.unique(y)) != len(y):
raise ValueError('select one channel candidate per distinct spectral line')
confirmed.append({**lookup[name], "wavelength_nm": nm})
if len(confirmed) < order + 1:
raise ValueError(f"order {order} needs at least {order + 1} confirmed peaks")
x = np.array([peak["x"] for peak in confirmed])
if len(np.unique(x)) != len(x):
raise ValueError("two confirmed peaks share the same sensor column")
sort = np.argsort(x)
if not (np.all(np.diff(y[sort]) > 0) or np.all(np.diff(y[sort]) < 0)):
raise ValueError('confirmed wavelength order must be monotonic in sensor x')
return result
y = np.array([peak["wavelength_nm"] for peak in confirmed])[sort]
if not (np.all(np.diff(y) > 0) or np.all(np.diff(y) < 0)):
raise ValueError("confirmed wavelengths must be monotonic in sensor column")
return confirmed
def fit(confirmed, order, width):
x = np.array([r['pixel'] for r in confirmed])
y = np.array([r['wavelength_nm'] for r in confirmed])
model = np.polynomial.Polynomial.fit(x, y, order).convert()
coeff = model.coef[::-1]
derivative = np.polyval(np.polyder(coeff), [0, width - 1])
def fit(confirmed: list[dict], order: int, width: int | None) -> tuple[np.ndarray, np.ndarray]:
x = np.array([peak["x"] for peak in confirmed])
y = np.array([peak["wavelength_nm"] for peak in confirmed])
coeff = np.polynomial.Polynomial.fit(x, y, order).convert().coef[::-1]
high = float(width - 1) if width else float(x.max())
derivative = np.polyval(np.polyder(coeff), [float(x.min()), high])
if np.any(np.abs(derivative) < 1e-12) or derivative[0] * derivative[1] <= 0:
raise ValueError('mapping is not monotonic across sensor; revise identities/order/ROI')
raise ValueError("mapping is not monotonic across the sensor; revise identities/order")
residual = np.polyval(coeff, x) - y
return coeff, residual
def write_csv(path, fields, rows):
with path.open('w', newline='', encoding='utf-8') as f:
w = csv.writer(f)
w.writerow(fields)
w.writerows(rows)
def mean_abs_gap(peaks: list[dict]) -> float:
xs = np.sort([peak["x"] for peak in peaks])
return float(np.mean(np.diff(xs))) if len(xs) > 1 else float("nan")
def run(args):
if args.frame < 0 or args.peak_distance <= 0 or args.max_peaks < 1:
raise ValueError('frame >= 0, peak-distance > 0 and max-peaks >= 1 required')
for v in (args.peak_height, args.peak_prominence):
if not np.isfinite(v) or not 0 <= v <= 1:
raise ValueError('normalized peak thresholds must be between 0 and 1')
import matplotlib
if args.headless:
matplotlib.use('Agg')
import matplotlib.pyplot as plt
# ----------------------------------------------------------------- outputs
def write_outputs(args, peaks, confirmed, coeff, residual, context) -> dict:
out = Path(args.output)
out.mkdir(parents=True, exist_ok=True)
source = hq.RecordingSource(args.input)
try:
source.start()
raw, meta = source.read_at(args.frame)
roi = hq.parse_roi(args.roi, *source.full_size)
preview = hq.make_preview(raw, source.pattern)
if roi is None:
roi = hq.detect_auto_roi(preview, *source.full_size)
if roi is None:
raise ValueError('auto ROI failed; specify --roi x0,y0,x1,y1')
curves, candidates, black, full = extract(raw, meta, roi, source.pattern,
source.bit_depth, args)
finally:
source.close()
fields = ['id', 'channel', 'sample', 'pixel', 'signal_adu', 'height_net_adu',
'normalized', 'fwhm_px', 'snr', 'center_source', 'deblended', 'saturated']
write_csv(out / 'candidates.csv', fields, ([v[k] for k in fields] for v in candidates))
write_csv(out / 'profiles.csv',
['channel', 'pixel', 'black_corrected_adu', 'baseline_adu', 'net_adu',
'normalized_for_detection'],
([c, x, y, b, n, q] for c, arrays in curves.items()
for x, y, b, n, q in zip(*arrays)))
fig, axes = plt.subplots(3, 1, figsize=(13, 10), constrained_layout=True)
axes[0].imshow(preview, extent=(0, source.full_size[0], source.full_size[1], 0))
from matplotlib.patches import Rectangle
axes[0].add_patch(Rectangle((roi.x0, roi.y0), roi.width, roi.height, fill=False, edgecolor='orange'))
axes[0].set_title(f'Frame {args.frame} (zero based); ROI {roi.text()}; black={black:g} ADU')
for c, (x, y, baseline, net, norm) in curves.items():
axes[1].plot(x, y, color=hq.CHANNEL_COLORS[c], label=c)
axes[1].plot(x, baseline, color=hq.CHANNEL_COLORS[c], lw=0.6, ls=':', alpha=0.7)
axes[2].plot(x, norm, color=hq.CHANNEL_COLORS[c], label=c)
for v in candidates:
marker = 's' if v['deblended'] else 'o'
axes[2].plot(v['pixel'], v['normalized'], marker, color=hq.CHANNEL_COLORS[v['channel']],
mfc='none' if v['deblended'] else hq.CHANNEL_COLORS[v['channel']])
axes[2].annotate(v['id'], (v['pixel'], v['normalized']), xytext=(0, 8),
textcoords='offset points')
print(f"{v['id']:5s} x={v['pixel']:9.3f} FWHM={v['fwhm_px']:7.2f} px "
f"SNR={v['snr']:8.0f} normalized={v['normalized']:.4f}"
f"{' [pair]' if v['deblended'] else ''}{' [sat]' if v['saturated'] else ''}")
axes[1].set_ylabel('Black-corrected ADU'); axes[1].legend()
axes[2].set_ylabel('Normalized (detection only)'); axes[2].set_xlabel('Sensor column / px')
axes[2].set_ylim(-0.05, 1.18)
fig.savefig(out / 'candidates.png', dpi=160)
print('Hg I air wavelengths (nm):', HG)
print('Blended pairs such as 576.960/579.066 nm (about 7 px apart) are split by a joint '
'two-line fit and marked [pair]; use one channel per line.')
print('Select ONE channel candidate per line. Review candidates.png before confirming identities.')
text = args.assign
if not args.headless:
print('Inspect/zoom the plot, then CLOSE its window to enter confirmed identities in the terminal.')
plt.show(block=True)
if text is None:
text = input('Confirmed IDs:wavelength (comma separated), blank to export candidates only: ').strip()
plt.close(fig)
if not text:
print(f'Candidates exported to {out.resolve()}; no calibration fitted.')
return 0
confirmed = assignments(text, candidates, args.order)
coeff, residual = fit(confirmed, args.order, source.full_size[0])
x = np.array([r['pixel'] for r in confirmed])
x = np.array([peak["x"] for peak in confirmed])
lo, hi = float(x.min()), float(x.max())
info = dict(recording=str(Path(args.input).resolve()), frame_zero_based=args.frame,
roi=[roi.x0, roi.y0, roi.x1, roi.y1], pattern=source.pattern,
raw_format=source.raw_format, black_adu=black, full_scale_adu=full,
reduce=args.reduce, order=args.order, coefficients_descending=coeff.tolist(),
formula='wavelength_nm = numpy.polyval(coefficients_descending, sensor_x_px)',
wavelength_medium='air', reference=SOURCE, confirmed=confirmed,
residual_definition='fitted - reference (nm)', residuals_nm=residual.tolist(),
rms_nm=float(np.sqrt(np.mean(residual ** 2))), max_abs_nm=float(np.max(np.abs(residual))),
residual_degrees_of_freedom=len(x)-args.order-1, calibrated_pixel_range=[lo, hi],
processing='spatial reduction, black subtraction, morphological baseline for '
'peak detection, topographic prominence, Gaussian centres with '
'peak-top fallback and joint fits for blended doublets')
(out / 'calibration.json').write_text(json.dumps(info, indent=2), encoding='utf-8')
write_csv(out / 'residuals.csv', ['id', 'pixel', 'reference_nm', 'fitted_nm', 'residual_nm'],
([r['id'], r['pixel'], r['wavelength_nm'], r['wavelength_nm']+e, e] for r, e in zip(confirmed, residual)))
grid = np.arange(source.full_size[0])
write_csv(out / 'pixel_wavelength.csv', ['sensor_x_px', 'wavelength_nm', 'extrapolated'],
([int(p), float(w), int(p < lo or p > hi)] for p, w in zip(grid, np.polyval(coeff, grid))))
fig, ax = plt.subplots(2, 1, figsize=(10, 7), constrained_layout=True)
ax[0].plot(grid, np.polyval(coeff, grid)); ax[0].scatter(x, [r['wavelength_nm'] for r in confirmed])
ax[0].axvspan(lo, hi, alpha=.1, color='green', label='Confirmed span'); ax[0].legend()
ax[0].set_ylabel('Wavelength / nm (air)')
ax[1].axhline(0, color='gray'); ax[1].scatter(x, residual)
for r, e in zip(confirmed, residual):
ax[1].annotate(r['id'], (r['pixel'], e))
ax[1].set_ylabel('Fit - reference / nm'); ax[1].set_xlabel('Sensor column / px')
fig.savefig(out / 'calibration.png', dpi=160); plt.close(fig)
print('Coefficients (descending):', coeff)
print(f"RMS={info['rms_nm']:.6g} nm; max |residual|={info['max_abs_nm']:.6g} nm")
if len(x) == args.order + 1:
print('Exact interpolation: zero residual does not establish calibration accuracy; add independent lines.')
print(f'Calibration exported to {out.resolve()}')
width = context.get("full_size")
width = int(width[0]) if isinstance(width, (list, tuple)) and width else None
info = {
"recording": context.get("recording"),
"peaks_file": context.get("source"),
"frames": context.get("frames"),
"roi": context.get("roi"),
"bayer_pattern": context.get("bayer_pattern"),
"raw_format": context.get("raw_format"),
"black_adu": context.get("black_adu"),
"full_scale_adu": context.get("full_scale_adu"),
"reduce": context.get("reduce"),
"order": args.order,
"coefficients_descending": [float(value) for value in coeff],
"formula": "wavelength_nm = numpy.polyval(coefficients_descending, sensor_x_px)",
"wavelength_medium": "air",
"reference": SOURCE,
"confirmed": [{key: peak[key] for key in
("id", "x", "wavelength_nm", "fwhm_px", "snr", "saturated", "channels")}
for peak in confirmed],
"residual_definition": "fitted - reference (nm)",
"residuals_nm": [float(value) for value in residual],
"rms_nm": float(np.sqrt(np.mean(residual ** 2))),
"max_abs_nm": float(np.max(np.abs(residual))),
"residual_degrees_of_freedom": len(x) - args.order - 1,
"calibrated_pixel_range": [lo, hi],
"processing": "peaks from peak_finder.py (baseline, topographic prominence, "
"Gaussian centres, joint fits for blended pairs); polynomial fit here",
}
core.write_json(out / "calibration.json", info)
core.write_csv(out / "residuals.csv",
["id", "sensor_x_px", "reference_nm", "fitted_nm", "residual_nm"],
([peak["id"], peak["x"], peak["wavelength_nm"],
peak["wavelength_nm"] + err, err]
for peak, err in zip(confirmed, residual)))
if width:
grid = np.arange(width)
core.write_csv(out / "pixel_wavelength.csv",
["sensor_x_px", "wavelength_nm", "extrapolated"],
([int(px), float(nm), int(px < lo or px > hi)]
for px, nm in zip(grid, np.polyval(coeff, grid))))
_plot(out, confirmed, coeff, residual)
return info
def _plot(out: Path, confirmed: list[dict], coeff: np.ndarray, residual: np.ndarray) -> None:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
x = np.array([peak["x"] for peak in confirmed])
y = np.array([peak["wavelength_nm"] for peak in confirmed])
padding = 0.05 * (x.max() - x.min()) if x.max() > x.min() else 1.0
grid = np.linspace(x.min() - padding, x.max() + padding, 400)
fig, axes = plt.subplots(2, 1, figsize=(9, 6.5), constrained_layout=True)
axes[0].plot(grid, np.polyval(coeff, grid), lw=1.2, color="#2c3e50")
axes[0].plot(x, y, "o", ms=6, color="#c0392b")
for peak in confirmed:
axes[0].annotate(peak["id"], (peak["x"], peak["wavelength_nm"]),
xytext=(6, -10), textcoords="offset points", fontsize=8)
axes[0].set_ylabel("Wavelength / nm (air)")
axes[0].set_title(f"order {len(coeff) - 1}: {len(confirmed)} peaks, "
f"RMS {np.sqrt(np.mean(residual ** 2)):.3f} nm", fontsize=10)
axes[1].axhline(0, color="#7f8c8d", lw=0.8)
axes[1].plot(x, residual, "o", ms=6, color="#2980b9")
for peak, err in zip(confirmed, residual):
axes[1].annotate(f"{peak['id']} {err:+.3f}", (peak["x"], err),
xytext=(6, 6), textcoords="offset points", fontsize=8)
axes[1].set_ylabel("Fit - reference / nm")
axes[1].set_xlabel("Sensor column / px")
axes[1].grid(alpha=0.25)
fig.savefig(out / "calibration.png", dpi=160)
plt.close(fig)
# ----------------------------------------------------------------- main
def run(args) -> int:
if bool(args.peaks) == bool(args.input):
raise ValueError("give exactly one of --peaks PATH or --input REC_DIR")
if args.peaks:
peaks, context = read_peaks(Path(args.peaks).expanduser())
else:
peaks, context = analyse_recording(args)
if not peaks:
raise ValueError("no peaks to calibrate from")
if not args.quiet:
print(f"{'id':6s}{'x_px':>11s}{'FWHM px':>9s}{'SNR':>9s}{'ch':>4s} flags")
for peak in peaks:
flags = " saturated" if peak["saturated"] else ""
print(f"{peak['id']:6s}{peak['x']:11.3f}{peak['fwhm_px']:9.2f}"
f"{peak['snr']:9.0f}{peak['channels']:>4s}{flags}")
print(f"\n{len(peaks)} peak(s); median spacing {mean_abs_gap(peaks):.2f} px")
print("Assign identities with --assign ID:NM,... (e.g. P003:546.0735)")
print(f"Hg I air wavelengths (nm): {HG}")
if not args.assign:
print(f"No --assign given; nothing fitted. Peaks read from {context['source']}.")
return 0
confirmed = parse_assign(args.assign, peaks, args.order)
width = context.get("full_size")
width = int(width[0]) if isinstance(width, (list, tuple)) and width else None
coeff, residual = fit(confirmed, args.order, width)
info = write_outputs(args, peaks, confirmed, coeff, residual, context)
print("Coefficients (descending):", [f"{value:.8g}" for value in coeff])
print(f"RMS={info['rms_nm']:.6g} nm; max |residual|={info['max_abs_nm']:.6g} nm; "
f"{len(confirmed)} peak(s), order {args.order}")
saturated = [peak["id"] for peak in confirmed if peak["saturated"]]
if saturated:
print(f"warning: confirmed peak(s) saturated, centres are biased: {', '.join(saturated)}")
if len(confirmed) == args.order + 1:
print("Exact interpolation: zero residual does not establish calibration accuracy; "
"add independent lines.")
print(f"Calibration exported to {Path(args.output).expanduser().resolve()}")
return 0
def main(argv=None):
args = parser().parse_args(argv)
def main(argv=None) -> int:
args = build_parser().parse_args(argv)
try:
return run(args)
except (ValueError, RuntimeError, OSError, IndexError, StopIteration, EOFError) as e:
print(f'Calibration error: {e or "frame outside recording"}', file=sys.stderr)
except (ValueError, RuntimeError, OSError, KeyError, IndexError) as exc:
print(f"Calibration error: {exc}", file=sys.stderr)
return 2
if __name__ == '__main__':
if __name__ == "__main__":
raise SystemExit(main())