Files
spectrometer/wavelength_calibration.py
starcells ccc7c62569 统一默认输出目录命名
- 命名规则移入 spectro_core.default_output_directory(root, stem, indices)

- wavelength_calibration 默认改为 calibration_exports/<recording>_f<frame>,与 peak_exports 一致;输入为 peaks.json 时镜像其所在目录名

- 新增 3 项命名测试
2026-11-14 14:54:27 +08:00

367 lines
17 KiB
Python

#!/usr/bin/env python3
"""Fit the sensor-column -> wavelength calibration from peaks found by peak_finder.
The peaks are the input, not the raw frames: either an exported peaks.json /
selected_peaks.csv, or a recording analysed by peak_finder in-process. Peak
identities are supplied as ID:WAVELENGTH pairs, e.g.
wavelength_calibration.py --peaks peak_exports/xxx/peaks.json \
--assign "P003:546.0735,P004:576.9598,P005:579.0663"
This program does not detect peaks; peak_finder.py does, and this one only fits
the polynomial and records the provenance.
"""
from __future__ import annotations
import argparse
import csv
import json
import sys
from pathlib import Path
import numpy as np
try: # shared core lives beside this script
import spectro_core as core
except ImportError as exc: # pragma: no cover - placement error
raise SystemExit('wavelength_calibration.py needs spectro_core.py in the same '
'directory (or on sys.path)') from exc
HG = (404.6563, 435.8328, 546.0735, 576.9598, 579.0663)
SOURCE = "https://physics.nist.gov/PhysRefData/Handbook/Tables/mercurytable2.htm"
# --------------------------------------------------------------------------- CLI
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
source = p.add_argument_group("peaks")
source.add_argument("--peaks", metavar="PATH",
help="peaks.json or selected_peaks.csv written by peak_finder.py")
source.add_argument("--input", metavar="REC_DIR",
help="hq_record recording directory; run peak_finder's detection on it")
source.add_argument("--frame", type=int, default=0, help="zero-based frame index (default 0)")
source.add_argument("--average", type=int, default=1,
help="mean this many consecutive frames starting at --frame (default 1)")
source.add_argument("--roi", default="auto", help="'auto' or x0,y0,x1,y1")
source.add_argument("--rows", default=None, metavar="Y0,Y1",
help="restrict the ROI to these sensor rows")
source.add_argument("--reduce", choices=("mean", "median"), default="mean")
source.add_argument("--black", type=float, default=None, help="black level in ADU")
source.add_argument("--full-scale", type=float, default=None)
source.add_argument("--module-dir", default=None,
help="directory containing hq_spectrum.py (default this folder)")
fit_group = p.add_argument_group("fitting")
fit_group.add_argument("--assign", metavar="ID:NM,...",
help="confirmed peak identities, e.g. P003:546.0735,P004:576.9598")
fit_group.add_argument("--order", type=int, choices=(1, 2), default=1)
fit_group.add_argument("--output", default=None, metavar="DIR",
help="output directory (default calibration_exports/<recording>_f<frame>)")
fit_group.add_argument("--quiet", action="store_true", help="suppress the peak table")
return p
# ----------------------------------------------------------------- inputs
def _float(value) -> float:
try:
number = float(value)
except (TypeError, ValueError):
return float("nan")
return number if np.isfinite(number) else float("nan")
def _truthy(value) -> bool:
return str(value).strip().lower() in ("1", "true", "yes", "y")
def _normalise(record: dict) -> dict:
return {
"id": str(record["id"]).strip(),
"x": _float(record.get("sensor_x_px")),
"fwhm_px": _float(record.get("fwhm_px")),
"snr": _float(record.get("snr")),
"saturated": _truthy(record.get("saturated", False)),
"channels": str(record.get("channels", "") or ""),
}
def read_peaks(path: Path) -> tuple[list[dict], dict]:
"""Load peaks from peak_finder's peaks.json or selected_peaks.csv."""
context: dict = {"source": str(path.resolve())}
text = path.read_text(encoding="utf-8-sig")
if path.suffix.lower() == ".json":
data = json.loads(text)
records = data.get("peaks") or []
if not records:
raise ValueError(f"{path.name} contains no peaks")
for key in ("recording", "frames", "roi", "bayer_pattern", "raw_format",
"bit_depth", "black_adu", "full_scale_adu", "reduce", "full_size"):
if data.get(key) is not None:
context[key] = data[key]
else:
rows = list(csv.DictReader(text.splitlines()))
if not rows:
raise ValueError(f"{path.name} contains no rows")
missing = {"id", "sensor_x_px"} - set(rows[0])
if missing:
raise ValueError(f"{path.name} is missing column(s): {', '.join(sorted(missing))}")
records = rows
corners = ("roi_x0", "roi_y0", "roi_x1", "roi_y1")
if all(key in rows[0] and str(rows[0][key]).strip() for key in corners):
context["roi"] = [int(float(rows[0][key])) for key in corners]
peaks = [_normalise(record) for record in records]
bad = [peak["id"] for peak in peaks if not np.isfinite(peak["x"])]
if bad:
raise ValueError(f"peaks without a finite sensor_x_px: {', '.join(bad)}")
return peaks, context
def analyse_recording(args) -> tuple[list[dict], dict]:
"""Run peak_finder's detection on a recording and return its lines."""
import peak_finder as pf
defaults = pf.build_parser().parse_args(["--input", str(args.input)])
for key, value in vars(defaults).items():
if not hasattr(args, key) or getattr(args, key) is None:
setattr(args, key, value)
args.calibration = None # calibrating, not applying one
analysis = pf.analyse(args, tell=(lambda *a, **k: None) if args.quiet else print)
peaks = [_normalise({"id": line["id"], "sensor_x_px": line["sensor_x_px"],
"fwhm_px": line["fwhm_px"], "snr": line["snr"],
"saturated": line["saturated"],
"channels": line.get("channel_names", "")})
for line in analysis.lines]
spectrum = analysis.spectrum
context = {
"source": "peak_finder detection",
"recording": str(Path(args.input).expanduser().resolve()),
"frames": list(analysis.indices),
"roi": [spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1],
"bayer_pattern": spectrum.pattern,
"raw_format": spectrum.raw_format,
"bit_depth": spectrum.bit_depth,
"black_adu": spectrum.black,
"full_scale_adu": spectrum.full_scale,
"reduce": args.reduce,
"full_size": list(spectrum.full_size),
}
return peaks, context
# ----------------------------------------------------------------- fitting
def parse_assign(text: str, peaks: list[dict], order: int) -> list[dict]:
lookup = {peak["id"].upper(): peak for peak in peaks}
confirmed, used = [], set()
for token in text.split(","):
token = token.strip()
if not token:
continue
if ":" not in token:
raise ValueError(f"expected ID:wavelength, got {token!r}")
name, wavelength = token.split(":", 1)
name = name.strip().upper()
if name not in lookup:
raise ValueError(f"unknown peak {name}; exported ids are "
f"{', '.join(p['id'] for p in peaks[:10])}"
f"{'...' if len(peaks) > 10 else ''}")
if name in used:
raise ValueError(f"peak {name} assigned twice")
nm = _float(wavelength)
if not np.isfinite(nm) or nm <= 0:
raise ValueError(f"wavelength for {name} must be finite and positive")
used.add(name)
confirmed.append({**lookup[name], "wavelength_nm": nm})
if len(confirmed) < order + 1:
raise ValueError(f"order {order} needs at least {order + 1} confirmed peaks")
x = np.array([peak["x"] for peak in confirmed])
if len(np.unique(x)) != len(x):
raise ValueError("two confirmed peaks share the same sensor column")
sort = np.argsort(x)
y = np.array([peak["wavelength_nm"] for peak in confirmed])[sort]
if not (np.all(np.diff(y) > 0) or np.all(np.diff(y) < 0)):
raise ValueError("confirmed wavelengths must be monotonic in sensor column")
return confirmed
def fit(confirmed: list[dict], order: int, width: int | None) -> tuple[np.ndarray, np.ndarray]:
x = np.array([peak["x"] for peak in confirmed])
y = np.array([peak["wavelength_nm"] for peak in confirmed])
coeff = np.polynomial.Polynomial.fit(x, y, order).convert().coef[::-1]
high = float(width - 1) if width else float(x.max())
derivative = np.polyval(np.polyder(coeff), [float(x.min()), high])
if np.any(np.abs(derivative) < 1e-12) or derivative[0] * derivative[1] <= 0:
raise ValueError("mapping is not monotonic across the sensor; revise identities/order")
residual = np.polyval(coeff, x) - y
return coeff, residual
def default_output_directory(peaks: str | None = None, recording: str | None = None,
frames=None, context=None) -> Path:
"""Same naming rule as peak_finder, under calibration_exports/."""
if peaks:
folder = Path(peaks).expanduser().resolve().parent
if "_f" in folder.name: # peak_exports/<recording>_f0007
return Path("calibration_exports") / folder.name
reference = (context or {}).get("recording")
if reference: # peaks.json moved somewhere else
return core.default_output_directory(
"calibration_exports", Path(reference).name,
(context or {}).get("frames") or frames or [0])
return Path("calibration_exports") / Path(peaks).stem
stem = Path(recording).expanduser().resolve().name
return core.default_output_directory("calibration_exports", stem, frames or [0])
def mean_abs_gap(peaks: list[dict]) -> float:
xs = np.sort([peak["x"] for peak in peaks])
return float(np.mean(np.diff(xs))) if len(xs) > 1 else float("nan")
# ----------------------------------------------------------------- outputs
def write_outputs(args, peaks, confirmed, coeff, residual, context) -> dict:
out = Path(args.output)
out.mkdir(parents=True, exist_ok=True)
x = np.array([peak["x"] for peak in confirmed])
lo, hi = float(x.min()), float(x.max())
width = context.get("full_size")
width = int(width[0]) if isinstance(width, (list, tuple)) and width else None
info = {
"recording": context.get("recording"),
"peaks_file": context.get("source"),
"frames": context.get("frames"),
"roi": context.get("roi"),
"bayer_pattern": context.get("bayer_pattern"),
"raw_format": context.get("raw_format"),
"black_adu": context.get("black_adu"),
"full_scale_adu": context.get("full_scale_adu"),
"reduce": context.get("reduce"),
"order": args.order,
"coefficients_descending": [float(value) for value in coeff],
"formula": "wavelength_nm = numpy.polyval(coefficients_descending, sensor_x_px)",
"wavelength_medium": "air",
"reference": SOURCE,
"confirmed": [{key: peak[key] for key in
("id", "x", "wavelength_nm", "fwhm_px", "snr", "saturated", "channels")}
for peak in confirmed],
"residual_definition": "fitted - reference (nm)",
"residuals_nm": [float(value) for value in residual],
"rms_nm": float(np.sqrt(np.mean(residual ** 2))),
"max_abs_nm": float(np.max(np.abs(residual))),
"residual_degrees_of_freedom": len(x) - args.order - 1,
"calibrated_pixel_range": [lo, hi],
"processing": "peaks from peak_finder.py (baseline, topographic prominence, "
"Gaussian centres, joint fits for blended pairs); polynomial fit here",
}
core.write_json(out / "calibration.json", info)
core.write_csv(out / "residuals.csv",
["id", "sensor_x_px", "reference_nm", "fitted_nm", "residual_nm"],
([peak["id"], peak["x"], peak["wavelength_nm"],
peak["wavelength_nm"] + err, err]
for peak, err in zip(confirmed, residual)))
if width:
grid = np.arange(width)
core.write_csv(out / "pixel_wavelength.csv",
["sensor_x_px", "wavelength_nm", "extrapolated"],
([int(px), float(nm), int(px < lo or px > hi)]
for px, nm in zip(grid, np.polyval(coeff, grid))))
_plot(out, confirmed, coeff, residual)
return info
def _plot(out: Path, confirmed: list[dict], coeff: np.ndarray, residual: np.ndarray) -> None:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
x = np.array([peak["x"] for peak in confirmed])
y = np.array([peak["wavelength_nm"] for peak in confirmed])
padding = 0.05 * (x.max() - x.min()) if x.max() > x.min() else 1.0
grid = np.linspace(x.min() - padding, x.max() + padding, 400)
fig, axes = plt.subplots(2, 1, figsize=(9, 6.5), constrained_layout=True)
axes[0].plot(grid, np.polyval(coeff, grid), lw=1.2, color="#2c3e50")
axes[0].plot(x, y, "o", ms=6, color="#c0392b")
for peak in confirmed:
axes[0].annotate(peak["id"], (peak["x"], peak["wavelength_nm"]),
xytext=(6, -10), textcoords="offset points", fontsize=8)
axes[0].set_ylabel("Wavelength / nm (air)")
axes[0].set_title(f"order {len(coeff) - 1}: {len(confirmed)} peaks, "
f"RMS {np.sqrt(np.mean(residual ** 2)):.3f} nm", fontsize=10)
axes[1].axhline(0, color="#7f8c8d", lw=0.8)
axes[1].plot(x, residual, "o", ms=6, color="#2980b9")
for peak, err in zip(confirmed, residual):
axes[1].annotate(f"{peak['id']} {err:+.3f}", (peak["x"], err),
xytext=(6, 6), textcoords="offset points", fontsize=8)
axes[1].set_ylabel("Fit - reference / nm")
axes[1].set_xlabel("Sensor column / px")
axes[1].grid(alpha=0.25)
fig.savefig(out / "calibration.png", dpi=160)
plt.close(fig)
# ----------------------------------------------------------------- main
def run(args) -> int:
if bool(args.peaks) == bool(args.input):
raise ValueError("give exactly one of --peaks PATH or --input REC_DIR")
if args.peaks:
peaks, context = read_peaks(Path(args.peaks).expanduser())
else:
peaks, context = analyse_recording(args)
if not peaks:
raise ValueError("no peaks to calibrate from")
if not args.quiet:
print(f"{'id':6s}{'x_px':>11s}{'FWHM px':>9s}{'SNR':>9s}{'ch':>4s} flags")
for peak in peaks:
flags = " saturated" if peak["saturated"] else ""
print(f"{peak['id']:6s}{peak['x']:11.3f}{peak['fwhm_px']:9.2f}"
f"{peak['snr']:9.0f}{peak['channels']:>4s}{flags}")
print(f"\n{len(peaks)} peak(s); median spacing {mean_abs_gap(peaks):.2f} px")
print("Assign identities with --assign ID:NM,... (e.g. P003:546.0735)")
print(f"Hg I air wavelengths (nm): {HG}")
if not args.assign:
print(f"No --assign given; nothing fitted. Peaks read from {context['source']}.")
return 0
if args.output is None:
args.output = str(default_output_directory(args.peaks, args.input,
context.get("frames"), context))
confirmed = parse_assign(args.assign, peaks, args.order)
width = context.get("full_size")
width = int(width[0]) if isinstance(width, (list, tuple)) and width else None
coeff, residual = fit(confirmed, args.order, width)
info = write_outputs(args, peaks, confirmed, coeff, residual, context)
print("Coefficients (descending):", [f"{value:.8g}" for value in coeff])
print(f"RMS={info['rms_nm']:.6g} nm; max |residual|={info['max_abs_nm']:.6g} nm; "
f"{len(confirmed)} peak(s), order {args.order}")
saturated = [peak["id"] for peak in confirmed if peak["saturated"]]
if saturated:
print(f"warning: confirmed peak(s) saturated, centres are biased: {', '.join(saturated)}")
if len(confirmed) == args.order + 1:
print("Exact interpolation: zero residual does not establish calibration accuracy; "
"add independent lines.")
print(f"Calibration exported to {Path(args.output).expanduser().resolve()}")
return 0
def main(argv=None) -> int:
args = build_parser().parse_args(argv)
try:
return run(args)
except (ValueError, RuntimeError, OSError, KeyError, IndexError) as exc:
print(f"Calibration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())