- 命名规则移入 spectro_core.default_output_directory(root, stem, indices) - wavelength_calibration 默认改为 calibration_exports/<recording>_f<frame>,与 peak_exports 一致;输入为 peaks.json 时镜像其所在目录名 - 新增 3 项命名测试
367 lines
17 KiB
Python
367 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Fit the sensor-column -> wavelength calibration from peaks found by peak_finder.
|
|
|
|
The peaks are the input, not the raw frames: either an exported peaks.json /
|
|
selected_peaks.csv, or a recording analysed by peak_finder in-process. Peak
|
|
identities are supplied as ID:WAVELENGTH pairs, e.g.
|
|
|
|
wavelength_calibration.py --peaks peak_exports/xxx/peaks.json \
|
|
--assign "P003:546.0735,P004:576.9598,P005:579.0663"
|
|
|
|
This program does not detect peaks; peak_finder.py does, and this one only fits
|
|
the polynomial and records the provenance.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
try: # shared core lives beside this script
|
|
import spectro_core as core
|
|
except ImportError as exc: # pragma: no cover - placement error
|
|
raise SystemExit('wavelength_calibration.py needs spectro_core.py in the same '
|
|
'directory (or on sys.path)') from exc
|
|
|
|
HG = (404.6563, 435.8328, 546.0735, 576.9598, 579.0663)
|
|
SOURCE = "https://physics.nist.gov/PhysRefData/Handbook/Tables/mercurytable2.htm"
|
|
|
|
|
|
# --------------------------------------------------------------------------- CLI
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(
|
|
description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
)
|
|
source = p.add_argument_group("peaks")
|
|
source.add_argument("--peaks", metavar="PATH",
|
|
help="peaks.json or selected_peaks.csv written by peak_finder.py")
|
|
source.add_argument("--input", metavar="REC_DIR",
|
|
help="hq_record recording directory; run peak_finder's detection on it")
|
|
source.add_argument("--frame", type=int, default=0, help="zero-based frame index (default 0)")
|
|
source.add_argument("--average", type=int, default=1,
|
|
help="mean this many consecutive frames starting at --frame (default 1)")
|
|
source.add_argument("--roi", default="auto", help="'auto' or x0,y0,x1,y1")
|
|
source.add_argument("--rows", default=None, metavar="Y0,Y1",
|
|
help="restrict the ROI to these sensor rows")
|
|
source.add_argument("--reduce", choices=("mean", "median"), default="mean")
|
|
source.add_argument("--black", type=float, default=None, help="black level in ADU")
|
|
source.add_argument("--full-scale", type=float, default=None)
|
|
source.add_argument("--module-dir", default=None,
|
|
help="directory containing hq_spectrum.py (default this folder)")
|
|
|
|
fit_group = p.add_argument_group("fitting")
|
|
fit_group.add_argument("--assign", metavar="ID:NM,...",
|
|
help="confirmed peak identities, e.g. P003:546.0735,P004:576.9598")
|
|
fit_group.add_argument("--order", type=int, choices=(1, 2), default=1)
|
|
fit_group.add_argument("--output", default=None, metavar="DIR",
|
|
help="output directory (default calibration_exports/<recording>_f<frame>)")
|
|
fit_group.add_argument("--quiet", action="store_true", help="suppress the peak table")
|
|
return p
|
|
|
|
|
|
# ----------------------------------------------------------------- inputs
|
|
def _float(value) -> float:
|
|
try:
|
|
number = float(value)
|
|
except (TypeError, ValueError):
|
|
return float("nan")
|
|
return number if np.isfinite(number) else float("nan")
|
|
|
|
|
|
def _truthy(value) -> bool:
|
|
return str(value).strip().lower() in ("1", "true", "yes", "y")
|
|
|
|
|
|
def _normalise(record: dict) -> dict:
|
|
return {
|
|
"id": str(record["id"]).strip(),
|
|
"x": _float(record.get("sensor_x_px")),
|
|
"fwhm_px": _float(record.get("fwhm_px")),
|
|
"snr": _float(record.get("snr")),
|
|
"saturated": _truthy(record.get("saturated", False)),
|
|
"channels": str(record.get("channels", "") or ""),
|
|
}
|
|
|
|
|
|
def read_peaks(path: Path) -> tuple[list[dict], dict]:
|
|
"""Load peaks from peak_finder's peaks.json or selected_peaks.csv."""
|
|
|
|
context: dict = {"source": str(path.resolve())}
|
|
text = path.read_text(encoding="utf-8-sig")
|
|
if path.suffix.lower() == ".json":
|
|
data = json.loads(text)
|
|
records = data.get("peaks") or []
|
|
if not records:
|
|
raise ValueError(f"{path.name} contains no peaks")
|
|
for key in ("recording", "frames", "roi", "bayer_pattern", "raw_format",
|
|
"bit_depth", "black_adu", "full_scale_adu", "reduce", "full_size"):
|
|
if data.get(key) is not None:
|
|
context[key] = data[key]
|
|
else:
|
|
rows = list(csv.DictReader(text.splitlines()))
|
|
if not rows:
|
|
raise ValueError(f"{path.name} contains no rows")
|
|
missing = {"id", "sensor_x_px"} - set(rows[0])
|
|
if missing:
|
|
raise ValueError(f"{path.name} is missing column(s): {', '.join(sorted(missing))}")
|
|
records = rows
|
|
corners = ("roi_x0", "roi_y0", "roi_x1", "roi_y1")
|
|
if all(key in rows[0] and str(rows[0][key]).strip() for key in corners):
|
|
context["roi"] = [int(float(rows[0][key])) for key in corners]
|
|
peaks = [_normalise(record) for record in records]
|
|
bad = [peak["id"] for peak in peaks if not np.isfinite(peak["x"])]
|
|
if bad:
|
|
raise ValueError(f"peaks without a finite sensor_x_px: {', '.join(bad)}")
|
|
return peaks, context
|
|
|
|
|
|
def analyse_recording(args) -> tuple[list[dict], dict]:
|
|
"""Run peak_finder's detection on a recording and return its lines."""
|
|
|
|
import peak_finder as pf
|
|
|
|
defaults = pf.build_parser().parse_args(["--input", str(args.input)])
|
|
for key, value in vars(defaults).items():
|
|
if not hasattr(args, key) or getattr(args, key) is None:
|
|
setattr(args, key, value)
|
|
args.calibration = None # calibrating, not applying one
|
|
|
|
analysis = pf.analyse(args, tell=(lambda *a, **k: None) if args.quiet else print)
|
|
peaks = [_normalise({"id": line["id"], "sensor_x_px": line["sensor_x_px"],
|
|
"fwhm_px": line["fwhm_px"], "snr": line["snr"],
|
|
"saturated": line["saturated"],
|
|
"channels": line.get("channel_names", "")})
|
|
for line in analysis.lines]
|
|
spectrum = analysis.spectrum
|
|
context = {
|
|
"source": "peak_finder detection",
|
|
"recording": str(Path(args.input).expanduser().resolve()),
|
|
"frames": list(analysis.indices),
|
|
"roi": [spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1],
|
|
"bayer_pattern": spectrum.pattern,
|
|
"raw_format": spectrum.raw_format,
|
|
"bit_depth": spectrum.bit_depth,
|
|
"black_adu": spectrum.black,
|
|
"full_scale_adu": spectrum.full_scale,
|
|
"reduce": args.reduce,
|
|
"full_size": list(spectrum.full_size),
|
|
}
|
|
return peaks, context
|
|
|
|
|
|
# ----------------------------------------------------------------- fitting
|
|
def parse_assign(text: str, peaks: list[dict], order: int) -> list[dict]:
|
|
lookup = {peak["id"].upper(): peak for peak in peaks}
|
|
confirmed, used = [], set()
|
|
for token in text.split(","):
|
|
token = token.strip()
|
|
if not token:
|
|
continue
|
|
if ":" not in token:
|
|
raise ValueError(f"expected ID:wavelength, got {token!r}")
|
|
name, wavelength = token.split(":", 1)
|
|
name = name.strip().upper()
|
|
if name not in lookup:
|
|
raise ValueError(f"unknown peak {name}; exported ids are "
|
|
f"{', '.join(p['id'] for p in peaks[:10])}"
|
|
f"{'...' if len(peaks) > 10 else ''}")
|
|
if name in used:
|
|
raise ValueError(f"peak {name} assigned twice")
|
|
nm = _float(wavelength)
|
|
if not np.isfinite(nm) or nm <= 0:
|
|
raise ValueError(f"wavelength for {name} must be finite and positive")
|
|
used.add(name)
|
|
confirmed.append({**lookup[name], "wavelength_nm": nm})
|
|
if len(confirmed) < order + 1:
|
|
raise ValueError(f"order {order} needs at least {order + 1} confirmed peaks")
|
|
x = np.array([peak["x"] for peak in confirmed])
|
|
if len(np.unique(x)) != len(x):
|
|
raise ValueError("two confirmed peaks share the same sensor column")
|
|
sort = np.argsort(x)
|
|
y = np.array([peak["wavelength_nm"] for peak in confirmed])[sort]
|
|
if not (np.all(np.diff(y) > 0) or np.all(np.diff(y) < 0)):
|
|
raise ValueError("confirmed wavelengths must be monotonic in sensor column")
|
|
return confirmed
|
|
|
|
|
|
def fit(confirmed: list[dict], order: int, width: int | None) -> tuple[np.ndarray, np.ndarray]:
|
|
x = np.array([peak["x"] for peak in confirmed])
|
|
y = np.array([peak["wavelength_nm"] for peak in confirmed])
|
|
coeff = np.polynomial.Polynomial.fit(x, y, order).convert().coef[::-1]
|
|
high = float(width - 1) if width else float(x.max())
|
|
derivative = np.polyval(np.polyder(coeff), [float(x.min()), high])
|
|
if np.any(np.abs(derivative) < 1e-12) or derivative[0] * derivative[1] <= 0:
|
|
raise ValueError("mapping is not monotonic across the sensor; revise identities/order")
|
|
residual = np.polyval(coeff, x) - y
|
|
return coeff, residual
|
|
|
|
|
|
def default_output_directory(peaks: str | None = None, recording: str | None = None,
|
|
frames=None, context=None) -> Path:
|
|
"""Same naming rule as peak_finder, under calibration_exports/."""
|
|
|
|
if peaks:
|
|
folder = Path(peaks).expanduser().resolve().parent
|
|
if "_f" in folder.name: # peak_exports/<recording>_f0007
|
|
return Path("calibration_exports") / folder.name
|
|
reference = (context or {}).get("recording")
|
|
if reference: # peaks.json moved somewhere else
|
|
return core.default_output_directory(
|
|
"calibration_exports", Path(reference).name,
|
|
(context or {}).get("frames") or frames or [0])
|
|
return Path("calibration_exports") / Path(peaks).stem
|
|
stem = Path(recording).expanduser().resolve().name
|
|
return core.default_output_directory("calibration_exports", stem, frames or [0])
|
|
|
|
|
|
def mean_abs_gap(peaks: list[dict]) -> float:
|
|
xs = np.sort([peak["x"] for peak in peaks])
|
|
return float(np.mean(np.diff(xs))) if len(xs) > 1 else float("nan")
|
|
|
|
|
|
# ----------------------------------------------------------------- outputs
|
|
def write_outputs(args, peaks, confirmed, coeff, residual, context) -> dict:
|
|
out = Path(args.output)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
x = np.array([peak["x"] for peak in confirmed])
|
|
lo, hi = float(x.min()), float(x.max())
|
|
width = context.get("full_size")
|
|
width = int(width[0]) if isinstance(width, (list, tuple)) and width else None
|
|
info = {
|
|
"recording": context.get("recording"),
|
|
"peaks_file": context.get("source"),
|
|
"frames": context.get("frames"),
|
|
"roi": context.get("roi"),
|
|
"bayer_pattern": context.get("bayer_pattern"),
|
|
"raw_format": context.get("raw_format"),
|
|
"black_adu": context.get("black_adu"),
|
|
"full_scale_adu": context.get("full_scale_adu"),
|
|
"reduce": context.get("reduce"),
|
|
"order": args.order,
|
|
"coefficients_descending": [float(value) for value in coeff],
|
|
"formula": "wavelength_nm = numpy.polyval(coefficients_descending, sensor_x_px)",
|
|
"wavelength_medium": "air",
|
|
"reference": SOURCE,
|
|
"confirmed": [{key: peak[key] for key in
|
|
("id", "x", "wavelength_nm", "fwhm_px", "snr", "saturated", "channels")}
|
|
for peak in confirmed],
|
|
"residual_definition": "fitted - reference (nm)",
|
|
"residuals_nm": [float(value) for value in residual],
|
|
"rms_nm": float(np.sqrt(np.mean(residual ** 2))),
|
|
"max_abs_nm": float(np.max(np.abs(residual))),
|
|
"residual_degrees_of_freedom": len(x) - args.order - 1,
|
|
"calibrated_pixel_range": [lo, hi],
|
|
"processing": "peaks from peak_finder.py (baseline, topographic prominence, "
|
|
"Gaussian centres, joint fits for blended pairs); polynomial fit here",
|
|
}
|
|
core.write_json(out / "calibration.json", info)
|
|
core.write_csv(out / "residuals.csv",
|
|
["id", "sensor_x_px", "reference_nm", "fitted_nm", "residual_nm"],
|
|
([peak["id"], peak["x"], peak["wavelength_nm"],
|
|
peak["wavelength_nm"] + err, err]
|
|
for peak, err in zip(confirmed, residual)))
|
|
if width:
|
|
grid = np.arange(width)
|
|
core.write_csv(out / "pixel_wavelength.csv",
|
|
["sensor_x_px", "wavelength_nm", "extrapolated"],
|
|
([int(px), float(nm), int(px < lo or px > hi)]
|
|
for px, nm in zip(grid, np.polyval(coeff, grid))))
|
|
_plot(out, confirmed, coeff, residual)
|
|
return info
|
|
|
|
|
|
def _plot(out: Path, confirmed: list[dict], coeff: np.ndarray, residual: np.ndarray) -> None:
|
|
import matplotlib
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
|
|
x = np.array([peak["x"] for peak in confirmed])
|
|
y = np.array([peak["wavelength_nm"] for peak in confirmed])
|
|
padding = 0.05 * (x.max() - x.min()) if x.max() > x.min() else 1.0
|
|
grid = np.linspace(x.min() - padding, x.max() + padding, 400)
|
|
fig, axes = plt.subplots(2, 1, figsize=(9, 6.5), constrained_layout=True)
|
|
axes[0].plot(grid, np.polyval(coeff, grid), lw=1.2, color="#2c3e50")
|
|
axes[0].plot(x, y, "o", ms=6, color="#c0392b")
|
|
for peak in confirmed:
|
|
axes[0].annotate(peak["id"], (peak["x"], peak["wavelength_nm"]),
|
|
xytext=(6, -10), textcoords="offset points", fontsize=8)
|
|
axes[0].set_ylabel("Wavelength / nm (air)")
|
|
axes[0].set_title(f"order {len(coeff) - 1}: {len(confirmed)} peaks, "
|
|
f"RMS {np.sqrt(np.mean(residual ** 2)):.3f} nm", fontsize=10)
|
|
axes[1].axhline(0, color="#7f8c8d", lw=0.8)
|
|
axes[1].plot(x, residual, "o", ms=6, color="#2980b9")
|
|
for peak, err in zip(confirmed, residual):
|
|
axes[1].annotate(f"{peak['id']} {err:+.3f}", (peak["x"], err),
|
|
xytext=(6, 6), textcoords="offset points", fontsize=8)
|
|
axes[1].set_ylabel("Fit - reference / nm")
|
|
axes[1].set_xlabel("Sensor column / px")
|
|
axes[1].grid(alpha=0.25)
|
|
fig.savefig(out / "calibration.png", dpi=160)
|
|
plt.close(fig)
|
|
|
|
|
|
# ----------------------------------------------------------------- main
|
|
def run(args) -> int:
|
|
if bool(args.peaks) == bool(args.input):
|
|
raise ValueError("give exactly one of --peaks PATH or --input REC_DIR")
|
|
|
|
if args.peaks:
|
|
peaks, context = read_peaks(Path(args.peaks).expanduser())
|
|
else:
|
|
peaks, context = analyse_recording(args)
|
|
if not peaks:
|
|
raise ValueError("no peaks to calibrate from")
|
|
|
|
if not args.quiet:
|
|
print(f"{'id':6s}{'x_px':>11s}{'FWHM px':>9s}{'SNR':>9s}{'ch':>4s} flags")
|
|
for peak in peaks:
|
|
flags = " saturated" if peak["saturated"] else ""
|
|
print(f"{peak['id']:6s}{peak['x']:11.3f}{peak['fwhm_px']:9.2f}"
|
|
f"{peak['snr']:9.0f}{peak['channels']:>4s}{flags}")
|
|
print(f"\n{len(peaks)} peak(s); median spacing {mean_abs_gap(peaks):.2f} px")
|
|
print("Assign identities with --assign ID:NM,... (e.g. P003:546.0735)")
|
|
print(f"Hg I air wavelengths (nm): {HG}")
|
|
|
|
if not args.assign:
|
|
print(f"No --assign given; nothing fitted. Peaks read from {context['source']}.")
|
|
return 0
|
|
|
|
if args.output is None:
|
|
args.output = str(default_output_directory(args.peaks, args.input,
|
|
context.get("frames"), context))
|
|
confirmed = parse_assign(args.assign, peaks, args.order)
|
|
width = context.get("full_size")
|
|
width = int(width[0]) if isinstance(width, (list, tuple)) and width else None
|
|
coeff, residual = fit(confirmed, args.order, width)
|
|
info = write_outputs(args, peaks, confirmed, coeff, residual, context)
|
|
|
|
print("Coefficients (descending):", [f"{value:.8g}" for value in coeff])
|
|
print(f"RMS={info['rms_nm']:.6g} nm; max |residual|={info['max_abs_nm']:.6g} nm; "
|
|
f"{len(confirmed)} peak(s), order {args.order}")
|
|
saturated = [peak["id"] for peak in confirmed if peak["saturated"]]
|
|
if saturated:
|
|
print(f"warning: confirmed peak(s) saturated, centres are biased: {', '.join(saturated)}")
|
|
if len(confirmed) == args.order + 1:
|
|
print("Exact interpolation: zero residual does not establish calibration accuracy; "
|
|
"add independent lines.")
|
|
print(f"Calibration exported to {Path(args.output).expanduser().resolve()}")
|
|
return 0
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
return run(args)
|
|
except (ValueError, RuntimeError, OSError, KeyError, IndexError) as exc:
|
|
print(f"Calibration error: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|