Files
spectrometer/peak_finder.py
T
starcells 6b4e86cebd pick 窗口支持手动合并/拆分(m / s / u)
- 选中多条按 m 合并:中心按峰高加权,保留最小 id,记录 merged_from

- s 拆回、u 撤销;合并结果直接作用于本次导出

- apply_merges 为纯函数,新增 3 项单元测试
2026-11-14 15:28:26 +08:00

1524 lines
66 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Peak finder and calibrated peak export for hq_record RAW recordings.
What it does
------------
1. Decodes one frame (or the mean of several frames) of a recording written by
``hq_record.py`` through the decoder of ``hq_spectrum.py`` (PiSP PC1B ->
uint16 Bayer image).
2. Splits the ROI into the B / Gb+Gr / R planes and reduces it along the slit
into three one-dimensional spectra, exactly like ``hq_spectrum.py`` does.
Sensor columns are full-resolution sensor x coordinates, not ROI counters.
3. Estimates a morphological baseline and a noise level for every channel, so
peaks are found relative to the local continuum instead of the global zero.
4. Detects peaks, suppresses neighbours, and refines every centre with a
Gaussian/Lorentzian fit, falling back to the local vertex of the measured
maximum; FWHM, net height, area and SNR come from the refined profile.
5. Merges peaks that the same spectral line produces in the R, G and B CFA
channels into one physical line.
6. Converts sensor columns to wavelength with a saved calibration JSON
(``wavelength_calibration/calibration.json``), flags extrapolation, and
optionally matches a line table.
7. Exports what was asked for: for every selected peak a peak-shape image
(PNG), the peak-shape numbers (CSV), and the pixel coordinates of the peak
in the raw frame (CSV/JSON, plus an optional raw patch dump).
Pixel coordinates
-----------------
All coordinates are 0-based full-resolution sensor coordinates; pixel ``x``
covers ``[x, x+1)`` and row 0 is the top of the raw frame. The reduced
profile samples the sensor every 2 px, so a fitted peak centre is fractional
and not restricted to the sample grid.
Typical use
-----------
# every detected peak, calibrated, with images and tables
python peak_finder.py --input recordings/hq_20261110_230441 --select all
# only peaks between 540 and 560 nm, plus the two strongest lines
python peak_finder.py --input recordings/hq_20261024_135709 \
--select nm:540-560,top:2
# look at the candidates first, then export a manual selection
python peak_finder.py --input recordings/hq_x --select none
python peak_finder.py --input recordings/hq_x --select id:3,id:7-9
# click peaks in a window instead of typing ids
python peak_finder.py --input recordings/hq_x --pick
# average 8 frames to lower the noise before peak finding
python peak_finder.py --input recordings/hq_x --frame 10 --average 8
No smoothing, denoising or background subtraction is applied to the exported
raw signal. The baseline exists only to measure peak height and width, and it
is exported next to the raw values so the choice stays auditable.
Requires ``hq_spectrum.py`` beside this file (or ``--module-dir``), NumPy and
Matplotlib. SciPy is not used.
"""
from __future__ import annotations
import argparse
import csv
import importlib.util
import json
import math
import sys
from pathlib import Path
from typing import Optional
import numpy as np
try: # shared core lives beside this script
from spectro_core import ( # noqa: F401 (re-exported for compatibility)
GAUSS_TO_FWHM, Calibration, _fit_pair, _jsonable, _num, attach_wavelength,
deblend_close_peaks, dedupe_channel_peaks, default_output_directory as _shared_output, estimate_baseline, estimate_noise,
find_peak_candidates, fit_profile, group_channels, half_max_crossings, is_saturated, load_hq,
merge_coincident_lines, model_values, parabola_center, peak_prominence,
refine_peak, summarise_line, write_csv, write_json)
except ImportError as exc: # pragma: no cover - placement error
raise SystemExit('peak_finder.py needs spectro_core.py in the same directory '
'(or on sys.path)') from exc
PROGRAM_DIR = Path(__file__).resolve().parent
SHAPES = ("gauss", "lorentz")
GAUSS_TO_FWHM = 2.3548200450309493
# ---------------------------------------------------------------------------
# Imports and small helpers
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Calibration
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Baseline, noise and local peak metrics
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Line-shape fitting (Levenberg-Marquardt, no SciPy)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Peak detection and refinement
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Grouping peaks from the three CFA channels
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Selection
# ---------------------------------------------------------------------------
def _match_ids(token: str, lines: list[dict]) -> set[str]:
available = {line["id"] for line in lines}
numbers = [int(line["id"][1:]) for line in lines]
highest = max(numbers) if numbers else 0
if "-" in token:
first, second = token.split("-", 1)
low, high = int(first), int(second)
if low > high:
low, high = high, low
chosen = {f"P{value:03d}" for value in range(low, high + 1)}
else:
chosen = {f"P{int(token):03d}"}
if not chosen:
raise ValueError(f"empty peak id selection {token!r}")
missing = sorted(chosen - available)
if missing:
raise ValueError(f"unknown peak id(s): {', '.join(missing)}")
return chosen
def _split_range(token: str):
"""Split 'LOW-HIGH' (optionally with a leading minus) into two strings."""
body = token[1:] if token.startswith("-") else token
if "-" not in body:
return None
first, second = body.split("-", 1)
return first, second
def _match_wavelengths(token: str, lines: list[dict], half_width_nm: float) -> set[str]:
values = [line["wavelength_nm"] for line in lines if np.isfinite(line["wavelength_nm"])]
if not values:
raise ValueError("wavelength selection needs --calibration")
parts = _split_range(token)
try:
if parts is None:
low, high = float(token) - half_width_nm, float(token) + half_width_nm
else:
low, high = float(parts[0]), float(parts[1])
except ValueError as exc:
raise ValueError(f"invalid wavelength selection {token!r}; use nm:LOW-HIGH") from exc
if low > high:
low, high = high, low
return {line["id"] for line in lines
if np.isfinite(line["wavelength_nm"]) and low <= line["wavelength_nm"] <= high}
def _fallback_half_width(lines: list[dict]) -> float:
"""Half-width (nm) for a bare wavelength token, from the typical line spacing."""
wavelengths = [line["wavelength_nm"] for line in lines if np.isfinite(line["wavelength_nm"])]
if len(wavelengths) < 2:
return 1.0
spacing = float(np.median(np.diff(np.sort(np.asarray(wavelengths, dtype=float)))))
return max(0.5, min(2.0, spacing)) if np.isfinite(spacing) and spacing > 0 else 1.0
def parse_selection(text: str, lines: list[dict]) -> set[str]:
"""Parse --select: all, none, ids, id ranges, nm ranges and top:N."""
tokens = [token.strip() for token in str(text).replace(";", ",").split(",") if token.strip()]
if not tokens:
raise ValueError("--select needs at least one token")
numbers = [int(line["id"][1:]) for line in lines]
highest = max(numbers) if numbers else 0
half_width = _fallback_half_width(lines)
selected: set[str] = set()
for token in tokens:
low = token.lower()
if low in ("all", "*"):
selected.update(line["id"] for line in lines)
elif low in ("none", "no", "null"):
continue
elif low.startswith("top:"):
count = int(low[4:])
if count < 1:
raise ValueError("top:N needs N >= 1")
ranked = sorted(lines, key=lambda item: (-item["snr"], item["sensor_x_px"]))
selected.update(item["id"] for item in ranked[:count])
elif low.startswith("id:"):
selected.update(_match_ids(low[3:], lines))
elif low.startswith("nm:"):
selected.update(_match_wavelengths(low[3:], lines, half_width))
elif _split_range(low) is not None:
first, second = _split_range(low)
try:
left, right = float(first), float(second)
except ValueError as exc:
raise ValueError(f"cannot parse selection token {token!r}") from exc
integers = "." not in first and "." not in second
if integers and right <= highest:
selected.update(_match_ids(low, lines))
elif left >= 200 and right >= 200:
selected.update(_match_wavelengths(low, lines, half_width))
else:
raise ValueError(f"selection {token!r} is neither a peak id range nor a "
f"wavelength range; use id:N-M or nm:LOW-HIGH")
elif low.replace(".", "", 1).isdigit():
if "." in low:
selected.update(_match_wavelengths(low, lines, half_width))
else:
selected.update(_match_ids(low, lines))
else:
raise ValueError(f"cannot parse selection token {token!r}")
return selected
# ---------------------------------------------------------------------------
# Line table matching
# ---------------------------------------------------------------------------
def load_line_table(path) -> list[dict]:
"""Read a CSV/JSON line table with wavelength_nm, species and label columns."""
location = Path(path).expanduser()
if not location.is_file():
raise ValueError(f"line table not found: {location}")
if location.suffix.lower() == ".json":
data = json.loads(location.read_text(encoding="utf-8-sig"))
rows = data.get("lines") if isinstance(data, dict) else data
if not isinstance(rows, list):
raise ValueError("line table JSON needs a list of lines")
table = []
for row in rows:
if not isinstance(row, dict):
raise ValueError("line table rows must be objects")
value = row.get("wavelength_nm", row.get("nm"))
if value is None:
continue
table.append({"wavelength_nm": float(value),
"species": str(row.get("species", row.get("element", ""))),
"label": str(row.get("label", ""))})
else:
with location.open("r", newline="", encoding="utf-8-sig") as handle:
reader = csv.DictReader(handle)
if not reader.fieldnames:
raise ValueError("line table needs a header row")
fields = {name.strip().lower(): name for name in reader.fieldnames}
wavelength = fields.get("wavelength_nm") or fields.get("nm") or fields.get("wavelength")
if wavelength is None:
raise ValueError("line table needs a wavelength_nm column")
species = fields.get("species") or fields.get("element") or fields.get("atom")
label = fields.get("label") or fields.get("line")
table = []
for row in reader:
try:
value = float(row[wavelength])
except (TypeError, ValueError):
continue
table.append({"wavelength_nm": value,
"species": (row.get(species) or "").strip() if species else "",
"label": (row.get(label) or "").strip() if label else ""})
if not table:
raise ValueError("line table contains no usable wavelengths")
return sorted(table, key=lambda item: item["wavelength_nm"])
def match_lines(lines: list[dict], table: list[dict], tolerance_nm: float) -> None:
known = np.asarray([entry["wavelength_nm"] for entry in table], dtype=float)
for line in lines:
line.update({"match_species": "", "match_nm": float("nan"),
"match_delta_nm": float("nan"), "match_label": ""})
if not np.isfinite(line["wavelength_nm"]) or known.size == 0:
continue
index = int(np.argmin(np.abs(known - line["wavelength_nm"])))
delta = float(line["wavelength_nm"] - known[index])
if abs(delta) <= tolerance_nm:
entry = table[index]
line.update({"match_species": entry["species"], "match_nm": entry["wavelength_nm"],
"match_delta_nm": delta, "match_label": entry["label"]})
# ---------------------------------------------------------------------------
# Reading the recording
# ---------------------------------------------------------------------------
def resolve_rows(value, roi, height: int, hq):
"""Intersect the ROI with an optional absolute row window."""
if not value:
return roi
parts = [part.strip() for part in str(value).split(",")]
if len(parts) != 2:
raise ValueError("--rows must be y0,y1 in sensor rows")
try:
y0, y1 = (int(round(float(part))) for part in parts)
except ValueError as exc:
raise ValueError("--rows values must be numbers") from exc
y0 = max(0, min(y0, height - 4))
y1 = max(4, min(y1, height))
if y1 - y0 < 4:
raise ValueError("--rows window is too small")
y0 -= y0 % 2
y1 -= y1 % 2
low, high = max(roi.y0, y0), min(roi.y1, y1)
if high - low < 4:
raise ValueError("--rows window does not overlap the ROI")
return hq.ROI(roi.x0, int(low), roi.x1, int(high))
def read_frames(hq, source, args, tell):
"""Read one frame or the mean of several; returns float image, raw copy, indices."""
total = int(source.total_frames)
if args.frame < 0 or args.frame >= total:
raise ValueError(f"--frame {args.frame} outside 0..{total - 1}")
count = max(1, int(args.average))
if args.frame + count > total:
raise ValueError(f"--average {count} from frame {args.frame} exceeds {total} frames")
indices = list(range(args.frame, args.frame + count))
show_progress = count > 1 and sys.stdout.isatty()
accumulation = None
original = None
frame_meta = None
for position, index in enumerate(indices):
image, meta = source.read_at(index)
if frame_meta is None:
frame_meta = meta
accumulation = (image.astype(np.float64) if accumulation is None
else accumulation + image)
if count == 1:
original = image
if show_progress:
tell(f"\r reading frame {position + 1}/{count}", end="")
if count > 1:
if show_progress:
tell("\r" + " " * 40, end="\r")
accumulation = accumulation / float(count)
return accumulation.astype(np.float32), original, indices, frame_meta or {}
# ---------------------------------------------------------------------------
# Spectrum construction
# ---------------------------------------------------------------------------
class SpectrumData:
"""Reduced spectra plus the raw image needed by the exports."""
def __init__(self, hq, source, image, original, indices, roi, args, black, full_scale):
self.hq = hq
self.source = source
self.image = image
self.original = original
self.indices = indices
self.roi = roi
self.args = args
self.black = black
self.full_scale = full_scale
self.pattern = source.pattern
self.full_size = tuple(source.full_size)
self.raw_format = source.raw_format
self.bit_depth = source.bit_depth
self.step_px = 2.0
self.x_px: dict[str, np.ndarray] = {}
self.raw: dict[str, np.ndarray] = {}
self.baseline: dict[str, np.ndarray] = {}
self.net: dict[str, np.ndarray] = {}
self.noise: dict[str, float] = {}
self._build()
def _build(self) -> None:
hq = self.hq
planes = hq.extract_planes(self.image, self.pattern, self.roi)
counts_g = planes["G"].shape[1]
counts_rb = min(planes["R"].shape[1], planes["B"].shape[1])
positions = hq.channel_x_positions(self.roi, self.pattern, counts_g, counts_rb)
window = 0 if self.args.baseline_window <= 0 else max(
3, int(round(self.args.baseline_window / self.step_px)))
for channel in hq.CHANNELS:
if self.args.reduce == "median":
profile = np.median(planes[channel], axis=0).astype(np.float64)
else:
profile = np.mean(planes[channel], axis=0, dtype=np.float64)
values = profile - self.black
x = positions[channel][:values.size]
baseline = estimate_baseline(values, window, hq.smooth_vector)
net = values - baseline
self.x_px[channel] = x
self.raw[channel] = values
self.baseline[channel] = baseline
self.net[channel] = net
self.noise[channel] = estimate_noise(net)
def detect(self) -> dict[str, list[dict]]:
args = self.args
result = {}
for channel in self.hq.CHANNELS:
net = self.net[channel]
noise = self.noise[channel]
top = float(np.max(net)) if net.size else 0.0
if top <= 0:
result[channel] = []
continue
min_height = max(args.peak_height * top, args.min_snr * noise)
min_prominence = max(args.peak_prominence * top, 0.4 * args.min_snr * noise)
min_distance = max(1, int(round(args.peak_distance / self.step_px)))
candidates = find_peak_candidates(net, min_distance, min_height,
min_prominence, args.max_peaks)
peaks = []
for candidate in candidates:
index = candidate["sample"]
x = self.x_px[channel][index]
saturated = is_saturated(self.image, self.roi, x, self.full_scale)
peaks.append(refine_peak(
self.x_px[channel], net, self.raw[channel], candidate, noise, self.step_px,
max(1, int(round(args.fwhm_span / self.step_px))),
max(0, int(round(args.fit_window / self.step_px))),
args.shape, saturated,
baseline_value=float(self.baseline[channel][index])))
result[channel] = deblend_close_peaks(
self.x_px[channel], net, dedupe_channel_peaks(peaks, args.peak_distance),
args.shape, self.step_px, noise)
return result
# ---------------------------------------------------------------------------
# Plotting
# ---------------------------------------------------------------------------
def _axis_wavelength(ax, calibration: Optional[Calibration]) -> None:
if calibration is None or calibration.width is None:
return
try:
secondary = ax.secondary_xaxis("top", functions=(calibration, calibration.inverse))
secondary.set_xlabel("Wavelength / nm (air)")
secondary.tick_params(labelsize=8)
except (ValueError, TypeError):
pass
def plot_overview(path: Path, spectrum: SpectrumData, lines: list[dict],
calibration, selected: set[str], meta: dict) -> None:
import matplotlib.pyplot as plt
hq = spectrum.hq
unit = "Wavelength / nm (air)" if calibration is not None else "Sensor column / px"
fig, axes = plt.subplots(2, 1, figsize=(15, 8.5), constrained_layout=True, sharex=True)
for channel in hq.CHANNELS:
x = spectrum.x_px[channel]
x_plot = calibration(x) if calibration is not None else x
axes[0].plot(x_plot, spectrum.raw[channel], color=hq.CHANNEL_COLORS[channel],
lw=0.8, label=channel)
axes[0].plot(x_plot, spectrum.baseline[channel], color=hq.CHANNEL_COLORS[channel],
lw=0.6, ls=":", alpha=0.7)
peak = float(np.max(spectrum.net[channel])) if spectrum.net[channel].size else 0.0
norm = spectrum.net[channel] / peak if peak > 0 else np.zeros_like(x)
axes[1].plot(x_plot, norm, color=hq.CHANNEL_COLORS[channel], lw=0.8, label=channel)
for line in lines:
color = "#c0392b" if line["id"] in selected else "#7f8c8d"
axes[1].axvline(line["x_plot"], color=color, lw=0.8,
alpha=0.9 if line["id"] in selected else 0.35)
if line["id"] in selected:
axes[1].annotate(line["id"], (line["x_plot"], 1.0), xytext=(-2, 6),
textcoords="offset points", rotation=90, fontsize=7,
color=color, ha="right")
if calibration is not None:
low, high = calibration.nm_span
for axis in axes:
axis.axvspan(low, high, color="#2ecc71", alpha=0.08, zorder=0)
axes[0].set_ylabel("Black-corrected ADU")
raw_top = max(float(np.max(values)) for values in spectrum.raw.values() if values.size)
raw_bottom = min(float(np.min(values)) for values in spectrum.raw.values() if values.size)
linthresh = max(1.0, 0.002 * raw_top)
# linscale keeps the empty linear band around zero thin: without it roughly
# half of the panel is spent below the data.
axes[0].set_yscale("symlog", linthresh=linthresh, linscale=0.15)
axes[0].set_ylim(min(raw_bottom * 1.2, -linthresh), raw_top * 1.15)
axes[0].legend(loc="upper right", fontsize=8)
span = (f"frame {spectrum.indices[0]}" if len(spectrum.indices) == 1 else
f"frames {spectrum.indices[0]}-{spectrum.indices[-1]} (mean {len(spectrum.indices)})")
note = ("" if calibration is None else
f" | green band = calibrated {calibration.nm_span[0]:.0f}-"
f"{calibration.nm_span[1]:.0f} nm, outside = extrapolated")
axes[0].set_title(f"{meta['recording']} {span} ROI {spectrum.roi.text()} "
f"black={spectrum.black:g} ADU{note}", fontsize=9)
axes[1].set_ylabel("Net / channel max")
axes[1].set_xlabel(unit)
axes[1].set_ylim(-0.05, 1.35)
fig.savefig(path, dpi=140)
plt.close(fig)
def peak_patch_box(spectrum: SpectrumData, line: dict, half_width_px: float,
min_rows: int = 24) -> tuple[int, int, int, int]:
"""Full-sensor pixel box (x0, y0, x1, y1) of the raw patch for one peak."""
roi = spectrum.roi
width, height = spectrum.full_size
x0 = int(math.floor(max(0.0, line["sensor_x_px"] - half_width_px)))
x1 = int(math.ceil(min(float(width), line["sensor_x_px"] + half_width_px)))
x0 -= x0 % 2
x1 = min(width, x1 + (x1 % 2))
x0 = max(0, min(x0, width - 2))
x1 = max(x0 + 2, x1)
block = spectrum.image[roi.y0:roi.y1, max(x0, roi.x0):min(x1, roi.x1)]
if block.size == 0:
return x0, roi.y0, x1, roi.y1
rows = np.maximum(block.astype(np.float64) - spectrum.black, 0.0).sum(axis=1)
if rows.max() <= 0:
return x0, roi.y0, x1, roi.y1
active = np.where(rows >= 0.2 * rows.max())[0]
y0 = int(active.min()) + roi.y0
y1 = int(active.max()) + 1 + roi.y0
pad = max(4, int(round(0.12 * (y1 - y0))))
y0 = max(roi.y0, y0 - pad)
y1 = min(roi.y1, y1 + pad)
if y1 - y0 < min_rows:
center = (y0 + y1) // 2
y0 = max(roi.y0, center - min_rows // 2)
y1 = min(roi.y1, y0 + min_rows)
y0 += y0 % 2
y1 -= y1 % 2
return x0, int(y0), x1, int(y1)
def slit_row_centroid(spectrum: SpectrumData, line: dict) -> float:
"""Intensity-weighted row centroid of the line inside the ROI."""
roi = spectrum.roi
center = int(round(line["sensor_x_px"]))
x0 = max(roi.x0, center - 2)
x1 = min(roi.x1, center + 3)
block = spectrum.image[roi.y0:roi.y1, x0:x1]
if block.size == 0:
return float("nan")
weight = np.maximum(block.astype(np.float64) - spectrum.black, 0.0).sum(axis=1)
total = float(weight.sum())
if total <= 0:
return float("nan")
rows = np.arange(roi.y0, roi.y0 + weight.size, dtype=float)
return float(np.dot(rows, weight) / total)
def zoom_range(spectrum: SpectrumData, line: dict) -> tuple[float, float]:
"""Sensor-column window shown around one peak."""
fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0
half = max(3.0, 2.5 * fwhm_px)
low = max(float(spectrum.roi.x0), line["sensor_x_px"] - half)
high = min(float(spectrum.roi.x1), line["sensor_x_px"] + half)
return low, high
def frame_preview(spectrum: SpectrumData) -> np.ndarray:
"""Auto-stretched RGB preview of the whole frame, as the player shows it."""
hq = spectrum.hq
source = spectrum.original
if source is None:
source = np.clip(np.rint(spectrum.image), 0.0, 65535.0).astype(np.uint16)
return hq.make_preview(source, spectrum.pattern)
def draw_preview_panel(ax, spectrum: SpectrumData, line: dict, preview: np.ndarray) -> None:
"""The whole-frame RAW preview the player shows, with the peak located on it.
Full sensor height and width (not just the ROI), so the spectrum band keeps
its real size and position in the frame; the ROI box and the peak column are
overlaid.
"""
from matplotlib.patches import Rectangle
width, height = spectrum.full_size # full_size is (width, height)
roi = spectrum.roi
fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0
if preview.size:
ax.imshow(preview, extent=(0.0, float(width), float(height), 0.0),
interpolation="nearest", aspect="auto")
ax.set_xlim(0.0, float(width))
ax.set_ylim(float(height), 0.0)
ax.add_patch(Rectangle((roi.x0, roi.y0), roi.width, roi.height,
fill=False, edgecolor="#f39c12", lw=0.9))
ax.axvline(line["sensor_x_px"], color="#00e5ff", lw=1.1)
ax.axvspan(line["sensor_x_px"] - 0.5 * fwhm_px, line["sensor_x_px"] + 0.5 * fwhm_px,
color="#00e5ff", alpha=0.20)
row = slit_row_centroid(spectrum, line)
if np.isfinite(row):
ax.plot([line["sensor_x_px"]], [row], marker="o", ms=5, mfc="none",
mec="#00e5ff", mew=1.2)
ax.set_title(f"RAW preview (auto-stretched) ROI {roi.text()} "
f"peak column x = {line['sensor_x_px']:.1f} px", fontsize=8)
ax.set_xlabel("Sensor column / px")
ax.set_ylabel("Sensor row / px")
ax.tick_params(labelsize=7)
def draw_raw_patch(ax, spectrum: SpectrumData, line: dict,
box: tuple[int, int, int, int]) -> None:
"""RAW Bayer crop around one peak, axes in full-sensor pixel coordinates."""
from matplotlib.colors import PowerNorm
x0, y0, x1, y1 = box
patch = spectrum.image[y0:y1, x0:x1]
fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0
if patch.size:
stretch = np.percentile(patch.astype(np.float64), [2, 99.95])
low_level = max(stretch[0], spectrum.black)
high_level = max(stretch[1], low_level + 1.0)
ax.imshow(patch, cmap="inferno", aspect="auto", interpolation="nearest",
extent=(x0, x1, y1, y0),
norm=PowerNorm(gamma=0.45, vmin=low_level, vmax=high_level))
ax.axvline(line["sensor_x_px"], color="#00e5ff", lw=1.0)
ax.axvspan(line["sensor_x_px"] - 0.5 * fwhm_px, line["sensor_x_px"] + 0.5 * fwhm_px,
color="#00e5ff", alpha=0.10)
ax.set_title(f"RAW Bayer patch (undemosaiced) x {x0}-{x1} y {y0}-{y1} (sensor px)",
fontsize=8)
ax.set_xlabel("Sensor column / px")
ax.set_ylabel("Sensor row / px")
ax.tick_params(labelsize=7)
def draw_zoom(ax, spectrum: SpectrumData, line: dict, calibration, args,
low: float, high: float, wavelength_axis: bool = True) -> None:
"""Net profiles of the channels forming this line, with fits and FWHM marks."""
hq = spectrum.hq
for channel in hq.CHANNELS:
peak = line["channels"].get(channel)
if peak is None:
continue
x = spectrum.x_px[channel]
mask = (x >= low) & (x <= high)
error = peak.get("center_err_px", float("nan"))
ax.plot(x[mask], spectrum.net[channel][mask], color=hq.CHANNEL_COLORS[channel],
lw=1.1, marker=".", ms=3.0,
label=f"{channel} {peak['height_net_adu']:.0f} ADU "
f"FWHM {peak['fwhm_px']:.2f} px"
+ (f" x±{error:.2f}" if np.isfinite(error) else ""))
if peak.get("fit_params"):
window = (x >= peak["window_lo_px"]) & (x <= peak["window_hi_px"])
values = model_values(x[window], peak["fit_params"], peak["fit_xref"], args.shape)
ax.plot(x[window], values, color=hq.CHANNEL_COLORS[channel], lw=0.9,
ls="--", alpha=0.9)
if np.isfinite(peak["fwhm_px"]):
half = 0.5 * peak["height_net_adu"]
ax.plot([peak["center_px"] - 0.5 * peak["fwhm_px"],
peak["center_px"] + 0.5 * peak["fwhm_px"]], [half, half],
color=hq.CHANNEL_COLORS[channel], lw=0.8, marker="|", ms=6)
# Mark every channel's own centre: the merged line centre can hide the fact
# that the channels disagree (a shoulder in one channel drags the average).
if np.isfinite(peak["center_px"]):
ax.axvline(peak["center_px"], color=hq.CHANNEL_COLORS[channel],
lw=0.8, ls="--", alpha=0.8)
ax.axvline(line["sensor_x_px"], color="#34495e", lw=0.9, ls=":")
spread = line.get("center_spread_px", float("nan"))
if np.isfinite(spread) and spread > 0:
used = "+".join(line.get("used_channels") or [])
ax.set_title(f"{line['id']} sensor x {line['sensor_x_px']:.2f} px "
f"channels disagree by {spread:.2f} px" +
(f" (centre from {used})" if used else ""), fontsize=9)
ax.set_xlim(low, high)
ax.set_xlabel("Sensor column / px")
ax.set_ylabel("Net ADU (baseline removed)")
ax.legend(fontsize=7, loc="upper right")
ax.grid(alpha=0.25)
ax.tick_params(labelsize=8)
if wavelength_axis:
_axis_wavelength(ax, calibration)
def plot_peak(path: Path, spectrum: SpectrumData, line: dict, calibration,
args, box: tuple[int, int, int, int]) -> None:
import matplotlib.pyplot as plt
hq = spectrum.hq
fwhm_px = line["fwhm_px"] if np.isfinite(line["fwhm_px"]) else 8.0
low, high = zoom_range(spectrum, line)
fig = plt.figure(figsize=(13.5, 9.0), constrained_layout=True)
grid = fig.add_gridspec(2, 2, height_ratios=[0.72, 1.0], width_ratios=[1.0, 1.25])
ax_context = fig.add_subplot(grid[0, :])
ax_raw = fig.add_subplot(grid[1, 0])
ax_zoom = fig.add_subplot(grid[1, 1])
for channel in hq.CHANNELS:
x = spectrum.x_px[channel]
x_plot = calibration(x) if calibration is not None else x
ax_context.plot(x_plot, spectrum.net[channel], color=hq.CHANNEL_COLORS[channel],
lw=0.7, alpha=0.85)
half_nm = (0.5 * line["fwhm_nm"] if np.isfinite(line["fwhm_nm"])
else 0.5 * fwhm_px * (line["dispersion_nm_per_px"]
if np.isfinite(line["dispersion_nm_per_px"]) else 1.0))
ax_context.axvspan(line["x_plot"] - 1.5 * half_nm, line["x_plot"] + 1.5 * half_nm,
color="#f1c40f", alpha=0.3, zorder=0)
frame_text = (f"frame {spectrum.indices[0]}"
if len(spectrum.indices) == 1 else
f"frames {spectrum.indices[0]}-{spectrum.indices[-1]}"
f" (mean {len(spectrum.indices)})")
title = (f"[{frame_text}] {line['id']} {line['wavelength_nm']:.2f} nm | sensor x = "
f"{line['sensor_x_px']:.3f} px | {line['channel_names']} "
f"SNR {line['snr']:.0f} FWHM {fwhm_px:.2f} px")
if np.isfinite(line["fwhm_nm"]):
title += f" ({line['fwhm_nm']:.2f} nm)"
if line["saturated"]:
title += " [SATURATED]"
if line["extrapolated"]:
title += " [extrapolated]"
if any(peak.get("deblended") for peak in line["channels"].values()):
title += " [pair fit]"
ax_context.set_title(title, fontsize=9)
ax_context.set_ylabel("Net ADU")
ax_context.set_xlabel("Wavelength / nm (air)" if calibration is not None
else "Sensor column / px")
ax_context.tick_params(labelsize=8)
ax_context.grid(alpha=0.25)
draw_raw_patch(ax_raw, spectrum, line, box)
draw_zoom(ax_zoom, spectrum, line, calibration, args, low, high, wavelength_axis=True)
fig.savefig(path, dpi=150)
plt.close(fig)
# ---------------------------------------------------------------------------
# Export
# ---------------------------------------------------------------------------
ALL_PEAK_FIELDS = [
"group_id", "channel", "sample_index", "sensor_x_px", "wavelength_nm", "extrapolated",
"height_net_adu", "prominence_adu", "raw_adu", "baseline_adu", "snr",
"fwhm_px", "fwhm_nm", "fwhm_halfmax_px", "center_px", "center_source", "center_parabola_px",
"center_fit_px", "center_err_px", "sigma_px", "area_net_adu_px",
"fit_ok", "fit_rms_adu", "deblended", "saturated",
]
SELECTED_FIELDS = [
"id", "wavelength_nm", "extrapolated", "channels", "primary_channel", "n_channels",
"sensor_x_px", "center_spread_px", "sample_index", "sample_step_px",
"dispersion_nm_per_px", "height_net_adu", "snr", "fwhm_px", "fwhm_nm",
"area_net_adu_px", "baseline_adu", "raw_adu", "saturated", "slit_y_px",
"roi_x0", "roi_y0", "roi_x1", "roi_y1",
"patch_x0", "patch_y0", "patch_x1", "patch_y1",
"frame", "frames_averaged", "image_png", "shape_csv", "raw_npy", "raw_csv",
"match_species", "match_nm", "match_delta_nm",
]
PROFILE_FIELDS = ["channel", "sample_index", "sensor_x_px", "wavelength_nm",
"raw_adu", "baseline_adu", "net_adu", "fit_adu", "residual_adu"]
def _clean_output(directory: Path) -> None:
for pattern in ("peak_*", "peaks*", "selected_peaks*"):
for stale in directory.glob(pattern):
if stale.is_file():
stale.unlink()
sub = directory / "peaks"
if sub.is_dir():
for stale in sub.glob("*"):
if stale.is_file():
stale.unlink()
def export_all(hq, spectrum: SpectrumData, lines: list[dict], calibration,
selected: set[str], args, meta: dict) -> dict:
import matplotlib
if not args.pick:
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: F401
out = Path(args.output).expanduser()
out.mkdir(parents=True, exist_ok=True)
_clean_output(out)
(out / "peaks").mkdir(exist_ok=True)
plot_overview(out / "peaks_overview.png", spectrum, lines, calibration, selected, meta)
all_rows = []
for line in lines:
for channel in hq.CHANNELS:
peak = line["channels"].get(channel)
if peak is None:
continue
values = []
for field in ALL_PEAK_FIELDS:
values.append(line["id"] if field == "group_id" else _num(peak.get(field)))
all_rows.append(values)
write_csv(out / "peaks_all.csv", ALL_PEAK_FIELDS, all_rows)
records = []
rows = []
for line in lines:
chosen = line["id"] in selected
box = peak_patch_box(spectrum, line, args.patch_half_width)
x0, y0, x1, y1 = box
slit_y = slit_row_centroid(spectrum, line)
link = {"image_png": "", "shape_csv": "", "raw_npy": "", "raw_csv": ""}
if chosen:
if not args.no_images:
plot_peak(out / "peaks" / f"peak_{line['id']}.png", spectrum, line,
calibration, args, box)
link["image_png"] = f"peaks/peak_{line['id']}.png"
link["shape_csv"] = write_peak_shape(out, spectrum, line, calibration, args)
link.update(write_patch(out, spectrum, line, box, args))
record = {
"id": line["id"],
"wavelength_nm": line["wavelength_nm"],
"extrapolated": line["extrapolated"],
"sensor_x_px": line["sensor_x_px"],
"center_spread_px": line["center_spread_px"],
"sample_index": line["sample_index"],
"sample_step_px": spectrum.step_px,
"dispersion_nm_per_px": line["dispersion_nm_per_px"],
"channels": line["channel_names"],
"primary_channel": line["primary_channel"],
"height_net_adu": line["height_net_adu"],
"snr": line["snr"],
"fwhm_px": line["fwhm_px"],
"fwhm_nm": line["fwhm_nm"],
"area_net_adu_px": line["area_net_adu_px"],
"saturated": line["saturated"],
"slit_y_px": slit_y,
"roi": [spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1],
"patch": [x0, y0, x1, y1],
"selected": chosen,
"match": {"species": line["match_species"], "nm": line["match_nm"],
"delta_nm": line["match_delta_nm"]},
"files": link,
"per_channel": {
channel: {key: peak.get(key) for key in (
"sample_index", "sensor_x_px", "center_px", "center_source",
"center_parabola_px", "center_fit_px",
"center_err_px", "sigma_px",
"fwhm_px", "fwhm_nm", "fwhm_halfmax_px", "wavelength_nm", "extrapolated",
"height_net_adu",
"prominence_adu", "raw_adu", "baseline_adu", "snr", "area_net_adu_px",
"fit_ok", "fit_rms_adu", "deblended", "saturated", "dispersion_nm_per_px",
"window_lo_px", "window_hi_px", "fit_params", "fit_xref")}
for channel, peak in line["channels"].items()
},
}
records.append(record)
if chosen:
primary = line["channels"][line["primary_channel"]]
rows.append([
line["id"], _num(line["wavelength_nm"]), _num(line["extrapolated"]),
line["channel_names"], line["primary_channel"], len(line["channels"]),
_num(line["sensor_x_px"]), _num(line["center_spread_px"]),
line["sample_index"], _num(spectrum.step_px),
_num(line["dispersion_nm_per_px"]), _num(line["height_net_adu"]),
_num(line["snr"]), _num(line["fwhm_px"]), _num(line["fwhm_nm"]),
_num(line["area_net_adu_px"]), _num(primary["baseline_adu"]),
_num(primary["raw_adu"]), _num(line["saturated"]), _num(slit_y),
spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1,
x0, y0, x1, y1, spectrum.indices[0], len(spectrum.indices),
link["image_png"], link["shape_csv"], link["raw_npy"], link["raw_csv"],
line["match_species"], _num(line["match_nm"]), _num(line["match_delta_nm"]),
])
write_csv(out / "selected_peaks.csv", SELECTED_FIELDS, rows)
manifest = {
"program": "peak_finder.py",
"created": meta["created"],
"recording": meta["recording"],
"frames": spectrum.indices,
"frames_averaged": len(spectrum.indices),
"full_size": list(spectrum.full_size),
"roi": [spectrum.roi.x0, spectrum.roi.y0, spectrum.roi.x1, spectrum.roi.y1],
"roi_text": spectrum.roi.text(),
"bayer_pattern": spectrum.pattern,
"raw_format": spectrum.raw_format,
"bit_depth": spectrum.bit_depth,
"black_adu": spectrum.black,
"full_scale_adu": spectrum.full_scale,
"reduce": args.reduce,
"baseline_window_px": args.baseline_window,
"noise_adu": spectrum.noise,
"shape": args.shape,
"thresholds": {
"peak_height_fraction": args.peak_height,
"peak_prominence_fraction": args.peak_prominence,
"min_snr": args.min_snr,
"peak_distance_px": args.peak_distance,
"max_peaks_per_channel": args.max_peaks,
"group_tolerance_px": args.group_tol,
},
"calibration": None if calibration is None else {
"path": calibration.path,
"coefficients_descending": calibration.coeff,
"calibrated_pixel_range": calibration.span,
"wavelength_offset_nm": calibration.offset_nm,
"reference_recording": calibration.data.get("recording"),
},
"coordinate_convention": {
"sensor_x_px": "0-based full-resolution sensor column; pixel x covers [x, x+1)",
"sensor_row": "0-based full-resolution sensor row; row 0 is the top of the frame",
"sample_index": "index into the reduced 1-D profile; samples are 2 px apart",
"roi": "[x0, y0, x1, y1) in sensor pixels",
"patch": "[x0, y0, x1, y1) of the exported raw patch",
"slit_y_px": "intensity-weighted row centroid of the line inside the ROI",
},
"selected_ids": sorted(selected),
"peaks": records,
}
write_json(out / "peaks.json", manifest)
return {"output": out, "records": records}
def write_peak_shape(out: Path, spectrum: SpectrumData, line: dict, calibration, args) -> str:
name = f"peaks/peak_{line['id']}_shape.csv"
rows = []
for channel in spectrum.hq.CHANNELS:
peak = line["channels"].get(channel)
if peak is None:
continue
x = spectrum.x_px[channel]
raw = spectrum.raw[channel]
baseline = spectrum.baseline[channel]
net = spectrum.net[channel]
mask = (x >= peak["window_lo_px"]) & (x <= peak["window_hi_px"])
model = (model_values(x, peak["fit_params"], peak["fit_xref"], args.shape)
if peak.get("fit_params") else np.full(x.size, np.nan))
for index in np.where(mask)[0]:
wavelength = float(calibration(x[index])) if calibration is not None else float("nan")
fitted = float(model[index])
rows.append([channel, int(index), _num(x[index]), _num(wavelength), _num(raw[index]),
_num(baseline[index]), _num(net[index]), _num(fitted),
_num(net[index] - fitted) if np.isfinite(fitted) else ""])
write_csv(out / name, PROFILE_FIELDS, rows)
return name
def write_patch(out: Path, spectrum: SpectrumData, line: dict,
box: tuple[int, int, int, int], args) -> dict:
x0, y0, x1, y1 = box
crop = (spectrum.original[y0:y1, x0:x1] if spectrum.original is not None
else spectrum.image[y0:y1, x0:x1].astype(np.float32))
# The file name carries the pixel box, so the dump is self-describing.
stem = f"peak_{line['id']}_raw_y{y0}-{y1}_x{x0}-{x1}"
links = {"raw_npy": "", "raw_csv": ""}
if args.patch_format in ("npy", "both"):
np.save(out / "peaks" / f"{stem}.npy", crop)
links["raw_npy"] = f"peaks/{stem}.npy"
if args.patch_format in ("csv", "both") or args.patch_csv:
rows = [[y0 + row, x0 + column, _num(crop[row, column])]
for row in range(crop.shape[0]) for column in range(crop.shape[1])]
write_csv(out / "peaks" / f"{stem}.csv", ["sensor_row", "sensor_col", "adu"], rows)
links["raw_csv"] = f"peaks/{stem}.csv"
return links
# ---------------------------------------------------------------------------
# Interactive picking
# ---------------------------------------------------------------------------
def fit_window_to_screen(fig, width_fraction: float = 0.94,
height_fraction: float = 0.90) -> None:
"""Shrink the figure window when it would be larger than the screen.
A figure is sized in inches x dpi, which on scaled displays can exceed the
screen; the window then opens partly off-screen and looks "cut in half".
"""
try:
window = getattr(fig.canvas.manager, "window", None)
if window is None or not hasattr(window, "winfo_screenwidth"):
return
screen_w = float(window.winfo_screenwidth())
screen_h = float(window.winfo_screenheight())
width_in, height_in = fig.get_size_inches()
dpi = float(fig.dpi)
scale = min(width_fraction * screen_w / (width_in * dpi),
height_fraction * screen_h / (height_in * dpi), 1.0)
if scale < 0.995:
fig.set_size_inches(width_in * scale, height_in * scale)
except Exception:
pass
def apply_merges(lines: list[dict], merges: list[list[str]], calibration) -> list[dict]:
"""Merge manually chosen lines; the smallest id survives the merge.
The centre is the height-weighted mean of the member centres, and each channel
keeps its strongest member. Two lines that both peak in G are one merged line
with one G entry, so the ids are recorded separately in `merged_from`.
"""
if not merges:
return list(lines)
by_id = {line["id"]: line for line in lines}
consumed: set[str] = set()
merged_lines: list[dict] = []
for group in merges:
members = [by_id[name] for name in group if name in by_id and name not in consumed]
if len(members) < 2:
continue
weights = np.array([max(0.0, item["height_net_adu"]) for item in members])
positions = np.array([item["sensor_x_px"] for item in members])
centre = (float(np.dot(weights, positions) / weights.sum())
if weights.sum() > 0 else float(positions.mean()))
strongest = max(members, key=lambda item: item["height_net_adu"])
channels: dict[str, dict] = {}
for member in members:
for channel, peak in member["channels"].items():
kept = channels.get(channel)
if kept is None or peak["height_net_adu"] > kept["height_net_adu"]:
channels[channel] = dict(peak)
consumed.add(member["id"])
merged = dict(strongest)
merged.update({
"id": members[0]["id"],
"sensor_x_px": centre,
"center_spread_px": float(positions.max() - positions.min()),
"channels": channels,
"channel_names": "".join(c for c in ("R", "G", "B") if c in channels),
"merged_from": [member["id"] for member in members],
})
merged_lines.append(merged)
result = merged_lines + [line for line in lines if line["id"] not in consumed]
attach_wavelength(result, calibration)
result.sort(key=lambda item: item["sensor_x_px"])
return result
def pick_peaks(spectrum: SpectrumData, lines: list[dict], calibration,
selected: set[str]) -> Optional[set[str]]:
"""Browse peaks with a live RAW/zoom preview.
Hover moves the preview, click toggles selection, ``a``/``n`` select all or
none, ``e`` exports and ``q`` cancels.
"""
import matplotlib.pyplot as plt
hq = spectrum.hq
args = spectrum.args
fig = plt.figure(figsize=(16.5, 8.5))
fit_window_to_screen(fig)
# Fixed axes rectangles instead of a layout engine: the three panels must be
# there at any window size, and constrained_layout can collapse a column on
# odd geometries.
ax_over = fig.add_axes([0.045, 0.085, 0.49, 0.84])
ax_raw = fig.add_axes([0.585, 0.60, 0.39, 0.33])
ax_zoom = fig.add_axes([0.585, 0.075, 0.39, 0.385])
for channel in hq.CHANNELS:
x = spectrum.x_px[channel]
x_plot = calibration(x) if calibration is not None else x
peak = float(np.max(spectrum.net[channel])) if spectrum.net[channel].size else 0.0
ax_over.plot(x_plot, spectrum.net[channel] / peak if peak > 0 else spectrum.net[channel],
color=hq.CHANNEL_COLORS[channel], lw=0.7, label=channel)
handles = {line["id"]: ax_over.axvline(line["x_plot"], lw=1.0) for line in lines}
ax_over.set_xlabel("Wavelength / nm (air)" if calibration is not None
else "Sensor column / px")
ax_over.set_ylabel("Net / channel max")
ax_over.set_ylim(-0.05, 1.25)
ax_over.grid(alpha=0.25)
ax_over.legend(loc="upper right", fontsize=8)
preview = frame_preview(spectrum)
base_lines = list(lines)
state = {"selected": set(selected),
"focus": base_lines[0]["id"] if base_lines else None,
"export": False,
"merges": [],
"view": list(base_lines)}
def nearest(x_value: float):
found, best = None, float("inf")
for line in state["view"]:
distance = abs(line["x_plot"] - x_value)
if distance < best:
found, best = line, distance
span = abs(ax_over.get_xlim()[1] - ax_over.get_xlim()[0])
return found if found is not None and best <= 0.02 * span else None
def rebuild_view() -> None:
state["view"] = apply_merges(base_lines, state["merges"], calibration)
for handle in handles.values():
handle.remove()
handles.clear()
for line in state["view"]:
handles[line["id"]] = ax_over.axvline(line["x_plot"], lw=1.0)
ids = {line["id"] for line in state["view"]}
state["selected"] &= ids
if state["focus"] not in ids:
state["focus"] = state["view"][0]["id"] if state["view"] else None
def refresh() -> None:
for key, handle in handles.items():
chosen = key in state["selected"]
handle.set_color("#c0392b" if chosen else "#bdc3c7")
handle.set_alpha(1.0 if chosen else 0.45)
handle.set_linewidth(2.2 if key == state["focus"] else 1.0)
chosen = sorted(state["selected"])
ax_over.set_title("Selected: " + (", ".join(chosen) if chosen else "none")
+ " | hover = preview, click = toggle, a = all, n = none, "
"m = merge selected, s = split, u = undo, "
"e = export, q = cancel", fontsize=9)
ax_raw.clear()
ax_zoom.clear()
line = {item["id"]: item for item in state["view"]}.get(state["focus"])
if line is not None:
draw_preview_panel(ax_raw, spectrum, line, preview)
low, high = zoom_range(spectrum, line)
draw_zoom(ax_zoom, spectrum, line, calibration, args, low, high,
wavelength_axis=False)
flags = (" [SATURATED]" if line["saturated"] else "") + \
(" [extrapolated]" if line["extrapolated"] else "")
wave = (f"{line['wavelength_nm']:.2f} nm "
if np.isfinite(line["wavelength_nm"]) else "")
mark = "* " if line["id"] in state["selected"] else ""
ax_zoom.set_title(f"{mark}{line['id']} {wave}sensor x "
f"{line['sensor_x_px']:.2f} px {line['channel_names']} "
f"SNR {line['snr']:.0f}{flags}", fontsize=8)
fig.canvas.draw_idle()
def on_click(event) -> None:
if event.inaxes is not ax_over or event.xdata is None:
return
line = nearest(event.xdata)
if line is None:
return
state["focus"] = line["id"]
if line["id"] in state["selected"]:
state["selected"].remove(line["id"])
else:
state["selected"].add(line["id"])
refresh()
def on_move(event) -> None:
if event.inaxes is not ax_over or event.xdata is None:
return
line = nearest(event.xdata)
if line is not None and line["id"] != state["focus"]:
state["focus"] = line["id"]
refresh()
def on_key(event) -> None:
if event.key == "m":
group = sorted(state["selected"])
if len(group) >= 2:
state["merges"].append(group)
rebuild_view()
state["selected"] = {state["view"][0]["id"]} if False else {group[0]}
state["focus"] = group[0]
refresh()
elif event.key == "u":
if state["merges"]:
state["merges"].pop()
rebuild_view()
refresh()
elif event.key == "s":
focused = {line["id"]: line for line in state["view"]}.get(state["focus"])
if focused is not None and focused.get("merged_from"):
for group in list(state["merges"]):
if focused["id"] in group:
state["merges"].remove(group)
rebuild_view()
refresh()
elif event.key == "a":
state["selected"] = {line["id"] for line in state["view"]}
refresh()
elif event.key == "n":
state["selected"] = set()
refresh()
elif event.key == "e":
state["export"] = True
plt.close(fig)
elif event.key == "q":
plt.close(fig)
fig.canvas.mpl_connect("button_press_event", on_click)
fig.canvas.mpl_connect("motion_notify_event", on_move)
fig.canvas.mpl_connect("key_press_event", on_key)
refresh()
plt.show(block=True)
if not state["export"]:
return None
return {"selected": state["selected"], "merges": state["merges"],
"lines": state["view"]}
# ---------------------------------------------------------------------------
# Command line
# ---------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Find spectral peaks in hq_record RAW recordings, calibrate them and "
"export peak shapes and pixel coordinates.",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
io_group = parser.add_argument_group("input")
io_group.add_argument("--input", required=True, metavar="REC_DIR",
help="hq_record recording directory")
io_group.add_argument("--frame", type=int, default=0, help="zero-based frame index (default 0)")
io_group.add_argument("--average", type=int, default=1,
help="mean this many consecutive frames starting at --frame (default 1)")
io_group.add_argument("--roi", default="auto",
help="'auto', x0,y0,x1,y1, or 'calibration' to reuse the saved ROI")
io_group.add_argument("--rows", default=None, metavar="Y0,Y1",
help="restrict the ROI to these sensor rows")
io_group.add_argument("--reduce", choices=("mean", "median"), default="mean",
help="row reduction inside the ROI (default mean)")
io_group.add_argument("--black", type=float, default=None, help="black level override in ADU")
io_group.add_argument("--full-scale", type=float, default=None,
help="white level override in decoded ADU")
io_group.add_argument("--calibration", default=None,
help="calibration JSON to apply to the sensor columns; omit it to export pixel coordinates only")
io_group.add_argument("--wavelength-offset", type=float, default=0.0, metavar="NM",
help="manual shift added to the calibration (default 0)")
io_group.add_argument("--module-dir", default=None,
help="directory containing hq_spectrum.py (default this folder)")
io_group.add_argument("--quiet", action="store_true", help="suppress progress output")
detect = parser.add_argument_group("peak detection")
detect.add_argument("--baseline-window", type=float, default=100.0, metavar="PX",
help="baseline opening width in sensor px; 0 disables the baseline")
detect.add_argument("--peak-distance", type=float, default=4.0, metavar="PX",
help="minimum distance between peaks in sensor px (2 px = one sample); "
"raise it to merge barely resolved blends")
detect.add_argument("--peak-height", type=float, default=0.02,
help="minimum peak height as a fraction of the channel maximum")
detect.add_argument("--peak-prominence", type=float, default=0.01,
help="minimum peak prominence as a fraction of the channel maximum")
detect.add_argument("--min-snr", type=float, default=15.0,
help="minimum peak height in units of the channel noise")
detect.add_argument("--max-peaks", type=int, default=40, help="maximum peaks per channel")
detect.add_argument("--group-tol", type=float, default=4.0, metavar="PX",
help="channel peaks closer than this are one line")
detect.add_argument("--fwhm-span", type=float, default=60.0, metavar="PX",
help="maximum half-width search half-range in sensor px")
detect.add_argument("--fit-window", type=float, default=0.0, metavar="PX",
help="half-width of the fit window in sensor px; 0 = automatic")
detect.add_argument("--shape", choices=SHAPES, default="gauss", help="line-shape model")
export = parser.add_argument_group("selection and export")
export.add_argument("--select", default="all", metavar="TOKENS",
help="all | none | 3 | 3-7 | nm:540-560 | top:5 (default all)")
export.add_argument("--pick", action="store_true",
help="choose peaks by clicking in a window, then export")
export.add_argument("--output", default=None, metavar="DIR",
help="output directory (default peak_exports/<recording>_f<frame>)")
export.add_argument("--patch-half-width", type=float, default=24.0, metavar="PX",
help="half-width of the exported raw patch in sensor px")
export.add_argument("--patch-format", choices=("npy", "csv", "both", "none"), default="npy",
help="raw patch dump format (default npy)")
export.add_argument("--patch-csv", action="store_true",
help="also dump the raw patch as a long CSV")
export.add_argument("--no-images", action="store_true", help="skip the per-peak PNG images")
export.add_argument("--match", default=None, metavar="TABLE",
help="CSV/JSON line table used to suggest peak identities")
export.add_argument("--match-tolerance", type=float, default=1.5, metavar="NM",
help="matching tolerance in nm (default 1.5)")
return parser
def validate_arguments(args) -> None:
if args.average < 1:
raise ValueError("--average must be >= 1")
if args.max_peaks < 1:
raise ValueError("--max-peaks must be >= 1")
if args.peak_distance <= 0 or args.baseline_window < 0 or args.fwhm_span <= 0:
raise ValueError("--peak-distance and --fwhm-span must be > 0; --baseline-window >= 0")
if not 0 <= args.peak_height <= 1 or not 0 <= args.peak_prominence <= 1:
raise ValueError("--peak-height and --peak-prominence are fractions between 0 and 1")
if args.min_snr < 0 or args.group_tol <= 0 or args.patch_half_width <= 0:
raise ValueError("--min-snr >= 0, --group-tol > 0 and --patch-half-width > 0 required")
if args.fit_window < 0 or args.match_tolerance <= 0:
raise ValueError("--fit-window >= 0 and --match-tolerance > 0 required")
class Analysis:
"""Everything one recording yields: reduced spectrum, grouped lines, calibration."""
def __init__(self, hq, spectrum: "SpectrumData", lines: list[dict], calibration,
indices: list[int], per_channel: dict[str, list[dict]]):
self.hq = hq
self.spectrum = spectrum
self.lines = lines
self.calibration = calibration
self.indices = indices
self.per_channel = per_channel
def analyse(args, tell=print) -> "Analysis":
"""Read the frames, reduce the spectrum and detect/group lines.
Shared by ``run()`` (which then exports) and by wavelength_calibration.py
(which only needs the line positions).
"""
validate_arguments(args)
hq = load_hq(args.module_dir)
# A calibration is only applied when the caller names one: this program does
# not calibrate and must not silently attach someone else's wavelength axis.
calibration = None
if getattr(args, "calibration", None):
calibration = Calibration(args.calibration, args.wavelength_offset)
source = hq.RecordingSource(args.input)
try:
source.start()
if calibration is not None:
calibration.validate_geometry(source, tell)
calibration.set_width(source.full_size[0])
image, original, indices, frame_meta = read_frames(hq, source, args, tell)
if str(args.roi).strip().lower() == "calibration":
saved = calibration.data.get("roi") if calibration is not None else None
if not saved or len(saved) != 4:
raise ValueError("--roi calibration needs a calibration JSON with a roi field")
roi = hq.parse_roi(",".join(str(value) for value in saved), *source.full_size)
else:
roi = hq.parse_roi(args.roi, *source.full_size)
if roi is None:
# hq_spectrum.make_preview() bins integer Bayer sites, so hand it
# the original uint16 frame (or a rounded copy of the mean frame).
preview_source = original if original is not None else np.clip(
np.rint(image), 0.0, 65535.0).astype(np.uint16)
roi = hq.detect_auto_roi(hq.make_preview(preview_source, source.pattern),
*source.full_size)
if roi is None:
raise ValueError("auto ROI failed; specify --roi x0,y0,x1,y1")
roi = resolve_rows(args.rows, roi, source.full_size[1], hq)
meta = dict(source.base_meta)
meta.update({key: value for key, value in frame_meta.items()
if not str(key).startswith("_")})
black, full_scale = hq.resolve_levels(meta, source.bit_depth, args.black, args.full_scale)
spectrum = SpectrumData(hq, source, image, original, indices, roi, args, black, full_scale)
finally:
source.close()
tell(f"recording : {Path(args.input).expanduser().resolve()}")
tell(f"frames : {indices[0]}-{indices[-1]} (mean of {len(indices)})")
tell(f"ROI : {roi.text()} ({roi.width} x {roi.height} px)")
tell(f"black level : {black:g} ADU full scale {full_scale:g} ADU")
tell("noise (ADU) : " + ", ".join(f"{c}={spectrum.noise[c]:.1f}" for c in hq.CHANNELS))
if calibration is not None:
tell(f"calibration : {calibration.describe()}")
else:
tell("calibration : none - pixel coordinates only "
"(pass --calibration JSON to convert columns to wavelength)")
per_channel = spectrum.detect()
lines = group_channels(per_channel, args.group_tol)
attach_wavelength(lines, calibration)
for line in lines:
line["sample_step_px"] = spectrum.step_px
if getattr(args, "match", None):
match_lines(lines, load_line_table(args.match), args.match_tolerance)
return Analysis(hq, spectrum, lines, calibration, indices, per_channel)
def run(args) -> int:
analysis = analyse(args)
hq, spectrum, lines = analysis.hq, analysis.spectrum, analysis.lines
calibration, indices = analysis.calibration, analysis.indices
tell = (lambda *parts, **kwargs: None) if args.quiet else print
roi = spectrum.roi
if args.pick:
import matplotlib
if matplotlib.get_backend().lower() in ("agg", "pdf", "svg", "ps", "cairo", "template"):
raise ValueError("--pick needs an interactive Matplotlib backend; use --select instead")
try:
picked = pick_peaks(spectrum, lines, calibration, set())
except Exception as exc: # no usable display, missing Tk, ...
raise ValueError(f"--pick could not open a window ({exc}); use --select instead") from exc
if picked is None:
tell("Selection cancelled; nothing exported.")
return 1
lines = picked["lines"] # manual merges applied
selected = picked["selected"]
elif not lines:
selected = set()
else:
selected = parse_selection(args.select, lines)
if args.output is None:
args.output = str(default_output_directory(args.input, indices))
meta_out = {"created": _timestamp(),
"recording": str(Path(args.input).expanduser().resolve())}
result = export_all(hq, spectrum, lines, calibration, selected, args, meta_out)
tell("")
tell(f"{'':2s}{'id':4s}{'nm':>10s}{'x_px':>10s}{'ch':>4s}{'SNR':>9s}{'height':>9s}"
f"{'FWHM px':>9s}{'FWHM nm':>9s}{'slit y':>9s} flags")
for line in lines:
flags = []
if line["extrapolated"]:
flags.append("extrapolated")
if line["saturated"]:
flags.append("saturated")
if line["match_species"]:
flags.append(f"{line['match_species']} {line['match_delta_nm']:+.2f} nm")
mark = "*" if line["id"] in selected else " "
tell(f"{mark:2s}{line['id']:4s}{_num(line['wavelength_nm'], 7):>10s}"
f"{line['sensor_x_px']:10.3f}{line['channel_names']:>4s}{line['snr']:9.0f}"
f"{line['height_net_adu']:9.0f}{_num(line['fwhm_px'], 4):>9s}"
f"{_num(line['fwhm_nm'], 4):>9s}{_num(slit_row_centroid(spectrum, line), 6):>9s}"
f" {', '.join(flags)}")
total = sum(len(items) for items in analysis.per_channel.values())
tell("")
tell(f"{len(lines)} line(s) from {total} channel peak(s); {len(selected)} selected.")
tell(f"exported : {result['output'].expanduser().resolve()}")
if calibration is not None:
outside = [line["id"] for line in lines if line["extrapolated"]]
if outside:
low, high = calibration.nm_span
tell(f"warning : {len(outside)} line(s) outside the calibrated range "
f"{low:.1f}-{high:.1f} nm ({', '.join(outside[:8])}"
f"{'...' if len(outside) > 8 else ''})")
return 0
def default_output_directory(recording, indices) -> Path:
"""Default export folder: ``peak_exports/<recording>_f0003[_avgN]``."""
stem = Path(recording).expanduser().resolve().name
return _shared_output("peak_exports", stem, indices)
def _timestamp() -> str:
from datetime import datetime
return datetime.now().astimezone().isoformat(timespec="seconds")
def main(argv: Optional[list[str]] = None) -> int:
args = build_parser().parse_args(argv)
try:
return run(args)
except (ValueError, RuntimeError, OSError, IndexError, StopIteration, KeyError) as exc:
print(f"Peak finder error: {exc}", file=sys.stderr)
return 2
except KeyboardInterrupt:
print("Interrupted.", file=sys.stderr)
return 130
if __name__ == "__main__":
raise SystemExit(main())