Files
starcells 45249b59f9 光谱工具集:采集、回放、寻峰、波长标定
- spectro_core.py: 共享核心(加载/标定/基线/寻峰/高斯拟合/双峰联合拟合/CSV-JSON)

- peak_finder.py: 寻峰与峰信息导出,--pick 交互选峰(含 RAW 预览)

- wavelength_calibration.py: 改用 core 寻峰,汞灯 577/579 双峰可分

- wavelength_player.py / hq_average.py: 复用 core 的标定与解码加载
2026-11-14 13:48:24 +08:00

397 lines
15 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
hq_record.py
============
Fast raw recorder for the Raspberry Pi HQ Camera.
This is the "record" half of the split workflow:
hq_record.py -> recordings/hq_YYYYmmdd_HHMMSS/
hq_spectrum.py --input REC_DIR -> replay / analysis
The recorder does not import Matplotlib and does not compute spectra. It only
pulls the Picamera2 RAW stream, queues each raw byte buffer, and writes it to
`frames.bin`. A background writer thread keeps disk I/O away from the camera
capture loop.
Recording layout
----------------
REC_DIR/
meta.json camera/format/stride/frame count
frames.bin raw byte buffers, one after another, no decoding
frames.jsonl one line of camera metadata per frame
For the usual Pi 5 HQ Camera configuration the raw stream is
`BGGR_PISP_COMP1` (PC1B). Each frame is height*stride bytes; at 2028x1520
that is typically about 3.1 MB/frame. At 15 fps that is about 47 MB/s, so a
USB SSD/NVMe is preferable to a slow SD card.
Examples
--------
# list modes first if needed
python3 hq_record.py --list-modes
# 10 seconds, 15 fps, manual exposure copied from rpicam-hello
python3 hq_record.py --duration 10 --fps 15 \
--exposure 33013 --gain 16 \
--record-dir recordings/hg_lamp
# unlimited until Ctrl+C
python3 hq_record.py --duration 0 --fps 15 \
--record-dir recordings/hg_long
# analyze later (on the Pi or on a PC)
python3 hq_spectrum.py --input recordings/hg_lamp --fps 15
The recorder is deliberately independent of the analyzer: a slow Matplotlib
redraw can no longer cause dropped frames or change the exposure timing.
"""
from __future__ import annotations
import argparse
import json
import queue
import shutil
import sys
import threading
import time
from datetime import datetime
from pathlib import Path
from types import SimpleNamespace
from typing import Optional
import numpy as np
try:
from hq_spectrum import CameraSource, list_camera_modes
except ImportError as exc: # pragma: no cover - import error path
raise SystemExit(
"hq_record.py must sit next to hq_spectrum.py "
"(run it from the hq_spectrum directory)."
) from exc
class RawWriter(threading.Thread):
"""Write queued raw buffers and metadata lines in the background."""
def __init__(self, frames_path: Path, metadata_path: Path, frame_bytes: int,
queue_size: int):
super().__init__(name="hq-raw-writer", daemon=False)
self.frames_path = frames_path
self.metadata_path = metadata_path
self.frame_bytes = frame_bytes
self.queue: queue.Queue = queue.Queue(maxsize=max(2, queue_size))
self.frames_written = 0
self.error: Optional[BaseException] = None
self.finished = threading.Event()
def put(self, raw: np.ndarray, metadata: dict, host_time_ns: int) -> None:
if self.error is not None:
raise RuntimeError(f"writer failed: {self.error}")
raw = np.ascontiguousarray(raw)
if raw.dtype != np.uint8 or raw.nbytes != self.frame_bytes:
raise ValueError(
f"unexpected raw buffer {raw.shape} {raw.dtype} {raw.nbytes}B; "
f"expected {self.frame_bytes}B uint8"
)
item = (raw, dict(metadata), int(host_time_ns))
while True:
try:
self.queue.put(item, timeout=0.5)
return
except queue.Full:
if self.error is not None:
raise RuntimeError(f"writer failed: {self.error}")
def run(self) -> None:
try:
with open(self.frames_path, "wb", buffering=4 * 1024 * 1024) as frames, \
open(self.metadata_path, "w", encoding="utf-8", buffering=1024 * 1024) as meta_out:
while True:
item = self.queue.get()
if item is None:
break
raw, camera_meta, host_time_ns = item
offset = self.frames_written * self.frame_bytes
record = dict(camera_meta)
record.update({
"i": self.frames_written,
"offset": offset,
"bytes": raw.nbytes,
"host_time_ns": host_time_ns,
})
frames.write(memoryview(raw).cast("B"))
meta_out.write(json.dumps(record, default=str, separators=(",", ":")) + "\n")
self.frames_written += 1
if self.frames_written % 30 == 0:
frames.flush()
meta_out.flush()
frames.flush()
meta_out.flush()
self.finished.set()
except BaseException as exc: # noqa: BLE001 - report writer failure to main
self.error = exc
self.finished.set()
def stop(self) -> None:
if self.is_alive():
try:
self.queue.put(None, timeout=2.0)
except queue.Full:
pass
self.join(timeout=30.0)
if self.is_alive():
raise RuntimeError("writer thread did not stop within 30 seconds")
if self.error is not None:
raise RuntimeError(f"writer failed: {self.error}")
def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Record Raspberry Pi HQ Camera RAW frames for later spectrum analysis.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--width", type=int, default=2028, help="raw stream width")
parser.add_argument("--height", type=int, default=1520, help="raw stream height")
parser.add_argument("--fps", type=float, default=15.0, help="requested frame rate")
parser.add_argument("--duration", type=float, default=10.0,
help="seconds to record; 0 means until Ctrl+C")
parser.add_argument("--frames", type=int, default=0,
help="optional frame limit; 0 means no limit")
parser.add_argument("--record-dir", default=None,
help="output directory (default recordings/hq_YYYYmmdd_HHMMSS)")
parser.add_argument("--queue-size", type=int, default=8,
help="raw frames buffered in RAM while the writer catches up")
parser.add_argument("--exposure", type=float, default=None,
help="manual exposure time in microseconds")
parser.add_argument("--gain", type=float, default=None, help="manual analogue gain")
parser.add_argument("--raw-format", default=None,
help="optional raw format, e.g. SBGGR12 or BGGR_PISP_COMP1")
parser.add_argument("--list-modes", action="store_true",
help="list camera raw modes and exit")
args = parser.parse_args(argv)
if args.width < 64 or args.height < 64:
parser.error("width and height must be at least 64")
if args.fps <= 0:
parser.error("--fps must be positive")
if args.duration < 0:
parser.error("--duration must be >= 0")
if args.frames < 0:
parser.error("--frames must be >= 0")
if args.queue_size < 2:
parser.error("--queue-size must be >= 2")
return args
def write_json(path: Path, data: dict) -> None:
tmp = path.with_suffix(path.suffix + ".tmp")
with open(tmp, "w", encoding="utf-8") as handle:
json.dump(data, handle, indent=2, default=str)
handle.write("\n")
tmp.replace(path)
def prepare_directory(path: Path) -> None:
if path.exists():
if not path.is_dir():
raise RuntimeError(f"{path} exists and is not a directory")
if any(path.iterdir()):
raise RuntimeError(
f"{path} is not empty; choose a new --record-dir "
f"(the recorder never overwrites old data)"
)
else:
path.mkdir(parents=True, exist_ok=True)
def main(argv: Optional[list[str]] = None) -> int:
args = parse_args(argv)
if args.list_modes:
list_camera_modes()
return 0
cam_args = SimpleNamespace(
width=args.width,
height=args.height,
fps=args.fps,
exposure=args.exposure,
gain=args.gain,
raw_format=args.raw_format,
)
source = None
writer = None
meta = None
record_dir = None
enqueued = 0
started = None
exit_code = 0
try:
source = CameraSource(cam_args)
print(f"camera : {source.description}")
print(f"raw stream : {source.raw_format} "
f"{source.full_size[0]}x{source.full_size[1]}")
print(f"stride : {source.stride}")
print(f"Bayer : {source.pattern} decoded {source.bit_depth}-bit buffer")
print(f"target rate : {args.fps:g} fps")
if "PISP_COMP" in source.raw_format.upper():
print("note : recording the compressed PC1B buffer; no decode on the Pi")
source.start()
last_meta = {}
# Discard warm-up frames; the recording should start with settled AE.
for _ in range(5):
_, last_meta = source.read_raw()
# Get one sample only to validate the actual buffer size.
sample_raw, sample_meta = source.read_raw()
last_meta = sample_meta
if sample_raw.dtype != np.uint8:
raise RuntimeError(f"unexpected raw dtype {sample_raw.dtype}; expected uint8 bytes")
frame_bytes = int(sample_raw.nbytes)
expected_bytes = source.stride * source.full_size[1]
if frame_bytes != expected_bytes:
raise RuntimeError(
f"raw buffer is {frame_bytes} B but stride*height is {expected_bytes} B; "
f"refusing to write a recording that the analyzer would misread"
)
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
record_dir = Path(args.record_dir).expanduser() if args.record_dir else \
Path("recordings") / f"hq_{stamp}"
prepare_directory(record_dir)
frames_path = record_dir / "frames.bin"
metadata_path = record_dir / "frames.jsonl"
meta_path = record_dir / "meta.json"
data_rate = frame_bytes * args.fps
free = shutil.disk_usage(record_dir).free
print(f"output : {record_dir}")
print(f"frame size : {frame_bytes / 1e6:.3f} MB")
print(f"data rate : {data_rate / 1e6:.1f} MB/s")
print(f"free space : {free / 1e9:.2f} GB")
planned_frames = None
if args.frames:
planned_frames = args.frames
if args.duration:
duration_frames = int(args.duration * args.fps)
planned_frames = duration_frames if planned_frames is None else min(planned_frames, duration_frames)
if planned_frames is not None:
needed = frame_bytes * planned_frames * 1.05
print(f"planned size : {needed / 1e9:.2f} GB for ~{planned_frames} frames")
if needed > free:
raise RuntimeError(
f"not enough free space: need ~{needed / 1e9:.2f} GB, "
f"have {free / 1e9:.2f} GB"
)
meta = {
"format_version": 1,
"kind": "hq_raw_recording",
"created_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"camera": source.description,
"raw_format": source.raw_format,
"width": source.full_size[0],
"height": source.full_size[1],
"stride": source.stride,
"frame_bytes": frame_bytes,
"storage_dtype": "uint8",
"pattern": source.pattern,
"bit_depth": source.bit_depth,
"fps_requested": args.fps,
"exposure_us": args.exposure,
"gain": args.gain,
"black_level": last_meta.get("BlackLevel"),
"duration_limit_s": args.duration,
"frame_limit": args.frames,
"frames": 0,
"completed": False,
}
write_json(meta_path, meta)
writer = RawWriter(frames_path, metadata_path, frame_bytes, args.queue_size)
writer.start()
# The first sample was taken to validate the format, so it is also the
# first frame of the recording.
writer.put(sample_raw, last_meta, time.time_ns())
enqueued = 1
started = time.monotonic()
last_report = started
while True:
if args.frames and enqueued >= args.frames:
break
elapsed = time.monotonic() - started
if args.duration and elapsed >= args.duration:
break
raw, camera_meta = source.read_raw()
writer.put(raw, camera_meta, time.time_ns())
enqueued += 1
now = time.monotonic()
if now - last_report >= 1.0:
written = writer.frames_written
rate = enqueued / max(now - started, 1e-6)
print(f"recording : {enqueued} frames captured, {written} written, "
f"{rate:.1f} fps")
last_report = now
elapsed = time.monotonic() - started
writer.stop()
writer_error = writer.error
writer = None
if writer_error is not None:
raise RuntimeError(f"writer failed: {writer_error}")
meta.update({
"frames": int(enqueued),
"duration_s": round(elapsed, 3),
"average_capture_fps": round(enqueued / max(elapsed, 1e-6), 3),
"completed": True,
"ended_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"frames_file": "frames.bin",
"metadata_file": "frames.jsonl",
})
write_json(meta_path, meta)
size = frames_path.stat().st_size
print(f"done : {enqueued} frames, {size / 1e9:.2f} GB, "
f"{enqueued / max(elapsed, 1e-6):.1f} fps")
print(f"analyze with : python3 hq_spectrum.py --input \"{record_dir}\"")
return 0
except KeyboardInterrupt:
print("\ninterrupted; finalizing recording", file=sys.stderr)
exit_code = 130
return exit_code
except Exception as exc:
print(f"error: {exc}", file=sys.stderr)
exit_code = 1
return exit_code
finally:
if source is not None:
try:
source.close()
except Exception:
pass
if writer is not None:
try:
writer.stop()
except Exception as exc:
print(f"writer shutdown error: {exc}", file=sys.stderr)
exit_code = 1
if meta is not None and record_dir is not None:
try:
meta.update({
"frames": int(writer.frames_written) if writer is not None else int(enqueued),
"completed": exit_code == 0 and meta.get("completed", False),
"ended_at": datetime.now().astimezone().isoformat(timespec="seconds"),
})
write_json(record_dir / "meta.json", meta)
except Exception as exc:
print(f"warning: could not finalize meta.json: {exc}", file=sys.stderr)
if __name__ == "__main__":
raise SystemExit(main())