- spectro_core.py: 共享核心(加载/标定/基线/寻峰/高斯拟合/双峰联合拟合/CSV-JSON) - peak_finder.py: 寻峰与峰信息导出,--pick 交互选峰(含 RAW 预览) - wavelength_calibration.py: 改用 core 寻峰,汞灯 577/579 双峰可分 - wavelength_player.py / hq_average.py: 复用 core 的标定与解码加载
397 lines
15 KiB
Python
397 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
hq_record.py
|
|
============
|
|
Fast raw recorder for the Raspberry Pi HQ Camera.
|
|
|
|
This is the "record" half of the split workflow:
|
|
|
|
hq_record.py -> recordings/hq_YYYYmmdd_HHMMSS/
|
|
hq_spectrum.py --input REC_DIR -> replay / analysis
|
|
|
|
The recorder does not import Matplotlib and does not compute spectra. It only
|
|
pulls the Picamera2 RAW stream, queues each raw byte buffer, and writes it to
|
|
`frames.bin`. A background writer thread keeps disk I/O away from the camera
|
|
capture loop.
|
|
|
|
Recording layout
|
|
----------------
|
|
REC_DIR/
|
|
meta.json camera/format/stride/frame count
|
|
frames.bin raw byte buffers, one after another, no decoding
|
|
frames.jsonl one line of camera metadata per frame
|
|
|
|
For the usual Pi 5 HQ Camera configuration the raw stream is
|
|
`BGGR_PISP_COMP1` (PC1B). Each frame is height*stride bytes; at 2028x1520
|
|
that is typically about 3.1 MB/frame. At 15 fps that is about 47 MB/s, so a
|
|
USB SSD/NVMe is preferable to a slow SD card.
|
|
|
|
Examples
|
|
--------
|
|
# list modes first if needed
|
|
python3 hq_record.py --list-modes
|
|
|
|
# 10 seconds, 15 fps, manual exposure copied from rpicam-hello
|
|
python3 hq_record.py --duration 10 --fps 15 \
|
|
--exposure 33013 --gain 16 \
|
|
--record-dir recordings/hg_lamp
|
|
|
|
# unlimited until Ctrl+C
|
|
python3 hq_record.py --duration 0 --fps 15 \
|
|
--record-dir recordings/hg_long
|
|
|
|
# analyze later (on the Pi or on a PC)
|
|
python3 hq_spectrum.py --input recordings/hg_lamp --fps 15
|
|
|
|
The recorder is deliberately independent of the analyzer: a slow Matplotlib
|
|
redraw can no longer cause dropped frames or change the exposure timing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import queue
|
|
import shutil
|
|
import sys
|
|
import threading
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from typing import Optional
|
|
|
|
import numpy as np
|
|
|
|
try:
|
|
from hq_spectrum import CameraSource, list_camera_modes
|
|
except ImportError as exc: # pragma: no cover - import error path
|
|
raise SystemExit(
|
|
"hq_record.py must sit next to hq_spectrum.py "
|
|
"(run it from the hq_spectrum directory)."
|
|
) from exc
|
|
|
|
|
|
class RawWriter(threading.Thread):
|
|
"""Write queued raw buffers and metadata lines in the background."""
|
|
|
|
def __init__(self, frames_path: Path, metadata_path: Path, frame_bytes: int,
|
|
queue_size: int):
|
|
super().__init__(name="hq-raw-writer", daemon=False)
|
|
self.frames_path = frames_path
|
|
self.metadata_path = metadata_path
|
|
self.frame_bytes = frame_bytes
|
|
self.queue: queue.Queue = queue.Queue(maxsize=max(2, queue_size))
|
|
self.frames_written = 0
|
|
self.error: Optional[BaseException] = None
|
|
self.finished = threading.Event()
|
|
|
|
def put(self, raw: np.ndarray, metadata: dict, host_time_ns: int) -> None:
|
|
if self.error is not None:
|
|
raise RuntimeError(f"writer failed: {self.error}")
|
|
raw = np.ascontiguousarray(raw)
|
|
if raw.dtype != np.uint8 or raw.nbytes != self.frame_bytes:
|
|
raise ValueError(
|
|
f"unexpected raw buffer {raw.shape} {raw.dtype} {raw.nbytes}B; "
|
|
f"expected {self.frame_bytes}B uint8"
|
|
)
|
|
item = (raw, dict(metadata), int(host_time_ns))
|
|
while True:
|
|
try:
|
|
self.queue.put(item, timeout=0.5)
|
|
return
|
|
except queue.Full:
|
|
if self.error is not None:
|
|
raise RuntimeError(f"writer failed: {self.error}")
|
|
|
|
def run(self) -> None:
|
|
try:
|
|
with open(self.frames_path, "wb", buffering=4 * 1024 * 1024) as frames, \
|
|
open(self.metadata_path, "w", encoding="utf-8", buffering=1024 * 1024) as meta_out:
|
|
while True:
|
|
item = self.queue.get()
|
|
if item is None:
|
|
break
|
|
raw, camera_meta, host_time_ns = item
|
|
offset = self.frames_written * self.frame_bytes
|
|
record = dict(camera_meta)
|
|
record.update({
|
|
"i": self.frames_written,
|
|
"offset": offset,
|
|
"bytes": raw.nbytes,
|
|
"host_time_ns": host_time_ns,
|
|
})
|
|
frames.write(memoryview(raw).cast("B"))
|
|
meta_out.write(json.dumps(record, default=str, separators=(",", ":")) + "\n")
|
|
self.frames_written += 1
|
|
if self.frames_written % 30 == 0:
|
|
frames.flush()
|
|
meta_out.flush()
|
|
frames.flush()
|
|
meta_out.flush()
|
|
self.finished.set()
|
|
except BaseException as exc: # noqa: BLE001 - report writer failure to main
|
|
self.error = exc
|
|
self.finished.set()
|
|
|
|
def stop(self) -> None:
|
|
if self.is_alive():
|
|
try:
|
|
self.queue.put(None, timeout=2.0)
|
|
except queue.Full:
|
|
pass
|
|
self.join(timeout=30.0)
|
|
if self.is_alive():
|
|
raise RuntimeError("writer thread did not stop within 30 seconds")
|
|
if self.error is not None:
|
|
raise RuntimeError(f"writer failed: {self.error}")
|
|
|
|
|
|
def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Record Raspberry Pi HQ Camera RAW frames for later spectrum analysis.",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
parser.add_argument("--width", type=int, default=2028, help="raw stream width")
|
|
parser.add_argument("--height", type=int, default=1520, help="raw stream height")
|
|
parser.add_argument("--fps", type=float, default=15.0, help="requested frame rate")
|
|
parser.add_argument("--duration", type=float, default=10.0,
|
|
help="seconds to record; 0 means until Ctrl+C")
|
|
parser.add_argument("--frames", type=int, default=0,
|
|
help="optional frame limit; 0 means no limit")
|
|
parser.add_argument("--record-dir", default=None,
|
|
help="output directory (default recordings/hq_YYYYmmdd_HHMMSS)")
|
|
parser.add_argument("--queue-size", type=int, default=8,
|
|
help="raw frames buffered in RAM while the writer catches up")
|
|
parser.add_argument("--exposure", type=float, default=None,
|
|
help="manual exposure time in microseconds")
|
|
parser.add_argument("--gain", type=float, default=None, help="manual analogue gain")
|
|
parser.add_argument("--raw-format", default=None,
|
|
help="optional raw format, e.g. SBGGR12 or BGGR_PISP_COMP1")
|
|
parser.add_argument("--list-modes", action="store_true",
|
|
help="list camera raw modes and exit")
|
|
args = parser.parse_args(argv)
|
|
if args.width < 64 or args.height < 64:
|
|
parser.error("width and height must be at least 64")
|
|
if args.fps <= 0:
|
|
parser.error("--fps must be positive")
|
|
if args.duration < 0:
|
|
parser.error("--duration must be >= 0")
|
|
if args.frames < 0:
|
|
parser.error("--frames must be >= 0")
|
|
if args.queue_size < 2:
|
|
parser.error("--queue-size must be >= 2")
|
|
return args
|
|
|
|
|
|
def write_json(path: Path, data: dict) -> None:
|
|
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
with open(tmp, "w", encoding="utf-8") as handle:
|
|
json.dump(data, handle, indent=2, default=str)
|
|
handle.write("\n")
|
|
tmp.replace(path)
|
|
|
|
|
|
def prepare_directory(path: Path) -> None:
|
|
if path.exists():
|
|
if not path.is_dir():
|
|
raise RuntimeError(f"{path} exists and is not a directory")
|
|
if any(path.iterdir()):
|
|
raise RuntimeError(
|
|
f"{path} is not empty; choose a new --record-dir "
|
|
f"(the recorder never overwrites old data)"
|
|
)
|
|
else:
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def main(argv: Optional[list[str]] = None) -> int:
|
|
args = parse_args(argv)
|
|
if args.list_modes:
|
|
list_camera_modes()
|
|
return 0
|
|
|
|
cam_args = SimpleNamespace(
|
|
width=args.width,
|
|
height=args.height,
|
|
fps=args.fps,
|
|
exposure=args.exposure,
|
|
gain=args.gain,
|
|
raw_format=args.raw_format,
|
|
)
|
|
|
|
source = None
|
|
writer = None
|
|
meta = None
|
|
record_dir = None
|
|
enqueued = 0
|
|
started = None
|
|
exit_code = 0
|
|
|
|
try:
|
|
source = CameraSource(cam_args)
|
|
print(f"camera : {source.description}")
|
|
print(f"raw stream : {source.raw_format} "
|
|
f"{source.full_size[0]}x{source.full_size[1]}")
|
|
print(f"stride : {source.stride}")
|
|
print(f"Bayer : {source.pattern} decoded {source.bit_depth}-bit buffer")
|
|
print(f"target rate : {args.fps:g} fps")
|
|
if "PISP_COMP" in source.raw_format.upper():
|
|
print("note : recording the compressed PC1B buffer; no decode on the Pi")
|
|
|
|
source.start()
|
|
last_meta = {}
|
|
# Discard warm-up frames; the recording should start with settled AE.
|
|
for _ in range(5):
|
|
_, last_meta = source.read_raw()
|
|
|
|
# Get one sample only to validate the actual buffer size.
|
|
sample_raw, sample_meta = source.read_raw()
|
|
last_meta = sample_meta
|
|
if sample_raw.dtype != np.uint8:
|
|
raise RuntimeError(f"unexpected raw dtype {sample_raw.dtype}; expected uint8 bytes")
|
|
frame_bytes = int(sample_raw.nbytes)
|
|
expected_bytes = source.stride * source.full_size[1]
|
|
if frame_bytes != expected_bytes:
|
|
raise RuntimeError(
|
|
f"raw buffer is {frame_bytes} B but stride*height is {expected_bytes} B; "
|
|
f"refusing to write a recording that the analyzer would misread"
|
|
)
|
|
|
|
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
record_dir = Path(args.record_dir).expanduser() if args.record_dir else \
|
|
Path("recordings") / f"hq_{stamp}"
|
|
prepare_directory(record_dir)
|
|
frames_path = record_dir / "frames.bin"
|
|
metadata_path = record_dir / "frames.jsonl"
|
|
meta_path = record_dir / "meta.json"
|
|
|
|
data_rate = frame_bytes * args.fps
|
|
free = shutil.disk_usage(record_dir).free
|
|
print(f"output : {record_dir}")
|
|
print(f"frame size : {frame_bytes / 1e6:.3f} MB")
|
|
print(f"data rate : {data_rate / 1e6:.1f} MB/s")
|
|
print(f"free space : {free / 1e9:.2f} GB")
|
|
planned_frames = None
|
|
if args.frames:
|
|
planned_frames = args.frames
|
|
if args.duration:
|
|
duration_frames = int(args.duration * args.fps)
|
|
planned_frames = duration_frames if planned_frames is None else min(planned_frames, duration_frames)
|
|
if planned_frames is not None:
|
|
needed = frame_bytes * planned_frames * 1.05
|
|
print(f"planned size : {needed / 1e9:.2f} GB for ~{planned_frames} frames")
|
|
if needed > free:
|
|
raise RuntimeError(
|
|
f"not enough free space: need ~{needed / 1e9:.2f} GB, "
|
|
f"have {free / 1e9:.2f} GB"
|
|
)
|
|
|
|
meta = {
|
|
"format_version": 1,
|
|
"kind": "hq_raw_recording",
|
|
"created_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
"camera": source.description,
|
|
"raw_format": source.raw_format,
|
|
"width": source.full_size[0],
|
|
"height": source.full_size[1],
|
|
"stride": source.stride,
|
|
"frame_bytes": frame_bytes,
|
|
"storage_dtype": "uint8",
|
|
"pattern": source.pattern,
|
|
"bit_depth": source.bit_depth,
|
|
"fps_requested": args.fps,
|
|
"exposure_us": args.exposure,
|
|
"gain": args.gain,
|
|
"black_level": last_meta.get("BlackLevel"),
|
|
"duration_limit_s": args.duration,
|
|
"frame_limit": args.frames,
|
|
"frames": 0,
|
|
"completed": False,
|
|
}
|
|
write_json(meta_path, meta)
|
|
|
|
writer = RawWriter(frames_path, metadata_path, frame_bytes, args.queue_size)
|
|
writer.start()
|
|
# The first sample was taken to validate the format, so it is also the
|
|
# first frame of the recording.
|
|
writer.put(sample_raw, last_meta, time.time_ns())
|
|
enqueued = 1
|
|
|
|
started = time.monotonic()
|
|
last_report = started
|
|
while True:
|
|
if args.frames and enqueued >= args.frames:
|
|
break
|
|
elapsed = time.monotonic() - started
|
|
if args.duration and elapsed >= args.duration:
|
|
break
|
|
raw, camera_meta = source.read_raw()
|
|
writer.put(raw, camera_meta, time.time_ns())
|
|
enqueued += 1
|
|
now = time.monotonic()
|
|
if now - last_report >= 1.0:
|
|
written = writer.frames_written
|
|
rate = enqueued / max(now - started, 1e-6)
|
|
print(f"recording : {enqueued} frames captured, {written} written, "
|
|
f"{rate:.1f} fps")
|
|
last_report = now
|
|
|
|
elapsed = time.monotonic() - started
|
|
writer.stop()
|
|
writer_error = writer.error
|
|
writer = None
|
|
if writer_error is not None:
|
|
raise RuntimeError(f"writer failed: {writer_error}")
|
|
|
|
meta.update({
|
|
"frames": int(enqueued),
|
|
"duration_s": round(elapsed, 3),
|
|
"average_capture_fps": round(enqueued / max(elapsed, 1e-6), 3),
|
|
"completed": True,
|
|
"ended_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
"frames_file": "frames.bin",
|
|
"metadata_file": "frames.jsonl",
|
|
})
|
|
write_json(meta_path, meta)
|
|
size = frames_path.stat().st_size
|
|
print(f"done : {enqueued} frames, {size / 1e9:.2f} GB, "
|
|
f"{enqueued / max(elapsed, 1e-6):.1f} fps")
|
|
print(f"analyze with : python3 hq_spectrum.py --input \"{record_dir}\"")
|
|
return 0
|
|
except KeyboardInterrupt:
|
|
print("\ninterrupted; finalizing recording", file=sys.stderr)
|
|
exit_code = 130
|
|
return exit_code
|
|
except Exception as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
exit_code = 1
|
|
return exit_code
|
|
finally:
|
|
if source is not None:
|
|
try:
|
|
source.close()
|
|
except Exception:
|
|
pass
|
|
if writer is not None:
|
|
try:
|
|
writer.stop()
|
|
except Exception as exc:
|
|
print(f"writer shutdown error: {exc}", file=sys.stderr)
|
|
exit_code = 1
|
|
if meta is not None and record_dir is not None:
|
|
try:
|
|
meta.update({
|
|
"frames": int(writer.frames_written) if writer is not None else int(enqueued),
|
|
"completed": exit_code == 0 and meta.get("completed", False),
|
|
"ended_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
})
|
|
write_json(record_dir / "meta.json", meta)
|
|
except Exception as exc:
|
|
print(f"warning: could not finalize meta.json: {exc}", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|