#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ hq_record.py ============ Fast raw recorder for the Raspberry Pi HQ Camera. This is the "record" half of the split workflow: hq_record.py -> recordings/hq_YYYYmmdd_HHMMSS/ hq_spectrum.py --input REC_DIR -> replay / analysis The recorder does not import Matplotlib and does not compute spectra. It only pulls the Picamera2 RAW stream, queues each raw byte buffer, and writes it to `frames.bin`. A background writer thread keeps disk I/O away from the camera capture loop. Recording layout ---------------- REC_DIR/ meta.json camera/format/stride/frame count frames.bin raw byte buffers, one after another, no decoding frames.jsonl one line of camera metadata per frame For the usual Pi 5 HQ Camera configuration the raw stream is `BGGR_PISP_COMP1` (PC1B). Each frame is height*stride bytes; at 2028x1520 that is typically about 3.1 MB/frame. At 15 fps that is about 47 MB/s, so a USB SSD/NVMe is preferable to a slow SD card. Examples -------- # list modes first if needed python3 hq_record.py --list-modes # 10 seconds, 15 fps, manual exposure copied from rpicam-hello python3 hq_record.py --duration 10 --fps 15 \ --exposure 33013 --gain 16 \ --record-dir recordings/hg_lamp # unlimited until Ctrl+C python3 hq_record.py --duration 0 --fps 15 \ --record-dir recordings/hg_long # analyze later (on the Pi or on a PC) python3 hq_spectrum.py --input recordings/hg_lamp --fps 15 The recorder is deliberately independent of the analyzer: a slow Matplotlib redraw can no longer cause dropped frames or change the exposure timing. """ from __future__ import annotations import argparse import json import queue import shutil import sys import threading import time from datetime import datetime from pathlib import Path from types import SimpleNamespace from typing import Optional import numpy as np try: from hq_spectrum import CameraSource, list_camera_modes except ImportError as exc: # pragma: no cover - import error path raise SystemExit( "hq_record.py must sit next to hq_spectrum.py " "(run it from the hq_spectrum directory)." ) from exc class RawWriter(threading.Thread): """Write queued raw buffers and metadata lines in the background.""" def __init__(self, frames_path: Path, metadata_path: Path, frame_bytes: int, queue_size: int): super().__init__(name="hq-raw-writer", daemon=False) self.frames_path = frames_path self.metadata_path = metadata_path self.frame_bytes = frame_bytes self.queue: queue.Queue = queue.Queue(maxsize=max(2, queue_size)) self.frames_written = 0 self.error: Optional[BaseException] = None self.finished = threading.Event() def put(self, raw: np.ndarray, metadata: dict, host_time_ns: int) -> None: if self.error is not None: raise RuntimeError(f"writer failed: {self.error}") raw = np.ascontiguousarray(raw) if raw.dtype != np.uint8 or raw.nbytes != self.frame_bytes: raise ValueError( f"unexpected raw buffer {raw.shape} {raw.dtype} {raw.nbytes}B; " f"expected {self.frame_bytes}B uint8" ) item = (raw, dict(metadata), int(host_time_ns)) while True: try: self.queue.put(item, timeout=0.5) return except queue.Full: if self.error is not None: raise RuntimeError(f"writer failed: {self.error}") def run(self) -> None: try: with open(self.frames_path, "wb", buffering=4 * 1024 * 1024) as frames, \ open(self.metadata_path, "w", encoding="utf-8", buffering=1024 * 1024) as meta_out: while True: item = self.queue.get() if item is None: break raw, camera_meta, host_time_ns = item offset = self.frames_written * self.frame_bytes record = dict(camera_meta) record.update({ "i": self.frames_written, "offset": offset, "bytes": raw.nbytes, "host_time_ns": host_time_ns, }) frames.write(memoryview(raw).cast("B")) meta_out.write(json.dumps(record, default=str, separators=(",", ":")) + "\n") self.frames_written += 1 if self.frames_written % 30 == 0: frames.flush() meta_out.flush() frames.flush() meta_out.flush() self.finished.set() except BaseException as exc: # noqa: BLE001 - report writer failure to main self.error = exc self.finished.set() def stop(self) -> None: if self.is_alive(): try: self.queue.put(None, timeout=2.0) except queue.Full: pass self.join(timeout=30.0) if self.is_alive(): raise RuntimeError("writer thread did not stop within 30 seconds") if self.error is not None: raise RuntimeError(f"writer failed: {self.error}") def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description="Record Raspberry Pi HQ Camera RAW frames for later spectrum analysis.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument("--width", type=int, default=2028, help="raw stream width") parser.add_argument("--height", type=int, default=1520, help="raw stream height") parser.add_argument("--fps", type=float, default=15.0, help="requested frame rate") parser.add_argument("--duration", type=float, default=10.0, help="seconds to record; 0 means until Ctrl+C") parser.add_argument("--frames", type=int, default=0, help="optional frame limit; 0 means no limit") parser.add_argument("--record-dir", default=None, help="output directory (default recordings/hq_YYYYmmdd_HHMMSS)") parser.add_argument("--queue-size", type=int, default=8, help="raw frames buffered in RAM while the writer catches up") parser.add_argument("--exposure", type=float, default=None, help="manual exposure time in microseconds") parser.add_argument("--gain", type=float, default=None, help="manual analogue gain") parser.add_argument("--raw-format", default=None, help="optional raw format, e.g. SBGGR12 or BGGR_PISP_COMP1") parser.add_argument("--list-modes", action="store_true", help="list camera raw modes and exit") args = parser.parse_args(argv) if args.width < 64 or args.height < 64: parser.error("width and height must be at least 64") if args.fps <= 0: parser.error("--fps must be positive") if args.duration < 0: parser.error("--duration must be >= 0") if args.frames < 0: parser.error("--frames must be >= 0") if args.queue_size < 2: parser.error("--queue-size must be >= 2") return args def write_json(path: Path, data: dict) -> None: tmp = path.with_suffix(path.suffix + ".tmp") with open(tmp, "w", encoding="utf-8") as handle: json.dump(data, handle, indent=2, default=str) handle.write("\n") tmp.replace(path) def prepare_directory(path: Path) -> None: if path.exists(): if not path.is_dir(): raise RuntimeError(f"{path} exists and is not a directory") if any(path.iterdir()): raise RuntimeError( f"{path} is not empty; choose a new --record-dir " f"(the recorder never overwrites old data)" ) else: path.mkdir(parents=True, exist_ok=True) def main(argv: Optional[list[str]] = None) -> int: args = parse_args(argv) if args.list_modes: list_camera_modes() return 0 cam_args = SimpleNamespace( width=args.width, height=args.height, fps=args.fps, exposure=args.exposure, gain=args.gain, raw_format=args.raw_format, ) source = None writer = None meta = None record_dir = None enqueued = 0 started = None exit_code = 0 try: source = CameraSource(cam_args) print(f"camera : {source.description}") print(f"raw stream : {source.raw_format} " f"{source.full_size[0]}x{source.full_size[1]}") print(f"stride : {source.stride}") print(f"Bayer : {source.pattern} decoded {source.bit_depth}-bit buffer") print(f"target rate : {args.fps:g} fps") if "PISP_COMP" in source.raw_format.upper(): print("note : recording the compressed PC1B buffer; no decode on the Pi") source.start() last_meta = {} # Discard warm-up frames; the recording should start with settled AE. for _ in range(5): _, last_meta = source.read_raw() # Get one sample only to validate the actual buffer size. sample_raw, sample_meta = source.read_raw() last_meta = sample_meta if sample_raw.dtype != np.uint8: raise RuntimeError(f"unexpected raw dtype {sample_raw.dtype}; expected uint8 bytes") frame_bytes = int(sample_raw.nbytes) expected_bytes = source.stride * source.full_size[1] if frame_bytes != expected_bytes: raise RuntimeError( f"raw buffer is {frame_bytes} B but stride*height is {expected_bytes} B; " f"refusing to write a recording that the analyzer would misread" ) stamp = datetime.now().strftime("%Y%m%d_%H%M%S") record_dir = Path(args.record_dir).expanduser() if args.record_dir else \ Path("recordings") / f"hq_{stamp}" prepare_directory(record_dir) frames_path = record_dir / "frames.bin" metadata_path = record_dir / "frames.jsonl" meta_path = record_dir / "meta.json" data_rate = frame_bytes * args.fps free = shutil.disk_usage(record_dir).free print(f"output : {record_dir}") print(f"frame size : {frame_bytes / 1e6:.3f} MB") print(f"data rate : {data_rate / 1e6:.1f} MB/s") print(f"free space : {free / 1e9:.2f} GB") planned_frames = None if args.frames: planned_frames = args.frames if args.duration: duration_frames = int(args.duration * args.fps) planned_frames = duration_frames if planned_frames is None else min(planned_frames, duration_frames) if planned_frames is not None: needed = frame_bytes * planned_frames * 1.05 print(f"planned size : {needed / 1e9:.2f} GB for ~{planned_frames} frames") if needed > free: raise RuntimeError( f"not enough free space: need ~{needed / 1e9:.2f} GB, " f"have {free / 1e9:.2f} GB" ) meta = { "format_version": 1, "kind": "hq_raw_recording", "created_at": datetime.now().astimezone().isoformat(timespec="seconds"), "camera": source.description, "raw_format": source.raw_format, "width": source.full_size[0], "height": source.full_size[1], "stride": source.stride, "frame_bytes": frame_bytes, "storage_dtype": "uint8", "pattern": source.pattern, "bit_depth": source.bit_depth, "fps_requested": args.fps, "exposure_us": args.exposure, "gain": args.gain, "black_level": last_meta.get("BlackLevel"), "duration_limit_s": args.duration, "frame_limit": args.frames, "frames": 0, "completed": False, } write_json(meta_path, meta) writer = RawWriter(frames_path, metadata_path, frame_bytes, args.queue_size) writer.start() # The first sample was taken to validate the format, so it is also the # first frame of the recording. writer.put(sample_raw, last_meta, time.time_ns()) enqueued = 1 started = time.monotonic() last_report = started while True: if args.frames and enqueued >= args.frames: break elapsed = time.monotonic() - started if args.duration and elapsed >= args.duration: break raw, camera_meta = source.read_raw() writer.put(raw, camera_meta, time.time_ns()) enqueued += 1 now = time.monotonic() if now - last_report >= 1.0: written = writer.frames_written rate = enqueued / max(now - started, 1e-6) print(f"recording : {enqueued} frames captured, {written} written, " f"{rate:.1f} fps") last_report = now elapsed = time.monotonic() - started writer.stop() writer_error = writer.error writer = None if writer_error is not None: raise RuntimeError(f"writer failed: {writer_error}") meta.update({ "frames": int(enqueued), "duration_s": round(elapsed, 3), "average_capture_fps": round(enqueued / max(elapsed, 1e-6), 3), "completed": True, "ended_at": datetime.now().astimezone().isoformat(timespec="seconds"), "frames_file": "frames.bin", "metadata_file": "frames.jsonl", }) write_json(meta_path, meta) size = frames_path.stat().st_size print(f"done : {enqueued} frames, {size / 1e9:.2f} GB, " f"{enqueued / max(elapsed, 1e-6):.1f} fps") print(f"analyze with : python3 hq_spectrum.py --input \"{record_dir}\"") return 0 except KeyboardInterrupt: print("\ninterrupted; finalizing recording", file=sys.stderr) exit_code = 130 return exit_code except Exception as exc: print(f"error: {exc}", file=sys.stderr) exit_code = 1 return exit_code finally: if source is not None: try: source.close() except Exception: pass if writer is not None: try: writer.stop() except Exception as exc: print(f"writer shutdown error: {exc}", file=sys.stderr) exit_code = 1 if meta is not None and record_dir is not None: try: meta.update({ "frames": int(writer.frames_written) if writer is not None else int(enqueued), "completed": exit_code == 0 and meta.get("completed", False), "ended_at": datetime.now().astimezone().isoformat(timespec="seconds"), }) write_json(record_dir / "meta.json", meta) except Exception as exc: print(f"warning: could not finalize meta.json: {exc}", file=sys.stderr) if __name__ == "__main__": raise SystemExit(main())