diff --git a/Python/OAK/datasets/oak-fcc-3/_0_capture.py b/Python/OAK/datasets/oak-fcc-3/_0_capture.py index ae9c5ce21..534d24fa9 100644 --- a/Python/OAK/datasets/oak-fcc-3/_0_capture.py +++ b/Python/OAK/datasets/oak-fcc-3/_0_capture.py @@ -126,6 +126,28 @@ def get_camera_map_from_status(status: dict) -> dict: return result +def build_preview_to_save(cam, frame_type_save, last_preview_bgr, last_packed_raw_by_camera, last_meta_stream, raw_w, raw_h, bayer): + preview_to_save = last_preview_bgr + method = "last_screen_preview" + + if frame_type_save == "RAW_BRUTO" and last_packed_raw_by_camera is not None: + rebuilt_preview = cam.build_save_preview_from_cam_a( + packed_raw_by_camera=last_packed_raw_by_camera, + meta_stream=last_meta_stream, + sensor_width=raw_w, + sensor_height=raw_h, + bayer_pattern=bayer, + ) + + if rebuilt_preview is not None: + preview_to_save = rebuilt_preview + method = "cam_a_reconstructed_raw10" + else: + method = "last_screen_preview_fallback" + + return preview_to_save, method + + # ========================= # MAIN # ========================= @@ -234,7 +256,6 @@ def main(): packed_by_camera = frame preview_bgr, raw3_preview, preview_source_id = cam.build_preview_from_raw_payload(frame=frame, meta=meta) - preview_bgr = cv2.cvtColor(preview_bgr, cv2.COLOR_RGB2BGR) if beauty_preview: rgb_preview = None @@ -348,13 +369,31 @@ def main(): else: line_rad = "RAD | OFF" + camera_info = meta.get("camera_info", {}) or {} + frame_controls = meta.get("frame_controls", {}) or {} + + role_controls = {} + for cam_id, info in camera_info.items(): + role = info.get("role", cam_id) + role_controls[role] = frame_controls.get(cam_id, {}) + + line_ae = ( + f"AE_REAL | " + f"RGB exp={role_controls.get('rgb', {}).get('exposure_time_us')} " + f"iso={role_controls.get('rgb', {}).get('sensitivity_iso')} | " + f"RE exp={role_controls.get('re', {}).get('exposure_time_us')} " + f"iso={role_controls.get('re', {}).get('sensitivity_iso')} | " + f"NIR exp={role_controls.get('nir', {}).get('exposure_time_us')} " + f"iso={role_controls.get('nir', {}).get('sensitivity_iso')}" + ) lines = [ f"CANA: {args.cana} | HORA: {args.horario} | Pasta: {os.path.basename(session_dir)}", f"Type={meta.get('frame_type')} | CaptureMode={effective_capture_mode} | RAW policy={args.raw_policy}", f"Sources={active_sources} | FPS_STREAM={fps_stream:.1f} | FPS_VIEW={fps_view:.1f}", f"frame_id={meta.get('frame_id')} | layout={meta.get('output_layout')} | dtype={meta.get('dtype') or meta.get('output_dtype')}", f"codec={meta.get('codec_name', meta.get('codec_family', '-'))} | comp={meta.get('dt_comp', 0):.4f}s | send={meta.get('dt_send_payload_prev', 0):.4f}s", - f"CAM_PARAMS={os.path.basename(args.module_calibration_json)} | controles fixos aplicados", + f"CAM_PARAMS={os.path.basename(args.module_calibration_json)}", + line_ae, line_rad, "Keys: C/SPACE=save | A=auto-save | M=preview | R=rad | Q/Esc=quit" ] @@ -410,10 +449,23 @@ def main(): "raw_preview_reference_camera": preview_source_id, } + preview_to_save, preview_method = build_preview_to_save( + cam=cam, + frame_type_save=frame_type_save, + last_preview_bgr=last_preview_bgr, + last_packed_raw_by_camera=last_packed_raw_by_camera, + last_meta_stream=last_meta_stream, + raw_w=raw_w, + raw_h=raw_h, + bayer=args.bayer, + ) + + meta_save["saved_preview_method"] = preview_method + save_sample( session_dir, frame_type=frame_type_save, - preview_bgr=last_preview_bgr, + preview_bgr=preview_to_save, meta=meta_save, raw_payload=last_payload_float, packed_raw=last_packed_raw, @@ -472,10 +524,23 @@ def main(): "raw_preview_reference_camera": preview_source_id, } + preview_to_save, preview_method = build_preview_to_save( + cam=cam, + frame_type_save=frame_type_save, + last_preview_bgr=last_preview_bgr, + last_packed_raw_by_camera=last_packed_raw_by_camera, + last_meta_stream=last_meta_stream, + raw_w=raw_w, + raw_h=raw_h, + bayer=args.bayer, + ) + + meta_save["saved_preview_method"] = preview_method + save_sample( session_dir, frame_type=frame_type_save, - preview_bgr=last_preview_bgr, + preview_bgr=preview_to_save, meta=meta_save, raw_payload=last_payload_float, packed_raw=last_packed_raw, diff --git a/Python/OAK/datasets/oak-fcc-3/calibration/module_params.json b/Python/OAK/datasets/oak-fcc-3/calibration/module_params.json index 9229c3548..6eb156d22 100644 --- a/Python/OAK/datasets/oak-fcc-3/calibration/module_params.json +++ b/Python/OAK/datasets/oak-fcc-3/calibration/module_params.json @@ -13,8 +13,8 @@ }, "camera_settings": { "rgb": { - "ae_enable": false, - "awb_enable": false, + "ae_enable": true, + "awb_enable": true, "exposure_time_us": 2000, "analogue_gain": 1.0, "colour_gains": [ @@ -23,14 +23,14 @@ ] }, "re": { - "ae_enable": false, + "ae_enable": true, "awb_enable": false, "exposure_time_us": 5000, "analogue_gain": 1.0, "colour_gains": null }, "nir": { - "ae_enable": false, + "ae_enable": true, "awb_enable": false, "exposure_time_us": 5000, "analogue_gain": 1.0, @@ -94,7 +94,7 @@ "target_size": null }, "radiometric_config": { - "enabled": true, + "enabled": false, "interval_s": 0.25, "verbose": true, "metering_mode": "reference_patches", @@ -570,27 +570,58 @@ "patch_white_roi_reject_p95": 0.995 }, "radiometric_normalization": { - "enabled": false, - "method": "exposure_gain_reference", + "enabled": true, + "method": "oak_ae_frame_controls_v1", "apply_stage": "after_dark_before_flat_gain", + + "control_source": "stream_meta.frame_controls", + "role_mapping_source": "camera_info", + + "factor_model": "exposure_time_us_x_iso", + "iso_base": 100.0, + + "reference_mode": "fixed", "reference_controls": { "rgb": { - "exposure_time_us": 3000, - "analogue_gain": 1.0 + "exposure_time_us": 10000, + "sensitivity_iso": 400 }, "re": { - "exposure_time_us": 7000, - "analogue_gain": 1.0 + "exposure_time_us": 15000, + "sensitivity_iso": 400 }, "nir": { - "exposure_time_us": 7000, - "analogue_gain": 1.0 + "exposure_time_us": 15000, + "sensitivity_iso": 400 } }, - "clip_output": true + + "scale_limits": { + "default": { + "min": 0.15, + "max": 6.0 + }, + "rgb": { + "min": 0.15, + "max": 6.0 + }, + "re": { + "min": 0.15, + "max": 8.0 + }, + "nir": { + "min": 0.15, + "max": 8.0 + } + }, + + "missing_controls_policy": "skip", + "invalid_controls_policy": "skip", + "clip_output": false, + "save_debug": true }, "patch_normalization": { - "enabled": true, + "enabled": false, "apply_when_metering_mode": "reference_patches", "apply_stage": "after_fusion", "method": "gray_scale_with_white_guard", diff --git a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py index aeae9143a..018926041 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py +++ b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py @@ -255,6 +255,12 @@ class OakFcc3Client: except Exception: return None + def get_last_radiometric_normalization_result(self): + try: + return self.core.get_last_radiometric_normalization_result() + except Exception: + return None + def build_infer_tensor(self, frame, meta, channels_expected, target_size=None): return self.core.build_infer_tensor_from_stream( frame, @@ -405,6 +411,96 @@ class OakFcc3Client: return previews + def build_save_preview_from_cam_a( + self, + packed_raw_by_camera: dict | None, + meta_stream: dict, + sensor_width: int, + sensor_height: int, + bayer_pattern: str, + ) -> np.ndarray | None: + """ + Gera o preview salvo no mesmo padrão do 'CAM_A reconstruido'. + + Usa apenas CAM_A do RAW_BRUTO: + CAM_A packed RAW10 + -> unpack_raw10_packed + -> RawProcessorPreview.raw16_to_preview_bgr + + Retorna BGR uint8 pronto para cv2.imwrite. + """ + if not packed_raw_by_camera or "CAM_A" not in packed_raw_by_camera: + return None + + stream_meta = meta_stream or {} + + camera_info = stream_meta.get("camera_info", {}) or {} + cam_meta = camera_info.get("CAM_A", {}) or {} + + bit_depth = int(cam_meta.get("bit_depth", 10)) + + bayer = ( + cam_meta.get("bayer_pattern") + or cam_meta.get("bayer") + or stream_meta.get("bayer_pattern") + or bayer_pattern + or "RGGB" + ) + bayer = str(bayer).upper() + + arr = packed_raw_by_camera["CAM_A"] + + if arr is None: + return None + + packed = arr + if packed.ndim == 3 and packed.shape[2] == 1: + packed = packed[:, :, 0] + + if packed.ndim != 2: + return None + + # Mantém a mesma lógica do validador: + # RAW10 packed => sensor_w = packed_w * 4 // 5 + packed_h, packed_w = packed.shape[:2] + + if bit_depth == 10: + real_w = int(cam_meta.get("width", sensor_width)) + real_h = int(cam_meta.get("height", sensor_height)) + + # Fallback caso o meta não tenha width/height confiáveis + if real_w <= 0 or real_h <= 0: + real_w = int((packed_w * 4) // 5) + real_h = int(packed_h) + else: + real_w = int(cam_meta.get("width", sensor_width)) + real_h = int(cam_meta.get("height", sensor_height)) + + core = RawProcessorCore( + sensor_width=real_w, + sensor_height=real_h, + bayer_pattern=bayer, + ) + + preview = RawProcessorPreview( + sensor_width=real_w, + sensor_height=real_h, + bayer_pattern=bayer, + ) + + raw16 = core.unpack_raw10_packed( + packed, + sensor_width=real_w, + sensor_height=real_h, + ) + + preview_bgr = preview.raw16_to_preview_bgr( + raw16, + bit_depth=bit_depth, + ) + + return preview_bgr + def _find_decoded_by_role(self, decoded, role): role = str(role).lower() diff --git a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py index 84f4a7f99..3879e0e4d 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py +++ b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py @@ -218,10 +218,16 @@ class OakFcc3Manager: synced = self._try_get_synced_packet() if synced is not None: - frames, timestamps, sync_dt_ms, sync_ok = synced + frames, timestamps, sync_dt_ms, sync_ok, frame_controls = synced self.frame_id += 1 - meta = self._build_meta(frames, timestamps, sync_dt_ms, sync_ok) + meta = self._build_meta( + frames, + timestamps, + sync_dt_ms, + sync_ok, + frame_controls=frame_controls, + ) return frames, meta @@ -233,6 +239,69 @@ class OakFcc3Manager: f"Tente aumentar para 25 ou 35 ms para diagnóstico." ) + def _extract_frame_controls(self, msg): + controls = { + "exposure_time_us": None, + "sensitivity_iso": None, + "analogue_gain_est": None, + "color_temperature_k": None, + "lens_position": None, + "sequence_num": None, + "errors": [], + } + + try: + if hasattr(msg, "getSequenceNum"): + controls["sequence_num"] = int(msg.getSequenceNum()) + except Exception as e: + controls["errors"].append(f"sequence_num:{type(e).__name__}:{e}") + + try: + if hasattr(msg, "getExposureTime"): + exp = msg.getExposureTime() + + if hasattr(exp, "total_seconds"): + controls["exposure_time_us"] = int(exp.total_seconds() * 1_000_000) + else: + controls["exposure_time_us"] = int(exp) + else: + controls["errors"].append("missing:getExposureTime") + except Exception as e: + controls["errors"].append(f"exposure:{type(e).__name__}:{e}") + + try: + if hasattr(msg, "getSensitivity"): + iso = msg.getSensitivity() + controls["sensitivity_iso"] = int(iso) + controls["analogue_gain_est"] = float(iso) / 100.0 + else: + controls["errors"].append("missing:getSensitivity") + except Exception as e: + controls["errors"].append(f"sensitivity:{type(e).__name__}:{e}") + + try: + if hasattr(msg, "getColorTemperature"): + ct = int(msg.getColorTemperature()) + controls["color_temperature_k"] = ct if ct > 0 else None + else: + controls["errors"].append("missing:getColorTemperature") + except Exception as e: + controls["errors"].append(f"color_temperature:{type(e).__name__}:{e}") + + try: + if hasattr(msg, "getLensPosition"): + lp = int(msg.getLensPosition()) + controls["lens_position"] = lp + else: + controls["errors"].append("missing:getLensPosition") + except Exception as e: + controls["errors"].append(f"lens_position:{type(e).__name__}:{e}") + + if not controls["errors"]: + controls.pop("errors", None) + + return controls + def _drain_queues_to_buffers(self): for cam_id, q in self.queues.items(): while q.has(): @@ -270,9 +339,12 @@ class OakFcc3Manager: "packed_width": stride, } + frame_controls = self._extract_frame_controls(msg) + self.buffers[cam_id].append({ "frame": frame, "timestamp": ts, + "controls": frame_controls, }) def _try_get_synced_packet(self): @@ -309,6 +381,11 @@ class OakFcc3Manager: for cam_id, item in selected.items() } + frame_controls = { + cam_id: item.get("controls", {}) + for cam_id, item in selected.items() + } + ts_values = list(timestamps.values()) sync_dt_ms = (max(ts_values) - min(ts_values)) * 1000.0 if len(ts_values) >= 2 else 0.0 sync_ok = sync_dt_ms <= self.sync_tolerance_ms @@ -331,7 +408,7 @@ class OakFcc3Manager: if item is used_item: break - return frames, timestamps, sync_dt_ms, sync_ok + return frames, timestamps, sync_dt_ms, sync_ok, frame_controls def _get_required_cam_ids(self): available = self._get_available_cam_ids_ordered() @@ -352,7 +429,7 @@ class OakFcc3Manager: return available - def _build_meta(self, frames, timestamps, sync_dt_ms, sync_ok): + def _build_meta(self, frames, timestamps, sync_dt_ms, sync_ok, frame_controls): payload_sources = list(frames.keys()) shapes = { @@ -407,6 +484,7 @@ class OakFcc3Manager: "payload_sources": payload_sources, "camera_info": camera_info, "timestamps": timestamps, + "frame_controls": frame_controls or {}, "sync_dt_ms": sync_dt_ms, "sync_ok": sync_ok, "sync_tolerance_ms": self.sync_tolerance_ms, @@ -538,17 +616,34 @@ class OakFcc3Manager: result = dict(self.camera_controls.get(cam_id, {})) + ae_requested = controls.get("ae_enable", None) + if "ae_enable" in controls: result = self.set_ae_enable(cam_id, bool(controls["ae_enable"])) if "awb_enable" in controls: result = self.set_awb_enable(cam_id, bool(controls["awb_enable"])) - if "exposure_time_us" in controls and controls["exposure_time_us"] is not None: - result = self.set_exposure_time(cam_id, int(controls["exposure_time_us"])) + # Se AE está ligado, NÃO aplicar exposição/ganho manual. + # exposure_time_us e analogue_gain ficam apenas como referência/snapshot. + ae_is_on = bool(self.camera_controls[cam_id].get("ae_enable", False)) - if "analogue_gain" in controls and controls["analogue_gain"] is not None: - result = self.set_analogue_gain(cam_id, float(controls["analogue_gain"])) + if not ae_is_on: + if "exposure_time_us" in controls and controls["exposure_time_us"] is not None: + result = self.set_exposure_time(cam_id, int(controls["exposure_time_us"])) + + if "analogue_gain" in controls and controls["analogue_gain"] is not None: + result = self.set_analogue_gain(cam_id, float(controls["analogue_gain"])) + + else: + # Mantém os valores no estado interno só como referência, sem mandar manual exposure. + if "exposure_time_us" in controls and controls["exposure_time_us"] is not None: + self.camera_controls[cam_id]["exposure_time_us"] = int(controls["exposure_time_us"]) + + if "analogue_gain" in controls and controls["analogue_gain"] is not None: + self.camera_controls[cam_id]["analogue_gain"] = float(controls["analogue_gain"]) + + result = dict(self.camera_controls[cam_id]) return result diff --git a/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py b/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py index b58fbf633..8814d1ec5 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py +++ b/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py @@ -60,10 +60,57 @@ class RawProcessorCore: self.radiometric_normalization_config = { "enabled": False, - "method": "exposure_gain_reference", - "reference_controls": {}, + "method": "oak_ae_frame_controls_v1", + "apply_stage": "after_dark_before_flat_gain", + + "control_source": "stream_meta.frame_controls", + "role_mapping_source": "camera_info", + + "factor_model": "exposure_time_us_x_iso", + "iso_base": 100.0, + + "reference_mode": "fixed", + "reference_controls": { + "rgb": { + "exposure_time_us": 10000, + "sensitivity_iso": 400, + }, + "re": { + "exposure_time_us": 15000, + "sensitivity_iso": 400, + }, + "nir": { + "exposure_time_us": 15000, + "sensitivity_iso": 400, + }, + }, + + "scale_limits": { + "default": { + "min": 0.15, + "max": 6.0, + }, + "rgb": { + "min": 0.15, + "max": 6.0, + }, + "re": { + "min": 0.15, + "max": 8.0, + }, + "nir": { + "min": 0.15, + "max": 8.0, + }, + }, + + "missing_controls_policy": "skip", + "invalid_controls_policy": "skip", "clip_output": False, + "save_debug": True, } + self.last_radiometric_normalization_result = None + self.radiometric_config = {} self.patch_normalization_config = { "enabled": False, @@ -377,9 +424,18 @@ class RawProcessorCore: raise RuntimeError("RGB obrigatório") channel_names = self._channel_names_from_decoded(decoded) - tensor = self.fuse_multispec_cameras(decoded, meta=None, channels_expected=len(channel_names)) + + tensor = self.fuse_multispec_cameras( + decoded, + meta=None, + channels_expected=len(channel_names), + ) + tensor = self.resize_tensor_chw(tensor, target_size=target_size) - tensor = self.apply_patch_normalization_to_tensor(tensor) + + if bool((self.patch_normalization_config or {}).get("enabled", False)): + tensor = self.apply_patch_normalization_to_tensor(tensor) + self.last_frame_quality_result = self.evaluate_frame_quality(tensor) return tensor, channel_names @@ -391,7 +447,13 @@ class RawProcessorCore: decoded = self.decode_stream_cameras(frame, meta) tensor = self.fuse_multispec_cameras(decoded, meta, channels_expected) tensor = self.resize_tensor_chw(tensor, target_size=target_size) - tensor = self.apply_patch_normalization_to_tensor(tensor) + + # Sem cartões neste modo novo. + # patch_normalization deve ficar desligado no JSON. + # Se quiser manter compatibilidade futura, deixe gateado: + if bool((self.patch_normalization_config or {}).get("enabled", False)): + tensor = self.apply_patch_normalization_to_tensor(tensor) + self.last_frame_quality_result = self.evaluate_frame_quality(tensor) return tensor @@ -402,6 +464,8 @@ class RawProcessorCore: if frame.ndim != 3: raise RuntimeError(f"Frame {frame_type} inválido: shape={frame.shape}") + dtype_str = str(meta.get("dtype") or meta.get("output_dtype") or "float32").lower() + if dtype_str == "uint8": raw_np = frame.astype(np.float32) / 255.0 elif dtype_str == "float32": @@ -412,10 +476,12 @@ class RawProcessorCore: raise RuntimeError(f"dtype {frame_type} não suportado: {dtype_str}") if raw_np.shape[0] != channels_expected: - raise RuntimeError(f"Frame {frame_type} com canais inesperados: {raw_np.shape[0]} | esperado={channels_expected}") - tensor = raw_np + raise RuntimeError( + f"Frame {frame_type} com canais inesperados: " + f"{raw_np.shape[0]} | esperado={channels_expected}" + ) - tensor = self.resize_tensor_chw(tensor, target_size=target_size) + tensor = self.resize_tensor_chw(raw_np, target_size=target_size) self.last_frame_quality_result = self.evaluate_frame_quality(tensor) return tensor @@ -2095,16 +2161,38 @@ class RawProcessorCore: def normalize_decoded_by_capture_controls(self, decoded: dict, meta: dict | None = None) -> dict: cfg = self.radiometric_normalization_config or {} + self.last_radiometric_normalization_result = None + + result = { + "enabled": bool(cfg.get("enabled", False)), + "applied": False, + "method": cfg.get("method", "oak_ae_frame_controls_v1"), + "warnings": [], + "by_role": {}, + "by_camera": {}, + "summary": {}, + } if not cfg.get("enabled", False): + result["warnings"].append("radiometric_normalization_disabled") + self.last_radiometric_normalization_result = result return decoded - method = str(cfg.get("method", "exposure_gain_reference")).lower() - if method != "exposure_gain_reference": + method = str(cfg.get("method", "oak_ae_frame_controls_v1")).lower() + if method not in ("oak_ae_frame_controls_v1", "exposure_iso_reference"): + result["warnings"].append(f"unsupported_method:{method}") + self.last_radiometric_normalization_result = result return decoded - controls = self._extract_actual_controls_from_meta(meta) - if not controls: + controls_by_role, controls_by_cam = self._extract_frame_controls_from_meta_by_role(meta) + + if not controls_by_role: + result["warnings"].append("missing_frame_controls") + self.last_radiometric_normalization_result = result + + if str(cfg.get("missing_controls_policy", "skip")).lower() == "raise": + raise RuntimeError("radiometric_normalization ativo, mas meta.frame_controls ausente.") + return decoded reference_controls = cfg.get("reference_controls", {}) or {} @@ -2118,47 +2206,225 @@ class RawProcessorCore: if img is None or not role: normalized[cam_id] = item + result["warnings"].append(f"{cam_id}:missing_image_or_role") continue - actual_ctrl = controls.get(role, {}) or {} + actual_ctrl = controls_by_role.get(role, {}) or {} + ref_ctrl = reference_controls.get(role, {}) or {} - ref_ctrl = ( - reference_controls.get(role) - or self.camera_settings.get(role) - or actual_ctrl - or {} - ) - - actual_factor = self._exposure_gain_factor(actual_ctrl) - ref_factor = self._exposure_gain_factor(ref_ctrl) + actual_factor = self._radiometric_factor_from_controls(actual_ctrl, cfg) + ref_factor = self._radiometric_factor_from_controls(ref_ctrl, cfg) if actual_factor <= 0 or ref_factor <= 0: normalized[cam_id] = item + result["warnings"].append( + f"{role}:invalid_factor actual={actual_factor:.6g} ref={ref_factor:.6g}" + ) + + if str(cfg.get("invalid_controls_policy", "skip")).lower() == "raise": + raise RuntimeError(f"Controles radiométricos inválidos para role={role}: {actual_ctrl}") + continue - scale = ref_factor / actual_factor + raw_scale = float(ref_factor / actual_factor) + scale = self._clip_radiometric_scale(raw_scale, role, cfg) - new_item = dict(item) - new_meta = dict(item.get("meta", {}) or {}) - - out = img.astype(np.float32) * float(scale) + out = img.astype(np.float32) * scale if clip_output: out = np.clip(out, 0.0, 1.0) + new_item = dict(item) + new_meta = dict(item.get("meta", {}) or {}) + + debug = { + "applied": True, + "method": method, + "role": role, + "camera_id": cam_id, + "actual_controls": dict(actual_ctrl), + "reference_controls": dict(ref_ctrl), + "actual_factor": float(actual_factor), + "reference_factor": float(ref_factor), + "scale_raw": float(raw_scale), + "scale_applied": float(scale), + "clip_output": clip_output, + } + + new_meta["radiometric_normalization"] = debug new_meta["radiometric_normalization_applied"] = True - new_meta["radiometric_normalization_method"] = method new_meta["radiometric_normalization_scale"] = float(scale) - new_meta["radiometric_actual_factor"] = float(actual_factor) - new_meta["radiometric_reference_factor"] = float(ref_factor) new_item["image"] = out.astype(np.float32, copy=False) new_item["meta"] = new_meta - normalized[cam_id] = new_item + result["by_role"][role] = debug + result["by_camera"][cam_id] = debug + + result["applied"] = any( + bool(v.get("applied", False)) + for v in result["by_camera"].values() + if isinstance(v, dict) + ) + + scales = [ + float(v.get("scale_applied", 1.0)) + for v in result["by_camera"].values() + if isinstance(v, dict) and v.get("applied", False) + ] + + result["summary"] = { + "applied_count": int(len(scales)), + "scale_min": float(min(scales)) if scales else None, + "scale_max": float(max(scales)) if scales else None, + "scale_mean": float(np.mean(scales)) if scales else None, + "warning_count": int(len(result["warnings"])), + } + + self.last_radiometric_normalization_result = result return normalized + def _extract_frame_controls_from_meta_by_role(self, meta: dict | None) -> tuple[dict, dict]: + """ + Retorna: + controls_by_role = { + "rgb": {...}, + "re": {...}, + "nir": {...} + } + + controls_by_cam = { + "CAM_A": {...}, + "CAM_B": {...}, + "CAM_C": {...} + } + + Fonte principal: + meta["frame_controls"] + + Também aceita: + meta["stream_meta"]["frame_controls"] + """ + if not isinstance(meta, dict): + return {}, {} + + stream_meta = meta.get("stream_meta") if isinstance(meta.get("stream_meta"), dict) else None + + frame_controls = meta.get("frame_controls") + if not isinstance(frame_controls, dict) and stream_meta is not None: + frame_controls = stream_meta.get("frame_controls") + + if not isinstance(frame_controls, dict) or not frame_controls: + return {}, {} + + camera_info = meta.get("camera_info") + if not isinstance(camera_info, dict) and stream_meta is not None: + camera_info = stream_meta.get("camera_info") + + camera_info = camera_info if isinstance(camera_info, dict) else {} + + controls_by_cam = {} + controls_by_role = {} + + for cam_id, ctrl in frame_controls.items(): + if not isinstance(ctrl, dict): + continue + + cam_id = str(cam_id) + controls_by_cam[cam_id] = dict(ctrl) + + role = str( + (camera_info.get(cam_id, {}) or {}).get("role", "") + ).lower() + + if not role: + # fallback defensivo caso algum meta antigo venha sem camera_info + role = self._role_from_cam_id_fallback(cam_id) + + if role: + controls_by_role[role] = dict(ctrl) + + return controls_by_role, controls_by_cam + + def _role_from_cam_id_fallback(self, cam_id: str) -> str: + """ + Fallback fraco. Só usado se o meta não tiver camera_info. + No fluxo novo, camera_info sempre deve existir. + """ + cam_id = str(cam_id).upper() + + role_map = { + "CAM_A": "rgb", + "CAM_B": "re", + "CAM_C": "nir", + } + + return role_map.get(cam_id, "") + + def _radiometric_factor_from_controls(self, ctrl: dict, cfg: dict) -> float: + """ + Modelo físico simples: + fator = exposure_time_us * (sensitivity_iso / iso_base) + + Esse fator representa a amplificação aproximada do sinal causada pela câmera. + """ + if not isinstance(ctrl, dict): + return 0.0 + + iso_base = float(cfg.get("iso_base", 100.0) or 100.0) + + exp = ctrl.get("exposure_time_us", None) + iso = ctrl.get("sensitivity_iso", None) + + # Compatibilidade com contrato antigo + gain = ctrl.get("analogue_gain", None) + gain_est = ctrl.get("analogue_gain_est", None) + + try: + exp = float(exp) + except Exception: + exp = 0.0 + + if exp <= 0: + return 0.0 + + try: + if iso is not None: + gain_factor = float(iso) / iso_base + elif gain_est is not None: + gain_factor = float(gain_est) + elif gain is not None: + gain_factor = float(gain) + else: + gain_factor = 1.0 + except Exception: + gain_factor = 1.0 + + if gain_factor <= 0: + gain_factor = 1.0 + + return float(exp * gain_factor) + + def _clip_radiometric_scale(self, scale: float, role: str, cfg: dict) -> float: + limits = cfg.get("scale_limits", {}) or {} + + role_limits = limits.get(role) + if not isinstance(role_limits, dict): + role_limits = limits.get("default", {}) or {} + + scale_min = float(role_limits.get("min", 0.15)) + scale_max = float(role_limits.get("max", 6.0)) + + if scale_min <= 0: + scale_min = 0.001 + + if scale_max < scale_min: + scale_max = scale_min + + return float(np.clip(float(scale), scale_min, scale_max)) + + def _extract_actual_controls_from_meta(self, meta: dict | None) -> dict: if not meta: return {} @@ -2201,3 +2467,8 @@ class RawProcessorCore: gain = 1.0 return float(exp * gain) + + + def get_last_radiometric_normalization_result(self): + return self.last_radiometric_normalization_result + \ No newline at end of file diff --git a/Python/OAK/datasets/oak-fcc-3/utils/check_saved_files.py b/Python/OAK/datasets/oak-fcc-3/utils/check_saved_files.py index 0c59ba62d..28e4566db 100644 --- a/Python/OAK/datasets/oak-fcc-3/utils/check_saved_files.py +++ b/Python/OAK/datasets/oak-fcc-3/utils/check_saved_files.py @@ -8,7 +8,7 @@ import cv2 import numpy as np from core.raw_processor_core import RawProcessorCore -from core.raw_processor_preview import RawProcessorPreview +from core.oak_fcc3_client import OakFcc3Client def load_json(path: Path) -> dict: @@ -278,7 +278,12 @@ def build_multispec_from_raw_native_multi(group: dict, meta: dict): return tensor, f"MULTISPEC gerado offline do RAW_BRUTO | shape={list(tensor.shape)}", processing_info -def build_visual_from_saved_payload(payload_path: Path, meta: dict, cam_id: str | None = None) -> tuple[np.ndarray, str]: +def build_visual_from_saved_payload( + payload_path: Path, + meta: dict, + cam_id: str | None = None, + client: OakFcc3Client | None = None, +) -> tuple[np.ndarray, str]: """ Retorna: preview_bgr_reconstructed @@ -293,66 +298,14 @@ def build_visual_from_saved_payload(payload_path: Path, meta: dict, cam_id: str if cam_id is None: raise RuntimeError("cam_id é obrigatório para saved_payload_type='raw_native_multi'") - saved_dtypes = meta.get("saved_payload_dtypes", {}) or {} - saved_shapes = meta.get("saved_payload_shapes", {}) or {} - - saved_dtype = saved_dtypes.get(cam_id) - saved_shape = saved_shapes.get(cam_id) - - if saved_dtype is None or saved_shape is None: - raise RuntimeError( - f"JSON não contém saved_payload_dtypes/saved_payload_shapes para {cam_id}" + if client is not None: + return build_camera_preview_with_client( + client=client, + payload_path=payload_path, + meta=meta, + cam_id=cam_id, ) - np_dtype = np.dtype(saved_dtype) - raw = np.fromfile(str(payload_path), dtype=np_dtype) - arr = raw.reshape(tuple(saved_shape)) - - # Busca metadados da câmera no stream_meta - stream_meta = meta.get("stream_meta", {}) or {} - cam_frames = stream_meta.get("camera_info", {}) or {} - cam_meta = cam_frames.get(cam_id, {}) or {} - - role = cam_meta.get("role", cam_id) - interface = cam_meta.get("interface", "") - bit_depth = int(cam_meta.get("bit_depth", 10)) - bayer = cam_meta.get("bayer_pattern", meta.get("bayer_pattern", "BGGR")) - - # USB RGB nativo - if interface.upper() == "USB" or (arr.ndim == 3 and arr.shape[2] == 3 and arr.dtype == np.uint8): - preview_bgr = arr.copy() - desc = f"{cam_id} | role={role} | USB/RGB nativo | dtype={arr.dtype} | shape={arr.shape}" - return preview_bgr, desc - - # CSI RAW packed mono - sensor_width, sensor_height = resolve_sensor_dims_for_raw10_packed(arr, cam_meta, meta) - - core = RawProcessorCore( - sensor_width=sensor_width, - sensor_height=sensor_height, - bayer_pattern=bayer, - ) - preview = RawProcessorPreview( - sensor_width=sensor_width, - sensor_height=sensor_height, - bayer_pattern=bayer, - ) - - packed = arr - if packed.ndim == 3 and packed.shape[2] == 1: - packed = packed[:, :, 0] - - raw16 = core.unpack_raw10_packed(packed) - preview_bgr = preview.raw16_to_preview_bgr(raw16, bit_depth=bit_depth) - - desc = ( - f"{cam_id} | role={role} | RAW packed mono | " - f"dtype={arr.dtype} | shape={arr.shape} | " - f"sensor={sensor_width}x{sensor_height} | " - f"bayer={bayer} | bit_depth={bit_depth}" - ) - return preview_bgr, desc - # ========================================================= # Caso payload único # ========================================================= @@ -477,7 +430,7 @@ def build_visual_from_saved_payload(payload_path: Path, meta: dict, cam_id: str raise RuntimeError(f"saved_payload_type não suportado neste script: {saved_type}") -def build_panels_from_group(group): +def build_panels_from_group(group, client: OakFcc3Client | None = None): panels = [] meta = load_json(group["json"]) @@ -539,7 +492,12 @@ def build_panels_from_group(group): # Continua mostrando CAM_A/CAM_B/CAM_C reconstruídas individualmente. # ========================================================= for cam_id, path in group["cameras"].items(): - img, desc = build_visual_from_saved_payload(path, meta, cam_id=cam_id) + img, desc = build_visual_from_saved_payload( + path, + meta, + cam_id=cam_id, + client=client, + ) panels.append((f"{cam_id} reconstruido", img, desc)) return panels @@ -1063,11 +1021,11 @@ def resolve_navigation_inputs(input_path: Path) -> tuple[list[Path], int]: return entries, idx -def render_group_to_canvas(json_path: Path, max_width: int): +def render_group_to_canvas(json_path: Path, max_width: int, client: OakFcc3Client | None = None): group = resolve_capture_group(json_path) meta = load_json(group["json"]) - panels = build_panels_from_group(group) + panels = build_panels_from_group(group, client=client) panels = sort_panels(panels) canvas = compose_panels(panels, max_width=max_width) @@ -1084,6 +1042,85 @@ def render_group_to_canvas(json_path: Path, max_width: int): return canvas, info +def load_camera_payload_from_saved(meta: dict, payload_path: Path, cam_id: str) -> np.ndarray: + saved_dtypes = meta.get("saved_payload_dtypes", {}) or {} + saved_shapes = meta.get("saved_payload_shapes", {}) or {} + + saved_dtype = saved_dtypes.get(cam_id) + saved_shape = saved_shapes.get(cam_id) + + if saved_dtype is None or saved_shape is None: + raise RuntimeError( + f"JSON não contém saved_payload_dtypes/saved_payload_shapes para {cam_id}" + ) + + raw = np.fromfile(str(payload_path), dtype=np.dtype(saved_dtype)) + return raw.reshape(tuple(saved_shape)) + + +def build_camera_preview_with_client( + client: OakFcc3Client, + payload_path: Path, + meta: dict, + cam_id: str, +) -> tuple[np.ndarray, str]: + stream_meta = meta.get("stream_meta", {}) or {} + camera_info = stream_meta.get("camera_info", {}) or {} + cam_meta = camera_info.get(cam_id, {}) or {} + + role = cam_meta.get("role", cam_id) + interface = str(cam_meta.get("interface", "")).upper() + + arr = load_camera_payload_from_saved(meta, payload_path, cam_id) + + # Caso RGB/USB nativo, mantém comportamento direto. + if interface == "USB" or (arr.ndim == 3 and arr.shape[2] == 3 and arr.dtype == np.uint8): + desc = f"{cam_id} | role={role} | USB/RGB nativo | dtype={arr.dtype} | shape={arr.shape}" + return arr.copy(), desc + + if cam_id == "CAM_A": + preview_bgr = client.build_save_preview_from_cam_a( + packed_raw_by_camera={"CAM_A": arr}, + meta_stream=stream_meta, + sensor_width=int(meta.get("sensor_width", 1280)), + sensor_height=int(meta.get("sensor_height", 800)), + bayer_pattern=meta.get("bayer_pattern", "RGGB"), + ) + + if preview_bgr is None: + raise RuntimeError("client.build_save_preview_from_cam_a retornou None para CAM_A.") + + bayer = ( + cam_meta.get("bayer_pattern") + or cam_meta.get("bayer") + or meta.get("bayer_pattern") + or "RGGB" + ) + + desc = ( + f"{cam_id} | role={role} | preview centralizado no OakFcc3Client | " + f"dtype={arr.dtype} | shape={arr.shape} | bayer={bayer}" + ) + return preview_bgr, desc + + # Para CAM_B/C por enquanto mantém mono visual simples via caminho antigo? + # Melhor: usar build_visual_preview_from_raw do client, que também está centralizado. + previews = client.build_visual_preview_from_raw( + frame={cam_id: arr}, + meta={"camera_info": {cam_id: cam_meta}}, + ) + + if cam_id not in previews: + raise RuntimeError(f"client.build_visual_preview_from_raw não retornou {cam_id}. Chaves={list(previews.keys())}") + + desc = ( + f"{cam_id} | role={role} | preview via OakFcc3Client.build_visual_preview_from_raw | " + f"dtype={arr.dtype} | shape={arr.shape}" + ) + return previews[cam_id], desc + + + def main(): parser = argparse.ArgumentParser( description="Valida visualmente payload salvo (.bin/.raw/.json/.png) comparando com o preview .png" @@ -1095,12 +1132,25 @@ def main(): input_path = Path(args.input_path) entries, current_idx = resolve_navigation_inputs(input_path) + client = OakFcc3Client( + width=1280, + height=800, + bayer="RGGB", + frame_type="RAW_BRUTO", + capture_mode="SINGLE", + raw_policy="allow_single", + ) + window_name = "Validacao payload | A=anterior | D=proximo | T=salva tensor offline | Q/Esc=sair" cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) while True: current_json = entries[current_idx] - canvas, info = render_group_to_canvas(current_json, max_width=args.max_width) + canvas, info = render_group_to_canvas( + current_json, + max_width=args.max_width, + client=client, + ) # Cabeçalho adicional na imagem overlay = canvas.copy()