Zendion/agrobot_base
Zendion
/
agrobot_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ajustes nas ferramentas do modulo multiespectral

This commit is contained in:
Diego Freitas 2026-04-24 15:03:44 -03:00
parent cd1f777415
commit 41dfb3c06e
24 changed files with 2776 additions and 10074 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

303
Python/OAK/datasets/multiespec_module/_0_capture.py
View File

@ -7,22 +7,13 @@ from datetime import datetime
import cv2 import cv2
import numpy as np import numpy as np
from multispectral_service import MultiSpectralService from pi.multispectral_client import MultiSpectralClient
from stream_receiver import StreamReceiver
from pi.raw_processor_core import RawProcessorCore
from pi.raw_processor_preview import RawProcessorPreview
STREAM_PORT = 6001
PI_HOST = "192.168.105.6"
PC_HOST = "192.168.105.5"
with open("config.json", "r", encoding="utf-8") as f: with open("config.json", "r", encoding="utf-8") as f:
config = json.load(f) config = json.load(f)
MODELO = config.get("camera", ".") RAW_SIZE = config.get("raw_size") # [W, H]
RAW_SIZE = config.get("raw_size", [1296, 1028]) # [W, H] MODULE_PARAMS = config.get("module_params_json")
CAMERA_PARAMS = config.get("camera_params_json")
# ========================= # =========================
@ -135,158 +126,6 @@ def get_camera_map_from_status(status: dict) -> dict:
return result return result
def validate_module_ready(status: dict, frame_type: str, raw_policy: str, capture_mode: str):
if not status.get("ok", True):
raise RuntimeError(f"Status inválido retornado pelo módulo: {status}")
active_ids = list(status.get("active_camera_ids", []))
active_count = int(status.get("camera_count_active", 0))
if frame_type == "RGB":
if "cam2" not in active_ids:
raise RuntimeError(
"Modo RGB requer cam2 ativa (USB RGB), mas o módulo não reportou cam2 como ativa."
)
return
if frame_type == "MULTISPEC":
if capture_mode == "TRIPLE":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"Modo MULTISPEC/TRIPLE requer cam0, cam1 e cam2 ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
return
if capture_mode == "DOUBLE":
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not has_rgb or not has_spec:
raise RuntimeError(
f"Modo MULTISPEC/DOUBLE requer cam2 + (cam0 ou cam1). "
f"Ativas atuais: {active_ids}"
)
return
# AUTO ou outros casos
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not (has_rgb and has_spec):
raise RuntimeError(
f"Modo MULTISPEC requer pelo menos RGB + 1 canal espectral. "
f"Ativas atuais: {active_ids}"
)
return
if frame_type == "RAW_BRUTO":
if raw_policy == "require_triple":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"RAW_BRUTO com política require_triple exige três câmeras ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
else:
if active_count < 1:
raise RuntimeError("RAW_BRUTO requer ao menos uma câmera ativa, mas nenhuma foi detectada.")
return
raise RuntimeError(f"frame_type desconhecido para validação: {frame_type}")
def build_preview_from_raw_payload(
frame,
meta: dict,
processor_core: RawProcessorCore,
processor_preview: RawProcessorPreview,
):
"""
Gera preview priorizando a câmera RGB (cam2).
Se cam2 não estiver presente, cai para fallback usando a primeira câmera mono disponível.
Retorna:
preview_bgr
payload_float_preview
preview_source_id
"""
payload_sources = meta.get("payload_sources", []) or []
# Caso multi-payload: tenta usar cam2 primeiro
if isinstance(frame, dict):
if "cam2" in frame:
rgb_frame = frame["cam2"]
if rgb_frame.ndim != 3 or rgb_frame.shape[2] != 3:
raise RuntimeError(f"cam2 recebida mas inválida para preview RGB: shape={rgb_frame.shape}")
preview_bgr = rgb_frame.copy()
payload_float = rgb_frame[:, :, ::-1].astype(np.float32) / 255.0
payload_float = np.transpose(payload_float, (2, 0, 1))
return preview_bgr, payload_float, "cam2"
# fallback: usa a primeira câmera mono disponível
fallback_id = None
for cid in ("cam0", "cam1"):
if cid in frame:
fallback_id = cid
break
if fallback_id is None:
raise RuntimeError("Nenhuma câmera disponível no payload para gerar preview")
packed = frame[fallback_id]
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
cam_frames = meta.get("camera_frames", {}) or {}
cam_meta = cam_frames.get(fallback_id, {})
bit_depth = int(cam_meta.get("bit_depth", 10))
raw16 = processor_core.unpack_raw10_packed(packed)
preview_bgr = processor_preview.raw16_to_preview_bgr(raw16, bit_depth=bit_depth)
payload_float = processor_core.build_training_rgb(
raw16,
output_dtype="float32",
bit_depth=bit_depth,
)
return preview_bgr, payload_float, fallback_id
# Caso single-payload
if isinstance(frame, np.ndarray):
# Se vier HWC/3ch, tratamos como RGB USB
if frame.ndim == 3 and frame.shape[2] == 3:
preview_bgr = frame.copy()
payload_float = frame[:, :, ::-1].astype(np.float32) / 255.0
payload_float = np.transpose(payload_float, (2, 0, 1))
return preview_bgr, payload_float, "cam2"
# Se vier mono packed, fallback antigo
packed = frame
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
source_camera = meta.get("source_camera") or {}
bit_depth = int(source_camera.get("bit_depth", meta.get("source_bit_depth", 10)))
raw16 = processor_core.unpack_raw10_packed(packed)
preview_bgr = processor_preview.raw16_to_preview_bgr(raw16, bit_depth=bit_depth)
payload_float = processor_core.build_training_rgb(
raw16,
output_dtype="float32",
bit_depth=bit_depth,
)
return preview_bgr, payload_float, source_camera.get("id", "unknown")
raise RuntimeError(f"Tipo de frame não suportado para preview: {type(frame)}")
# ========================= # =========================
# MAIN # MAIN
# ========================= # =========================
@ -300,9 +139,9 @@ def main():
parser.add_argument("--cana", required=True, choices=["baixa", "media", "alta"], help="Estado da cana no momento da coleta.") parser.add_argument("--cana", required=True, choices=["baixa", "media", "alta"], help="Estado da cana no momento da coleta.")
parser.add_argument("--horario", required=True, choices=["cedo", "meio_dia", "entardecer", "nublado"], help="Janela de iluminação / horário da coleta.") parser.add_argument("--horario", required=True, choices=["cedo", "meio_dia", "entardecer", "nublado"], help="Janela de iluminação / horário da coleta.")
parser.add_argument("--out_root", default="dataset", help="Pasta raiz do dataset.") parser.add_argument("--out_root", default="dataset", help="Pasta raiz do dataset.")
parser.add_argument("--pi_host", default=PI_HOST, help="IP do servidor no Raspberry Pi.") parser.add_argument("--pi_host", default="192.168.105.6", help="IP do servidor no Raspberry Pi.")
parser.add_argument("--pc_host", default=PC_HOST, help="IP local do notebook/PC que receberá o stream.") parser.add_argument("--pc_host", default="192.168.105.5", help="IP local do notebook/PC que receberá o stream.")
parser.add_argument("--stream_port", type=int, default=STREAM_PORT, help="Porta TCP do receiver de stream.") parser.add_argument("--stream_port", type=int, default=6001, help="Porta TCP do receiver de stream.")
parser.add_argument("--server_port", type=int, default=5000, help="Porta TCP do servidor de comandos no Pi.") parser.add_argument("--server_port", type=int, default=5000, help="Porta TCP do servidor de comandos no Pi.")
parser.add_argument("--fps", type=int, default=20, help="FPS desejado.") parser.add_argument("--fps", type=int, default=20, help="FPS desejado.")
parser.add_argument("--width", type=int, default=RAW_SIZE[0], help="Largura óptica da câmera.") parser.add_argument("--width", type=int, default=RAW_SIZE[0], help="Largura óptica da câmera.")
@ -314,7 +153,7 @@ def main():
parser.add_argument("--frame_type", default="RAW_BRUTO", choices=["RAW_BRUTO", "RGB", "MULTISPEC"], help="Tipo de payload pedido ao Pi.") parser.add_argument("--frame_type", default="RAW_BRUTO", choices=["RAW_BRUTO", "RGB", "MULTISPEC"], help="Tipo de payload pedido ao Pi.")
parser.add_argument("--capture_mode", default="AUTO", choices=["AUTO", "SINGLE", "DOUBLE", "TRIPLE"], help="Modo de captura desejado no módulo.") parser.add_argument("--capture_mode", default="AUTO", choices=["AUTO", "SINGLE", "DOUBLE", "TRIPLE"], help="Modo de captura desejado no módulo.")
parser.add_argument("--raw_policy", default="allow_single", choices=["allow_single", "require_triple"], help="Quando frame_type=RAW_BRUTO, define se o script aceita 1 câmera ou exige 3.") parser.add_argument("--raw_policy", default="allow_single", choices=["allow_single", "require_triple"], help="Quando frame_type=RAW_BRUTO, define se o script aceita 1 câmera ou exige 3.")
parser.add_argument("--camera_params_json", default=CAMERA_PARAMS, help="JSON salvo pelo calibrador de sensores com parâmetros fixos por câmera.") parser.add_argument("--module_calibration_json", default=MODULE_PARAMS, help="JSON salvo pelo calibrador de sensores com parâmetros fixos por câmera.")
args = parser.parse_args() args = parser.parse_args()
@ -359,30 +198,9 @@ def main():
last_msg = "" last_msg = ""
last_msg_t = 0.0 last_msg_t = 0.0
receiver = StreamReceiver(host="0.0.0.0", port=args.stream_port)
svc = MultiSpectralService(host=args.pi_host, port=args.server_port, timeout=10)
print(f"[INFO] Verificando conexão com o módulo em {args.pi_host}:{args.server_port}...")
if not svc.check_connection(2):
raise RuntimeError("Módulo não encontrado ou não respondeu ao ping.")
print("[OK] Módulo conectado e respondendo.")
window_name = "Dataset Capture (C/SPACE=save | A=auto-save | M=preview scale | Q=quit)" window_name = "Dataset Capture (C/SPACE=save | A=auto-save | M=preview scale | Q=quit)"
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
processor_core_cam0 = RawProcessorCore(
sensor_width=raw_w,
sensor_height=raw_h,
bayer_pattern=args.bayer,
)
processor_preview_cam0 = RawProcessorPreview(
sensor_width=raw_w,
sensor_height=raw_h,
bayer_pattern=args.bayer,
)
last_frame_id = -1 last_frame_id = -1
last_payload_float = None last_payload_float = None
last_packed_raw = None last_packed_raw = None
@ -391,70 +209,25 @@ def main():
last_meta_stream = None last_meta_stream = None
try: try:
receiver.start() with MultiSpectralClient(
time.sleep(0.5) pi_host=args.pi_host,
pc_host=args.pc_host,
svc.connect() server_port=args.server_port,
stream_port=args.stream_port,
if args.frame_type in ("RAW_BRUTO", "MULTISPEC"): width=raw_w,
modes_resp = svc.get_sensor_modes() height=raw_h,
if not modes_resp.get("ok"): bayer=args.bayer,
print(f"[WARN] Falha ao obter sensor_modes: {modes_resp}") fps=args.fps,
else:
for mode in modes_resp.get("sensor_modes", []):
print(
f"[cam={mode.get('camera_id')} mode={mode.get('mode_index')}] "
f"size={mode.get('size')} "
f"format={mode.get('format')} "
f"bit_depth={mode.get('bit_depth')} "
f"fps={mode.get('fps')}"
)
else:
print("[INFO] get_sensor_modes pulado para frame_type=RGB")
print("SET CAM0 RES:", svc.set_camera_resolution(0, raw_w, raw_h))
print("SET CAM1 RES:", svc.set_camera_resolution(1, raw_w, raw_h))
print("SET CAM2 RES:", svc.set_camera_resolution(2, raw_w, raw_h))
print("SET CAM0 BAYER:", svc.set_camera_bayer(0, args.bayer))
print("SET CAM1 BAYER:", svc.set_camera_bayer(1, args.bayer))
print("SET FPS:", svc.set_fps(args.fps))
print("SET CAPTURE MODE:", svc.set_capture_mode(effective_capture_mode))
print("SET FRAME TYPE:", svc.set_frame_type(args.frame_type))
print("SET OUTPUT DTYPE:", svc.set_output_dtype(args.output_dtype))
begin_resp = svc.begin(
frame_type=args.frame_type, frame_type=args.frame_type,
output_dtype=args.output_dtype, output_dtype=args.output_dtype,
capture_mode=effective_capture_mode, capture_mode=effective_capture_mode,
) raw_policy=args.raw_policy,
print("BEGIN:", begin_resp) module_calibration_json=args.module_calibration_json,
) as cam:
status = svc.get_status()
print("STATUS:", json.dumps({
"status": status.get("status"),
"detected_mode": status.get("detected_mode"),
"camera_count_active": status.get("camera_count_active"),
"active_camera_ids": status.get("active_camera_ids"),
}, ensure_ascii=False))
validate_module_ready(status, args.frame_type, args.raw_policy, effective_capture_mode)
params_resp = svc.apply_camera_params_json(args.camera_params_json)
if params_resp is not None:
camera_settings = params_resp["camera_settings"]
applied_camera_controls = params_resp["applied"]
print("[OK] Parâmetros fixos das câmeras aplicados:")
print(json.dumps(applied_camera_controls, ensure_ascii=False, indent=2))
else:
print("[OK] Parâmetros fixos das câmeras não aplicados")
print("START STREAM:", svc.start_stream(args.pc_host, args.stream_port, fps=args.fps))
while True: while True:
t0 = time.time() t0 = time.time()
meta = receiver.last_meta frame, meta = cam.get_next_frame(timeout=1.0)
frame = receiver.last_frame
if meta is not None and frame is not None and meta.get("frame_id") != last_frame_id: if meta is not None and frame is not None and meta.get("frame_id") != last_frame_id:
last_frame_id = meta["frame_id"] last_frame_id = meta["frame_id"]
@ -468,24 +241,14 @@ def main():
if isinstance(frame, dict): if isinstance(frame, dict):
packed_by_camera = frame packed_by_camera = frame
preview_bgr, raw3_preview, preview_source_id = build_preview_from_raw_payload( preview_bgr, raw3_preview, preview_source_id = cam.build_preview_from_raw_payload(frame=frame, meta=meta)
frame=frame,
meta=meta,
processor_core=processor_core_cam0,
processor_preview=processor_preview_cam0,
)
last_packed_raw = None last_packed_raw = None
last_packed_raw_by_camera = {cam_id: arr.copy() for cam_id, arr in packed_by_camera.items()} last_packed_raw_by_camera = {cam_id: arr.copy() for cam_id, arr in packed_by_camera.items()}
last_payload_float = raw3_preview.copy() last_payload_float = raw3_preview.copy()
else: else:
preview_bgr, raw3_preview, preview_source_id = build_preview_from_raw_payload( preview_bgr, raw3_preview, preview_source_id = cam.build_preview_from_raw_payload(frame=frame, meta=meta)
frame=frame,
meta=meta,
processor_core=processor_core_cam0,
processor_preview=processor_preview_cam0,
)
last_packed_raw = frame.copy() last_packed_raw = frame.copy()
last_packed_raw_by_camera = None last_packed_raw_by_camera = None
@ -579,7 +342,7 @@ def main():
f"Sources={active_sources} | FPS_STREAM={fps_stream:.1f} | FPS_VIEW={fps_view:.1f}", f"Sources={active_sources} | FPS_STREAM={fps_stream:.1f} | FPS_VIEW={fps_view:.1f}",
f"frame_id={meta.get('frame_id')} | layout={meta.get('output_layout')} | dtype={meta.get('dtype') or meta.get('output_dtype')}", f"frame_id={meta.get('frame_id')} | layout={meta.get('output_layout')} | dtype={meta.get('dtype') or meta.get('output_dtype')}",
f"codec={meta.get('codec_name', meta.get('codec_family', '-'))} | comp={meta.get('dt_comp', 0):.4f}s | send={meta.get('dt_send_payload_prev', 0):.4f}s", f"codec={meta.get('codec_name', meta.get('codec_family', '-'))} | comp={meta.get('dt_comp', 0):.4f}s | send={meta.get('dt_send_payload_prev', 0):.4f}s",
f"CAM_PARAMS={os.path.basename(args.camera_params_json)} | controles fixos aplicados", f"CAM_PARAMS={os.path.basename(args.module_calibration_json)} | controles fixos aplicados",
"Keys: C/SPACE=save | A=auto-save | M=preview | Q/Esc=quit" "Keys: C/SPACE=save | A=auto-save | M=preview | Q/Esc=quit"
] ]
overlay_hud(preview_show, lines, base_h=raw_h) overlay_hud(preview_show, lines, base_h=raw_h)
@ -626,8 +389,8 @@ def main():
"capture_mode_effective": effective_capture_mode, "capture_mode_effective": effective_capture_mode,
"raw_policy": args.raw_policy, "raw_policy": args.raw_policy,
"stream_meta": last_meta_stream, "stream_meta": last_meta_stream,
"applied_camera_controls": applied_camera_controls, "applied_camera_controls": cam.applied_camera_controls,
"camera_params_json": args.camera_params_json, "camera_params_json": args.module_calibration_json,
"note": "autosave", "note": "autosave",
"raw_preview_reference_camera": preview_source_id, "raw_preview_reference_camera": preview_source_id,
} }
@ -676,8 +439,8 @@ def main():
"capture_mode_effective": effective_capture_mode, "capture_mode_effective": effective_capture_mode,
"raw_policy": args.raw_policy, "raw_policy": args.raw_policy,
"stream_meta": last_meta_stream, "stream_meta": last_meta_stream,
"applied_camera_controls": applied_camera_controls, "applied_camera_controls": cam.applied_camera_controls,
"camera_params_json": args.camera_params_json, "camera_params_json": args.module_calibration_json,
"note": "manual", "note": "manual",
"raw_preview_reference_camera": preview_source_id, "raw_preview_reference_camera": preview_source_id,
} }
@ -700,18 +463,6 @@ def main():
time.sleep(0.001) time.sleep(0.001)
finally: finally:
try:
print("STOP STREAM:", svc.stop_stream())
except Exception:
pass
try:
print("STOP:", svc.stop())
except Exception:
pass
svc.disconnect()
receiver.stop()
cv2.destroyAllWindows() cv2.destroyAllWindows()
print("Fim da captura.") print("Fim da captura.")

14
Python/OAK/datasets/multiespec_module/_6_normalize.py
View File

@ -32,6 +32,8 @@ with open("config.json", "r", encoding="utf-8") as f:
config = json.load(f) config = json.load(f)
RES = tuple(config["resolucao"]) # (W,H) RES = tuple(config["resolucao"]) # (W,H)
raw_w, raw_h = tuple(config["raw_size"]) # (W,H)
MODULE_PARAMS = config.get("module_params_json")
pasta_base = "dataset" pasta_base = "dataset"
INPUTS = [ INPUTS = [
@ -71,7 +73,7 @@ def process():
cor_para_id, _, _, ignore_rgb = carregar_labelmap_completo(labelmap_path) cor_para_id, _, _, ignore_rgb = carregar_labelmap_completo(labelmap_path)
ignore_id = _infer_ignore_id(ignore_rgb, 255) ignore_id = _infer_ignore_id(ignore_rgb, 255)
core = RawProcessorCore(640, 480) core = RawProcessorCore(raw_w, raw_h, calibration_json_path=MODULE_PARAMS)
total = 0 total = 0
@ -127,15 +129,7 @@ def process():
bins_meta.append(cam_meta) bins_meta.append(cam_meta)
# ===== BUILD TENSOR ===== # ===== BUILD TENSOR =====
tensor, channel_names = core.build_multispectral_tensor(bins_data, bins_meta) tensor, channel_names = core.build_multispectral_tensor(bins_data, bins_meta, target_size=RES)
# ===== RESIZE =====
chans = []
for ch in tensor:
ch_res = cv2.resize(ch, RES, interpolation=cv2.INTER_AREA)
chans.append(ch_res.astype(np.float32))
tensor = np.stack(chans, axis=0)
# ===== STATS ===== # ===== STATS =====
c, h, w = tensor.shape c, h, w = tensor.shape

7
Python/OAK/datasets/multiespec_module/_8_train_segformer.py
View File

@ -342,10 +342,8 @@ def main():
with open(args.config, "r") as f: with open(args.config, "r") as f:
config = json.load(f) config = json.load(f)
MODELO = config["camera"]
MODEL_NAME = config["model_name"] MODEL_NAME = config["model_name"]
RESOLUCAO = config["resolucao"] W, H = config["resolucao"]
W, H = RESOLUCAO[0], RESOLUCAO[1]
MAIN_CLASS_NAME = str(config.get("main_class_name", "erva")).lower() MAIN_CLASS_NAME = str(config.get("main_class_name", "erva")).lower()
# novos parâmetros do config # novos parâmetros do config
@ -415,8 +413,7 @@ def main():
resize_hw = (args.resize_h, args.resize_w) resize_hw = (args.resize_h, args.resize_w)
else: else:
# Se quiser, pode forçar pra RESOLUCAO do config (H,W) # Se quiser, pode forçar pra RESOLUCAO do config (H,W)
# resize_hw = (RESOLUCAO[1], RESOLUCAO[0]) resize_hw = (H, W)
pass
# Datasets (RAW com 4 ou 5 canais) # Datasets (RAW com 4 ou 5 canais)
ds_train = MultispecSegDataset( ds_train = MultispecSegDataset(

96
Python/OAK/datasets/multiespec_module/_9_test_segformer.py
View File

@ -28,10 +28,7 @@ import numpy as np
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from utils import converter_mask_ids_para_bgr, desenhar_legenda_horizontal from utils import converter_mask_ids_para_bgr, desenhar_legenda_horizontal
from multispec_segformer_service import ( from multispec_segformer_service import (MultispecSegformerService, MultispecSegDataset)
MultispecSegformerService,
MultispecSegDataset,
)
# ============================================================ # ============================================================
@ -97,9 +94,9 @@ def main():
parser.add_argument("--resize_h", type=int, default=None, help="Altura para inferência (override)") parser.add_argument("--resize_h", type=int, default=None, help="Altura para inferência (override)")
parser.add_argument("--resize_w", type=int, default=None, help="Largura para inferência (override)") parser.add_argument("--resize_w", type=int, default=None, help="Largura para inferência (override)")
parser.add_argument("--alpha", type=float, default=0.45, help="Alpha do overlay da máscara") parser.add_argument("--alpha", type=float, default=0.45, help="Alpha do overlay da máscara")
parser.add_argument("--camera_frame_type", type=str, default="MULTISPEC", choices=["RAW_BRUTO", "RGB", "MULTISPEC"]) parser.add_argument("--camera_frame_type", type=str, default="RAW_BRUTO", choices=["RAW_BRUTO", "RGB", "MULTISPEC"])
parser.add_argument("--camera_capture_mode", type=str, default="AUTO", choices=["AUTO", "SINGLE", "DOUBLE", "TRIPLE"]) parser.add_argument("--camera_capture_mode", type=str, default="AUTO", choices=["AUTO", "SINGLE", "DOUBLE", "TRIPLE"])
parser.add_argument("--camera_params_json", default=None, help="JSON salvo pelo calibrador de sensores com parâmetros fixos por câmera.") parser.add_argument("--module_calibration_json", default=None, help="JSON salvo pelo calibrador de sensores com parâmetros fixos por câmera.")
args = parser.parse_args() args = parser.parse_args()
@ -110,17 +107,16 @@ def main():
with open(args.config, "r", encoding="utf-8") as f: with open(args.config, "r", encoding="utf-8") as f:
config = json.load(f) config = json.load(f)
MODELO = config["camera"]
MODEL_NAME = config["model_name"] MODEL_NAME = config["model_name"]
modelo_folder = config["modelo"] modelo_folder = config["modelo"]
CAMERA_PARAMS = config.get("camera_params_json") MODULE_PARAMS = config.get("module_params_json")
if args.camera_params_json is not None: if args.module_calibration_json is not None:
CAMERA_PARAMS = args.camera_params_json MODULE_PARAMS = args.module_calibration_json
CHANNELS = int(config.get("channels", 5)) CHANNELS = int(config.get("channels", 5))
FUSION_MODE = config.get("fusion_mode", "stacked") FUSION_MODE = config.get("fusion_mode", "stacked")
RESOLUCAO = config["resolucao"] W, H = config["resolucao"]
W, H = RESOLUCAO[0], RESOLUCAO[1] faw_w, raw_h = config["raw_size"]
if args.resize_h is not None: if args.resize_h is not None:
H = args.resize_h H = args.resize_h
if args.resize_w is not None: if args.resize_w is not None:
@ -205,50 +201,7 @@ def main():
if args.camera: if args.camera:
print("[mode] Câmera MULTISPEC + SegFormer") print("[mode] Câmera MULTISPEC + SegFormer")
from multispectral_service import MultiSpectralService from pi.multispectral_client import MultiSpectralClient
from stream_receiver import StreamReceiver
from pi.raw_processor_core import RawProcessorCore
STREAM_PORT = 6001
PI_HOST = "192.168.105.6"
PC_HOST = "192.168.105.5"
receiver = StreamReceiver(host="0.0.0.0", port=STREAM_PORT)
svc = MultiSpectralService(host=PI_HOST, port=5000, timeout=10)
if not svc.check_connection(2):
raise RuntimeError("Módulo não encontrado ou não respondeu ao ping.")
receiver.start()
time.sleep(0.5)
svc.connect()
core = RawProcessorCore(sensor_width=W, sensor_height=H)
camera_frame_type = args.camera_frame_type
camera_output_dtype = "uint8"
camera_capture_mode = args.camera_capture_mode
print("SET FRAME TYPE:", svc.set_frame_type(camera_frame_type))
print("SET OUTPUT DTYPE:", svc.set_output_dtype(camera_output_dtype))
print("SET FPS:", svc.set_fps(15))
print("BEGIN:", svc.begin(
frame_type=camera_frame_type,
output_dtype=camera_output_dtype,
capture_mode=camera_capture_mode,
))
params_resp = svc.apply_camera_params_json(CAMERA_PARAMS)
if params_resp is not None:
camera_settings = params_resp["camera_settings"]
applied_camera_controls = params_resp["applied"]
print("[OK] Parâmetros fixos das câmeras aplicados:")
print(json.dumps(applied_camera_controls, ensure_ascii=False, indent=2))
else:
print("[OK] Parâmetros fixos das câmeras não aplicados")
print("START STREAM:", svc.start_stream(PC_HOST, STREAM_PORT, fps=15))
win = "CAMERA MULTISPEC + SEGFORMER (Q=quit)" win = "CAMERA MULTISPEC + SEGFORMER (Q=quit)"
cv2.namedWindow(win, cv2.WINDOW_NORMAL) cv2.namedWindow(win, cv2.WINDOW_NORMAL)
@ -261,9 +214,22 @@ def main():
fps_smooth = 0.15 fps_smooth = 0.15
try: try:
with MultiSpectralClient(
pi_host="192.168.105.6",
pc_host="192.168.105.5",
server_port=5000,
stream_port=6001,
width=raw_h,
height=raw_h,
fps=15,
frame_type=args.camera_frame_type,
output_dtype="uint8",
capture_mode=args.camera_capture_mode,
raw_policy="allow_single",
module_calibration_json=MODULE_PARAMS,
) as cam:
while True: while True:
meta = receiver.last_meta frame, meta = cam.get_next_frame(timeout=0.5)
frame = receiver.last_frame
if meta is None or frame is None: if meta is None or frame is None:
continue continue
@ -296,7 +262,7 @@ def main():
fps_pi = (1.0 - fps_smooth) * fps_pi + fps_smooth * inst_fps_pi fps_pi = (1.0 - fps_smooth) * fps_pi + fps_smooth * inst_fps_pi
try: try:
raw_np = core.build_infer_tensor_from_stream(frame, meta, channels_expected=CHANNELS) raw_np = cam.build_infer_tensor(frame, meta, channels_expected=CHANNELS, target_size=(W, H))
# ========================================================= # =========================================================
# INFERÊNCIA # INFERÊNCIA
@ -337,18 +303,6 @@ def main():
break break
finally: finally:
try:
svc.stop_stream()
except:
pass
try:
svc.stop()
except:
pass
svc.disconnect()
receiver.stop()
cv2.destroyAllWindows() cv2.destroyAllWindows()
return return

2
Python/OAK/datasets/multiespec_module/config.json
View File

@ -16,5 +16,5 @@
"backbone": "nvidia/mit-b1", "backbone": "nvidia/mit-b1",
"fusion_mode": "stacked", "fusion_mode": "stacked",
"stats_source_tag": "stacked_raw4", "stats_source_tag": "stacked_raw4",
"camera_params_json": "camera_params.json" "module_params_json": "module_params.json"
} }

77
Python/OAK/datasets/multiespec_module/module_params.json Normal file
View File

@ -0,0 +1,77 @@
{
"schema": "multispec_module_params_v1",
"saved_at": "2026-04-24 10:46:59",
"frame_type": "RAW_BRUTO",
"capture_mode_requested": "AUTO",
"capture_mode_effective": "AUTO",
"raw_policy": "allow_single",
"sensor_width": 640,
"sensor_height": 480,
"bayer_pattern": "GBRG",
"camera_settings": {
"cam0": {
"ae_enable": false,
"awb_enable": false,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": null
},
"cam1": {
"ae_enable": false,
"awb_enable": false,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": null
},
"cam2": {
"ae_enable": true,
"awb_enable": true,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": [
1.0,
1.0
]
}
},
"fusion_config": {
"alignment_mode": "homography",
"baseline_mm": 75.0,
"reference_camera": "cam2",
"manual_offsets": {
"cam0": {
"dx": 0,
"dy": 0,
"theta_deg": 0.0
},
"cam1": {
"dx": 0,
"dy": 0,
"theta_deg": 0.0
}
},
"homographies": {
"cam0_to_cam2": [
[
0.6438707381367006,
-0.44541142339026096,
171.26012619956475
],
[
-0.04083362402731768,
0.6855417219007448,
17.531834653386724
],
[
-0.00015145530359180688,
-0.0006687764363592821,
1.0
]
],
"cam1_to_cam2": null
},
"crop_valid_common": true,
"resize_after_crop": true,
"target_size": null
}
}

97
Python/OAK/datasets/multiespec_module/multispectral_service.py
View File

@ -95,6 +95,68 @@ class MultiSpectralService:
return json.loads(line.strip()) return json.loads(line.strip())
def validate_module_ready(self, status: dict, frame_type: str, raw_policy: str, capture_mode: str):
if not status.get("ok", True):
raise RuntimeError(f"Status inválido retornado pelo módulo: {status}")
active_ids = list(status.get("active_camera_ids", []))
active_count = int(status.get("camera_count_active", 0))
if frame_type == "RGB":
if "cam2" not in active_ids:
raise RuntimeError(
"Modo RGB requer cam2 ativa (USB RGB), mas o módulo não reportou cam2 como ativa."
)
return
if frame_type == "MULTISPEC":
if capture_mode == "TRIPLE":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"Modo MULTISPEC/TRIPLE requer cam0, cam1 e cam2 ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
return
if capture_mode == "DOUBLE":
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not has_rgb or not has_spec:
raise RuntimeError(
f"Modo MULTISPEC/DOUBLE requer cam2 + (cam0 ou cam1). "
f"Ativas atuais: {active_ids}"
)
return
# AUTO ou outros casos
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not (has_rgb and has_spec):
raise RuntimeError(
f"Modo MULTISPEC requer pelo menos RGB + 1 canal espectral. "
f"Ativas atuais: {active_ids}"
)
return
if frame_type == "RAW_BRUTO":
if raw_policy == "require_triple":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"RAW_BRUTO com política require_triple exige três câmeras ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
else:
if active_count < 1:
raise RuntimeError("RAW_BRUTO requer ao menos uma câmera ativa, mas nenhuma foi detectada.")
return
raise RuntimeError(f"frame_type desconhecido para validação: {frame_type}")
# ========================================================= # =========================================================
# Helpers numpy # Helpers numpy
# ========================================================= # =========================================================
@ -259,13 +321,20 @@ class MultiSpectralService:
def get_config(self): def get_config(self):
return self._send_command({"cmd": "get_config"}) return self._send_command({"cmd": "get_config"})
def begin(self, frame_type: str = "RAW_BRUTO", output_dtype: str = "uint8", capture_mode: str = "AUTO"): def begin(self, frame_type="RAW_BRUTO", output_dtype="uint8", capture_mode="AUTO", timeout=15):
old_timeout = self.sock.gettimeout() if self.sock else None
if self.sock and timeout is not None:
self.sock.settimeout(timeout)
try:
return self._send_command({ return self._send_command({
"cmd": "begin", "cmd": "begin",
"frame_type": frame_type, "frame_type": frame_type,
"output_dtype": output_dtype, "output_dtype": output_dtype,
"capture_mode": capture_mode, "capture_mode": capture_mode,
}) })
finally:
if self.sock and old_timeout is not None:
self.sock.settimeout(old_timeout)
def stop(self): def stop(self):
return self._send_command({"cmd": "stop"}) return self._send_command({"cmd": "stop"})
@ -307,6 +376,31 @@ class MultiSpectralService:
"height": height "height": height
}) })
def set_resolution(self, width: int, height: int):
res = []
for index in range(0, 3):
res.append(
self._send_command({
"cmd": "set_camera_resolution",
"index": index,
"width": width,
"height": height
})
)
return res
def set_bayer(self, bayer_pattern: str):
res = []
for index in range(0, 2):
res.append(
self._send_command({
"cmd": "set_camera_bayer",
"index": index,
"pattern": bayer_pattern
})
)
return res
# ========================================================= # =========================================================
# Captura # Captura
# ========================================================= # =========================================================
@ -431,6 +525,7 @@ class MultiSpectralService:
def get_sensor_modes(self): def get_sensor_modes(self):
return self._send_command({"cmd": "get_sensor_modes"}) return self._send_command({"cmd": "get_sensor_modes"})
def load_camera_params_json(self, path: str) -> dict: def load_camera_params_json(self, path: str) -> dict:
if not path or not os.path.isfile(path): if not path or not os.path.isfile(path):
raise FileNotFoundError(f"Arquivo de parâmetros das câmeras não encontrado: {path}") raise FileNotFoundError(f"Arquivo de parâmetros das câmeras não encontrado: {path}")

304
Python/OAK/datasets/multiespec_module/pi/multispectral_client.py Normal file
View File

@ -0,0 +1,304 @@
import time
import json
import numpy as np
from multispectral_service import MultiSpectralService
from stream_receiver import StreamReceiver
from pi.raw_processor_core import RawProcessorCore
from pi.raw_processor_preview import RawProcessorPreview
class MultiSpectralClient:
def __init__(
self,
pi_host="192.168.105.6",
pc_host="192.168.105.5",
server_port=5000,
stream_port=6001,
timeout=10,
width=640,
height=480,
bayer="GBRG",
fps=15,
frame_type="RAW_BRUTO",
output_dtype="uint8",
capture_mode="AUTO",
raw_policy="allow_single",
module_calibration_json=None,
):
self.pi_host = pi_host
self.pc_host = pc_host
self.server_port = server_port
self.stream_port = stream_port
self.width = width
self.height = height
self.bayer = bayer
self.fps = fps
self.frame_type = frame_type
self.output_dtype = output_dtype
self.capture_mode = capture_mode
self.raw_policy = raw_policy
self.module_calibration_json = module_calibration_json
self.svc = MultiSpectralService(
host=pi_host,
port=server_port,
timeout=timeout,
)
self.receiver = StreamReceiver(
host="0.0.0.0",
port=stream_port,
)
self.core = RawProcessorCore(
sensor_width=width,
sensor_height=height,
bayer_pattern=bayer,
calibration_json_path=module_calibration_json,
)
self.preview = RawProcessorPreview(
sensor_width=width,
sensor_height=height,
bayer_pattern=bayer,
)
self.last_frame_id = None
self.status = None
self.begin_resp = None
self.applied_params = None
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc, tb):
self.stop()
def start(self, print_debug=True):
if print_debug:
print(f"[INFO] Verificando módulo em {self.pi_host}:{self.server_port}...")
if not self.svc.check_connection(2):
raise RuntimeError("Módulo não encontrado ou não respondeu ao ping.")
self.receiver.start()
time.sleep(0.5)
self.svc.connect()
if print_debug:
print("[OK] Módulo conectado.")
self._configure_module(print_debug=print_debug)
self._begin_module(print_debug=print_debug)
self._apply_module_params(print_debug=print_debug)
self._start_stream(print_debug=print_debug)
return self
def _configure_module(self, print_debug=True):
r0 = self.svc.set_resolution(self.width, self.height)
r1 = self.svc.set_bayer(self.bayer)
r2 = self.svc.set_fps(self.fps)
r3 = self.svc.set_capture_mode(self.capture_mode)
r4 = self.svc.set_frame_type(self.frame_type)
r5 = self.svc.set_output_dtype(self.output_dtype)
if print_debug:
print("SET RES:", r0)
print("SET BAYER:", r1)
print("SET FPS:", r2)
print("SET CAPTURE MODE:", r3)
print("SET FRAME TYPE:", r4)
print("SET OUTPUT DTYPE:", r5)
def _begin_module(self, print_debug=True):
self.begin_resp = self.svc.begin(
frame_type=self.frame_type,
output_dtype=self.output_dtype,
capture_mode=self.capture_mode,
)
self.status = self.svc.get_status()
self.svc.validate_module_ready(
self.status,
self.frame_type,
self.raw_policy,
self.capture_mode,
)
if print_debug:
print("BEGIN:", self.begin_resp)
print("STATUS:", json.dumps({
"status": self.status.get("status"),
"detected_mode": self.status.get("detected_mode"),
"camera_count_active": self.status.get("camera_count_active"),
"active_camera_ids": self.status.get("active_camera_ids"),
}, ensure_ascii=False))
def _apply_module_params(self, print_debug=True):
if not self.module_calibration_json:
return None
self.applied_params = self.svc.apply_camera_params_json(
self.module_calibration_json
)
if print_debug:
print("[OK] Parâmetros do módulo aplicados:")
print(json.dumps(self.applied_params.get("applied"), ensure_ascii=False, indent=2))
return self.applied_params
def _start_stream(self, print_debug=True):
resp = self.svc.start_stream(
self.pc_host,
self.stream_port,
fps=self.fps,
)
if print_debug:
print("START STREAM:", resp)
return resp
def get_latest(self):
return self.receiver.last_frame, self.receiver.last_meta
def get_next_frame(self, timeout=2.0):
t0 = time.perf_counter()
while time.perf_counter() - t0 < timeout:
frame = self.receiver.last_frame
meta = self.receiver.last_meta
if frame is None or meta is None:
time.sleep(0.001)
continue
frame_id = meta.get("frame_id")
if frame_id != self.last_frame_id:
self.last_frame_id = frame_id
return frame, meta
time.sleep(0.001)
raise TimeoutError("Timeout aguardando novo frame do stream.")
def build_infer_tensor(self, frame, meta, channels_expected, target_size=None):
return self.core.build_infer_tensor_from_stream(
frame,
meta,
channels_expected=channels_expected,
target_size=target_size,
)
def build_preview_from_raw_payload(self, frame, meta: dict):
"""
Gera preview priorizando a câmera RGB (cam2).
Se cam2 não estiver presente, cai para fallback usando a primeira câmera mono disponível.
Retorna:
preview_bgr
payload_float_preview
preview_source_id
"""
payload_sources = meta.get("payload_sources", []) or []
# Caso multi-payload: tenta usar cam2 primeiro
if isinstance(frame, dict):
if "cam2" in frame:
rgb_frame = frame["cam2"]
if rgb_frame.ndim != 3 or rgb_frame.shape[2] != 3:
raise RuntimeError(f"cam2 recebida mas inválida para preview RGB: shape={rgb_frame.shape}")
preview_bgr = rgb_frame.copy()
payload_float = rgb_frame[:, :, ::-1].astype(np.float32) / 255.0
payload_float = np.transpose(payload_float, (2, 0, 1))
return preview_bgr, payload_float, "cam2"
# fallback: usa a primeira câmera mono disponível
fallback_id = None
for cid in ("cam0", "cam1"):
if cid in frame:
fallback_id = cid
break
if fallback_id is None:
raise RuntimeError("Nenhuma câmera disponível no payload para gerar preview")
packed = frame[fallback_id]
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
cam_frames = meta.get("camera_frames", {}) or {}
cam_meta = cam_frames.get(fallback_id, {})
bit_depth = int(cam_meta.get("bit_depth", 10))
raw16 = self.core.unpack_raw10_packed(packed)
preview_bgr = self.preview.raw16_to_preview_bgr(raw16, bit_depth=bit_depth)
payload_float = self.core.build_training_rgb(
raw16,
output_dtype="float32",
bit_depth=bit_depth,
)
return preview_bgr, payload_float, fallback_id
# Caso single-payload
if isinstance(frame, np.ndarray):
# Se vier HWC/3ch, tratamos como RGB USB
if frame.ndim == 3 and frame.shape[2] == 3:
preview_bgr = frame.copy()
payload_float = frame[:, :, ::-1].astype(np.float32) / 255.0
payload_float = np.transpose(payload_float, (2, 0, 1))
return preview_bgr, payload_float, "cam2"
# Se vier mono packed, fallback antigo
packed = frame
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
source_camera = meta.get("source_camera") or {}
bit_depth = int(source_camera.get("bit_depth", meta.get("source_bit_depth", 10)))
raw16 = self.core.unpack_raw10_packed(packed)
preview_bgr = self.preview.raw16_to_preview_bgr(raw16, bit_depth=bit_depth)
payload_float = self.core.build_training_rgb(
raw16,
output_dtype="float32",
bit_depth=bit_depth,
)
return preview_bgr, payload_float, source_camera.get("id", "unknown")
raise RuntimeError(f"Tipo de frame não suportado para preview: {type(frame)}")
def stop(self):
try:
self.svc.stop_stream()
except Exception:
pass
try:
self.svc.stop()
except Exception:
pass
try:
self.svc.disconnect()
except Exception:
pass
try:
self.receiver.stop()
except Exception:
pass

80
Python/OAK/datasets/multiespec_module/pi/raw_processor_core.py
View File

@ -1,3 +1,4 @@
import json
import os import os
import cv2 import cv2
import numpy as np import numpy as np
@ -6,7 +7,7 @@ from typing import Optional
class RawProcessorCore: class RawProcessorCore:
def __init__(self, sensor_width: int, sensor_height: int, bayer_pattern: str = "GBRG"): def __init__(self, sensor_width: int, sensor_height: int, bayer_pattern: str = "GBRG", calibration_json_path=None):
self.sensor_width = sensor_width self.sensor_width = sensor_width
self.sensor_height = sensor_height self.sensor_height = sensor_height
self.bayer_pattern = bayer_pattern.upper() self.bayer_pattern = bayer_pattern.upper()
@ -25,6 +26,8 @@ class RawProcessorCore:
"resize_after_crop": True, "resize_after_crop": True,
"target_size": None, # (w, h) ou None para manter o shape da RGB "target_size": None, # (w, h) ou None para manter o shape da RGB
} }
if calibration_json_path:
self.load_fusion_config_json(calibration_json_path)
def unpack_raw10_packed( def unpack_raw10_packed(
self, self,
@ -181,14 +184,22 @@ class RawProcessorCore:
return decoded return decoded
def build_multispectral_tensor(self, bins_data, bins_meta): def build_multispectral_tensor(self, bins_data, bins_meta, target_size=None):
decoded = self.decode_bins_cameras(bins_data, bins_meta) decoded = self.decode_bins_cameras(bins_data, bins_meta)
if "cam2" not in decoded: if "cam2" not in decoded:
raise RuntimeError("RGB obrigatório") raise RuntimeError("RGB obrigatório")
channel_names = self._channel_names_from_decoded(decoded) channel_names = self._channel_names_from_decoded(decoded)
tensor = self.fuse_multispec_cameras(decoded, meta=None, channels_expected=len(channel_names))
tensor = self.fuse_multispec_cameras(
decoded,
meta=None,
channels_expected=len(channel_names)
)
tensor = self.resize_tensor_chw(tensor, target_size=target_size)
return tensor, channel_names return tensor, channel_names
def build_infer_tensor_from_stream_old(self, frame, meta, channels_expected): def build_infer_tensor_from_stream_old(self, frame, meta, channels_expected):
@ -306,15 +317,17 @@ class RawProcessorCore:
raise RuntimeError(f"frame_type não suportado para inferência: {frame_type}") raise RuntimeError(f"frame_type não suportado para inferência: {frame_type}")
def build_infer_tensor_from_stream(self, frame, meta, channels_expected): def build_infer_tensor_from_stream(self, frame, meta, channels_expected, target_size=None):
frame_type = meta.get("frame_type") frame_type = meta.get("frame_type")
if frame_type == "RAW_BRUTO": if frame_type == "RAW_BRUTO":
decoded = self.decode_stream_cameras(frame, meta) decoded = self.decode_stream_cameras(frame, meta)
return self.fuse_multispec_cameras(decoded, meta, channels_expected) tensor = self.fuse_multispec_cameras(decoded, meta, channels_expected)
return self.resize_tensor_chw(tensor, target_size=target_size)
if frame_type in ("RGB", "MULTISPEC"): if frame_type in ("RGB", "MULTISPEC"):
return self.build_infer_tensor_from_stream_old(frame, meta, channels_expected) tensor = self.build_infer_tensor_from_stream_old(frame, meta, channels_expected)
return self.resize_tensor_chw(tensor, target_size=target_size)
raise RuntimeError(f"frame_type não suportado para inferência: {frame_type}") raise RuntimeError(f"frame_type não suportado para inferência: {frame_type}")
@ -472,16 +485,23 @@ class RawProcessorCore:
elif mode == "homography": elif mode == "homography":
H = cfg.get("homographies", {}).get(f"{cam_id}_to_cam2") H = cfg.get("homographies", {}).get(f"{cam_id}_to_cam2")
if H is None: if H is None:
warped = img warped = img
warped_mask = mask warped_mask = mask
else: else:
H = np.asarray(H, dtype=np.float32)
if H.shape != (3, 3):
raise RuntimeError(f"Homografia inválida para {cam_id}: shape={H.shape}")
warped = cv2.warpPerspective( warped = cv2.warpPerspective(
img, H, (ref_w, ref_h), img, H, (ref_w, ref_h),
flags=cv2.INTER_LINEAR, flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT, borderMode=cv2.BORDER_CONSTANT,
borderValue=0 borderValue=0
) )
warped_mask = cv2.warpPerspective( warped_mask = cv2.warpPerspective(
mask, H, (ref_w, ref_h), mask, H, (ref_w, ref_h),
flags=cv2.INTER_NEAREST, flags=cv2.INTER_NEAREST,
@ -544,6 +564,29 @@ class RawProcessorCore:
return resized return resized
def resize_tensor_chw(self, tensor, target_size=None):
if target_size is None:
return tensor
target_w, target_h = target_size
if tensor.ndim != 3:
raise RuntimeError(f"Tensor esperado em CHW. Veio shape={tensor.shape}")
_, h, w = tensor.shape
if (w, h) == (target_w, target_h):
return tensor.astype(np.float32, copy=False)
interp = cv2.INTER_AREA if target_w < w or target_h < h else cv2.INTER_LINEAR
chans = []
for ch in tensor:
ch_res = cv2.resize(ch, (target_w, target_h), interpolation=interp)
chans.append(ch_res.astype(np.float32))
return np.stack(chans, axis=0)
def extract_camera_meta(self, meta_json: dict, cam_id: str) -> dict: def extract_camera_meta(self, meta_json: dict, cam_id: str) -> dict:
cam_frames = meta_json.get("camera_frames", {}) or meta_json.get("stream_meta", {}).get("camera_frames", {}) cam_frames = meta_json.get("camera_frames", {}) or meta_json.get("stream_meta", {}).get("camera_frames", {})
@ -712,3 +755,28 @@ class RawProcessorCore:
f"Formato não suportado para salvar: channels={channels}, bit_depth={bit_depth}" f"Formato não suportado para salvar: channels={channels}, bit_depth={bit_depth}"
) )
def load_fusion_config_json(self, path: str):
if not path or not os.path.isfile(path):
raise FileNotFoundError(f"Arquivo de calibração não encontrado: {path}")
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
fusion = data.get("fusion_config")
if not isinstance(fusion, dict):
print("[WARN] JSON sem fusion_config. Mantendo config padrão.")
return
self.fusion_config = self._merge_fusion_config(self.fusion_config, fusion)
def _merge_fusion_config(self, default_cfg: dict, loaded_cfg: dict) -> dict:
cfg = json.loads(json.dumps(default_cfg))
for key, value in loaded_cfg.items():
if isinstance(value, dict) and isinstance(cfg.get(key), dict):
cfg[key].update(value)
else:
cfg[key] = value
return cfg

64
Python/raspi/build_module_params.py Normal file
View File

@ -0,0 +1,64 @@
import json
import argparse
from datetime import datetime
def now_str():
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def load_json(path):
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--camera_json", default="calibration/sensor_calibration.json")
parser.add_argument("--fusion_json", default="calibration/manual_offsets.json")
parser.add_argument("--out", default="calibration/module_params.json")
args = parser.parse_args()
cam_data = load_json(args.camera_json)
fusion_data = load_json(args.fusion_json)
camera_settings = cam_data.get("camera_settings")
if not isinstance(camera_settings, dict):
raise RuntimeError("camera_json sem camera_settings válido")
fusion_config = {
"alignment_mode": fusion_data.get("alignment_mode", "manual_affine"),
"baseline_mm": fusion_data.get("baseline_mm", 75.0),
"reference_camera": fusion_data.get("reference_camera", "cam2"),
"manual_offsets": fusion_data.get("manual_offsets", {}),
"homographies": fusion_data.get("homographies", {}),
"crop_valid_common": fusion_data.get("crop_valid_common", True),
"resize_after_crop": fusion_data.get("resize_after_crop", True),
"target_size": fusion_data.get("target_size", None),
}
module_params = {
"schema": "multispec_module_params_v1",
"saved_at": now_str(),
"frame_type": cam_data.get("frame_type", fusion_data.get("frame_type", "RAW_BRUTO")),
"capture_mode_requested": cam_data.get("capture_mode_requested", "AUTO"),
"capture_mode_effective": cam_data.get("capture_mode_effective", "AUTO"),
"raw_policy": cam_data.get("raw_policy", "allow_single"),
"sensor_width": cam_data.get("sensor_width", fusion_data.get("sensor_width")),
"sensor_height": cam_data.get("sensor_height", fusion_data.get("sensor_height")),
"bayer_pattern": cam_data.get("bayer_pattern", fusion_data.get("bayer_pattern", "GBRG")),
"camera_settings": camera_settings,
"fusion_config": fusion_config,
}
with open(args.out, "w", encoding="utf-8") as f:
json.dump(module_params, f, ensure_ascii=False, indent=2)
print(f"[OK] module_params gerado em: {args.out}")
if __name__ == "__main__":
main()

50
Python/raspi/calibration/manual_offsets.json Normal file
View File

@ -0,0 +1,50 @@
{
"schema": "manual_multispec_offsets_v1",
"saved_at": "2026-04-24 09:28:21",
"pi_host": "192.168.105.6",
"pc_host": "192.168.105.5",
"stream_port": 6001,
"frame_type": "RAW_BRUTO",
"capture_mode_requested": "AUTO",
"capture_mode_effective": "AUTO",
"raw_policy": "allow_single",
"sensor_width": 640,
"sensor_height": 480,
"bayer_pattern": "GBRG",
"reference_camera": "cam2",
"baseline_mm": 75.0,
"alignment_mode": "homography",
"manual_offsets": {
"cam0": {
"dx": 0,
"dy": 0,
"theta_deg": 0.0
},
"cam1": {
"dx": 0,
"dy": 0,
"theta_deg": 0.0
}
},
"homographies": {
"cam0_to_cam2": [
[
0.6438707381367006,
-0.44541142339026096,
171.26012619956475
],
[
-0.04083362402731768,
0.6855417219007448,
17.531834653386724
],
[
-0.00015145530359180688,
-0.0006687764363592821,
1.0
]
],
"cam1_to_cam2": null
},
"notes": ""
}

77
Python/raspi/calibration/module_params.json Normal file
View File

@ -0,0 +1,77 @@
{
"schema": "multispec_module_params_v1",
"saved_at": "2026-04-24 10:46:59",
"frame_type": "RAW_BRUTO",
"capture_mode_requested": "AUTO",
"capture_mode_effective": "AUTO",
"raw_policy": "allow_single",
"sensor_width": 640,
"sensor_height": 480,
"bayer_pattern": "GBRG",
"camera_settings": {
"cam0": {
"ae_enable": false,
"awb_enable": false,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": null
},
"cam1": {
"ae_enable": false,
"awb_enable": false,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": null
},
"cam2": {
"ae_enable": true,
"awb_enable": true,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": [
1.0,
1.0
]
}
},
"fusion_config": {
"alignment_mode": "homography",
"baseline_mm": 75.0,
"reference_camera": "cam2",
"manual_offsets": {
"cam0": {
"dx": 0,
"dy": 0,
"theta_deg": 0.0
},
"cam1": {
"dx": 0,
"dy": 0,
"theta_deg": 0.0
}
},
"homographies": {
"cam0_to_cam2": [
[
0.6438707381367006,
-0.44541142339026096,
171.26012619956475
],
[
-0.04083362402731768,
0.6855417219007448,
17.531834653386724
],
[
-0.00015145530359180688,
-0.0006687764363592821,
1.0
]
],
"cam1_to_cam2": null
},
"crop_valid_common": true,
"resize_after_crop": true,
"target_size": null
}
}

8143
Python/raspi/calibration/sensor_calibration.json
View File

File diff suppressed because it is too large Load Diff

824
Python/raspi/cam_3/capture_dataset.py
View File

@ -1,824 +0,0 @@
import os
import time
import json
import argparse
from datetime import datetime
import cv2
import numpy as np
from multispectral_service import MultiSpectralService
from stream_receiver import StreamReceiver
from pi.raw_processor_core import RawProcessorCore
from pi.raw_processor_preview import RawProcessorPreview
STREAM_PORT = 6001
PI_HOST = "192.168.105.6"
PC_HOST = "192.168.105.5"
# =========================
# Helpers gerais
# =========================
def ts_name() -> str:
return datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
def overlay_hud(
img_bgr: np.ndarray,
lines: list[str],
base_h: int = 720,
base_font_scale: float = 0.75,
base_line_step: int = 28,
):
h, w = img_bgr.shape[:2]
scale = h / float(base_h)
scale = max(scale, 0.4)
font_scale = base_font_scale * scale
line_step = int(base_line_step * scale)
thick_outline = max(1, int(3 * scale))
thick_text = max(1, int(2 * scale))
y = int(24 * scale)
x = int(12 * scale)
for s in lines:
cv2.putText(img_bgr, s, (x, y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 0), thick_outline, cv2.LINE_AA)
cv2.putText(img_bgr, s, (x, y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (255, 255, 255), thick_text, cv2.LINE_AA)
y += line_step
def save_sample(
base_dir: str,
frame_type: str,
preview_bgr: np.ndarray,
meta: dict,
raw_payload: np.ndarray | None = None,
packed_raw: np.ndarray | None = None,
packed_raw_by_camera: dict | None = None,
):
os.makedirs(base_dir, exist_ok=True)
name = ts_name()
png_path = os.path.join(base_dir, f"{name}.png")
json_path = os.path.join(base_dir, f"{name}.json")
if frame_type in ("RGB", "MULTISPEC"):
if raw_payload is None:
raise ValueError(f"raw_payload não pode ser None quando frame_type='{frame_type}'")
payload_path = os.path.join(base_dir, f"{name}.raw")
raw_payload.astype(np.float32).tofile(payload_path)
meta["saved_payload_type"] = frame_type.lower()
meta["saved_payload_path"] = os.path.basename(payload_path)
meta["saved_payload_dtype"] = "float32"
meta["saved_payload_shape"] = list(raw_payload.shape)
elif frame_type == "RAW_BRUTO":
if packed_raw_by_camera is not None:
payload_files = {}
payload_shapes = {}
payload_dtypes = {}
for cam_id, arr in packed_raw_by_camera.items():
path = os.path.join(base_dir, f"{name}_{cam_id}.bin")
arr.tofile(path)
payload_files[cam_id] = os.path.basename(path)
payload_shapes[cam_id] = list(arr.shape)
payload_dtypes[cam_id] = str(arr.dtype)
meta["saved_payload_type"] = "raw_native_multi"
meta["saved_payload_paths"] = payload_files
meta["saved_payload_shapes"] = payload_shapes
meta["saved_payload_dtypes"] = payload_dtypes
else:
if packed_raw is None:
raise ValueError("packed_raw não pode ser None quando frame_type='RAW_BRUTO'")
payload_path = os.path.join(base_dir, f"{name}.bin")
packed_raw.tofile(payload_path)
meta["saved_payload_type"] = "raw_native_single"
meta["saved_payload_path"] = os.path.basename(payload_path)
meta["saved_payload_dtype"] = str(packed_raw.dtype)
meta["saved_payload_shape"] = list(packed_raw.shape)
else:
raise ValueError(f"frame_type não suportado para save: {frame_type}")
cv2.imwrite(png_path, preview_bgr)
with open(json_path, "w", encoding="utf-8") as f:
json.dump(meta, f, ensure_ascii=False, indent=2)
return png_path, json_path
def get_camera_map_from_status(status: dict) -> dict:
result = {}
for cam in status.get("cameras", []):
result[cam.get("id")] = cam
return result
def validate_module_ready(status: dict, frame_type: str, raw_policy: str, capture_mode: str):
if not status.get("ok", True):
raise RuntimeError(f"Status inválido retornado pelo módulo: {status}")
active_ids = list(status.get("active_camera_ids", []))
active_count = int(status.get("camera_count_active", 0))
if frame_type == "RGB":
if "cam2" not in active_ids:
raise RuntimeError(
"Modo RGB requer cam2 ativa (USB RGB), mas o módulo não reportou cam2 como ativa."
)
return
if frame_type == "MULTISPEC":
if capture_mode == "TRIPLE":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"Modo MULTISPEC/TRIPLE requer cam0, cam1 e cam2 ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
return
if capture_mode == "DOUBLE":
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not has_rgb or not has_spec:
raise RuntimeError(
f"Modo MULTISPEC/DOUBLE requer cam2 + (cam0 ou cam1). "
f"Ativas atuais: {active_ids}"
)
return
# AUTO ou outros casos
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not (has_rgb and has_spec):
raise RuntimeError(
f"Modo MULTISPEC requer pelo menos RGB + 1 canal espectral. "
f"Ativas atuais: {active_ids}"
)
return
if frame_type == "RAW_BRUTO":
if raw_policy == "require_triple":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"RAW_BRUTO com política require_triple exige três câmeras ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
else:
if active_count < 1:
raise RuntimeError("RAW_BRUTO requer ao menos uma câmera ativa, mas nenhuma foi detectada.")
return
raise RuntimeError(f"frame_type desconhecido para validação: {frame_type}")
def build_preview_from_raw_payload(
frame,
meta: dict,
processor_core: RawProcessorCore,
processor_preview: RawProcessorPreview,
):
"""
Gera preview priorizando a câmera RGB (cam2).
Se cam2 não estiver presente, cai para fallback usando a primeira câmera mono disponível.
Retorna:
preview_bgr
payload_float_preview
preview_source_id
"""
payload_sources = meta.get("payload_sources", []) or []
# Caso multi-payload: tenta usar cam2 primeiro
if isinstance(frame, dict):
if "cam2" in frame:
rgb_frame = frame["cam2"]
if rgb_frame.ndim != 3 or rgb_frame.shape[2] != 3:
raise RuntimeError(f"cam2 recebida mas inválida para preview RGB: shape={rgb_frame.shape}")
preview_bgr = rgb_frame.copy()
payload_float = rgb_frame[:, :, ::-1].astype(np.float32) / 255.0
payload_float = np.transpose(payload_float, (2, 0, 1))
return preview_bgr, payload_float, "cam2"
# fallback: usa a primeira câmera mono disponível
fallback_id = None
for cid in ("cam0", "cam1"):
if cid in frame:
fallback_id = cid
break
if fallback_id is None:
raise RuntimeError("Nenhuma câmera disponível no payload para gerar preview")
packed = frame[fallback_id]
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
cam_frames = meta.get("camera_frames", {}) or {}
cam_meta = cam_frames.get(fallback_id, {})
bit_depth = int(cam_meta.get("bit_depth", 10))
raw16 = processor_core.unpack_raw10_packed(packed)
preview_bgr = processor_preview.raw16_to_preview_bgr(raw16, bit_depth=bit_depth)
payload_float = processor_core.build_training_rgb(
raw16,
output_dtype="float32",
bit_depth=bit_depth,
)
return preview_bgr, payload_float, fallback_id
# Caso single-payload
if isinstance(frame, np.ndarray):
# Se vier HWC/3ch, tratamos como RGB USB
if frame.ndim == 3 and frame.shape[2] == 3:
preview_bgr = frame.copy()
payload_float = frame[:, :, ::-1].astype(np.float32) / 255.0
payload_float = np.transpose(payload_float, (2, 0, 1))
return preview_bgr, payload_float, "cam2"
# Se vier mono packed, fallback antigo
packed = frame
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
source_camera = meta.get("source_camera") or {}
bit_depth = int(source_camera.get("bit_depth", meta.get("source_bit_depth", 10)))
raw16 = processor_core.unpack_raw10_packed(packed)
preview_bgr = processor_preview.raw16_to_preview_bgr(raw16, bit_depth=bit_depth)
payload_float = processor_core.build_training_rgb(
raw16,
output_dtype="float32",
bit_depth=bit_depth,
)
return preview_bgr, payload_float, source_camera.get("id", "unknown")
raise RuntimeError(f"Tipo de frame não suportado para preview: {type(frame)}")
def resolve_effective_capture_mode(frame_type: str, raw_policy: str, requested_mode: str) -> str:
"""
Decide o modo real que será pedido ao módulo.
Nova regra:
- Se o usuário pediu explicitamente SINGLE/DOUBLE/TRIPLE, respeitamos.
- Se pediu AUTO, deixamos AUTO ir para o módulo.
- A única exceção opcional é MULTISPEC + require_triple implícito,
mas mesmo assim podemos deixar o módulo resolver se preferirmos.
"""
if requested_mode in ("SINGLE", "DOUBLE", "TRIPLE"):
return requested_mode
# requested_mode == AUTO
return "AUTO"
# =========================
# MAIN
# =========================
def main():
parser = argparse.ArgumentParser(
description="Captura de dataset usando módulo multispectral Pi + StreamReceiver.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--cana", required=True, choices=["baixa", "media", "alta"], help="Estado da cana no momento da coleta.")
parser.add_argument("--horario", required=True, choices=["cedo", "meio_dia", "entardecer", "nublado"], help="Janela de iluminação / horário da coleta.")
parser.add_argument("--out_root", default="dataset", help="Pasta raiz do dataset.")
parser.add_argument("--modelo", default="imx296_pi", help="Nome do módulo/câmera para montar a pasta.")
parser.add_argument("--pi_host", default=PI_HOST, help="IP do servidor no Raspberry Pi.")
parser.add_argument("--pc_host", default=PC_HOST, help="IP local do notebook/PC que receberá o stream.")
parser.add_argument("--stream_port", type=int, default=STREAM_PORT, help="Porta TCP do receiver de stream.")
parser.add_argument("--server_port", type=int, default=5000, help="Porta TCP do servidor de comandos no Pi.")
parser.add_argument("--fps", type=int, default=20, help="FPS desejado.")
parser.add_argument("--width", type=int, default=640, help="Largura óptica da câmera.")
parser.add_argument("--height", type=int, default=480, help="Altura óptica da câmera.")
parser.add_argument("--interval", type=float, default=1.0, help="Intervalo em segundos para auto-save quando ligado.")
parser.add_argument("--preview_upscale", type=int, default=2, help="Fator de upscale visual do preview.")
parser.add_argument("--bayer", default="GBRG", choices=["GBRG", "GRBG", "RGGB", "BGGR"], help="Padrão Bayer das câmeras.")
parser.add_argument("--output_dtype", default="float32", choices=["uint8", "uint16", "float32"], help="Dtype do payload processado no Pi.")
parser.add_argument("--frame_type", default="RAW_BRUTO", choices=["RAW_BRUTO", "RGB", "MULTISPEC"], help="Tipo de payload pedido ao Pi.")
parser.add_argument("--capture_mode", default="AUTO", choices=["AUTO", "SINGLE", "DOUBLE", "TRIPLE"], help="Modo de captura desejado no módulo.")
parser.add_argument("--raw_policy", default="allow_single", choices=["allow_single", "require_triple"], help="Quando frame_type=RAW_BRUTO, define se o script aceita 1 câmera ou exige 3.")
args = parser.parse_args()
effective_capture_mode = resolve_effective_capture_mode(
frame_type=args.frame_type,
raw_policy=args.raw_policy,
requested_mode=args.capture_mode,
)
raw_w = args.width
raw_h = args.height
session_dir = os.path.join(
args.modelo,
args.out_root,
"brutas",
f"cana_{args.cana}",
args.horario,
datetime.now().strftime("%Y%m%d"),
)
os.makedirs(session_dir, exist_ok=True)
print("============================================")
print("Coleta de dataset - Módulo Multiespectral")
print(f"Cana : {args.cana}")
print(f"Horário : {args.horario}")
print(f"Saída : {session_dir}")
print(f"Sensor : {raw_w}x{raw_h} | Bayer={args.bayer}")
print(f"FrameType : {args.frame_type}")
print(f"CaptureMode : {args.capture_mode} -> efetivo={effective_capture_mode}")
print(f"RAW policy : {args.raw_policy}")
print("============================================")
auto_save = False
last_auto_t = 0.0
preview_upscale = args.preview_upscale
t_view_fps = time.time()
view_frames = 0
fps_view = 0.0
t_stream_fps = time.time()
last_stream_frame_id = None
stream_frames_accum = 0
fps_stream = 0.0
last_msg = ""
last_msg_t = 0.0
receiver = StreamReceiver(host="0.0.0.0", port=args.stream_port)
svc = MultiSpectralService(host=args.pi_host, port=args.server_port, timeout=10)
print(f"[INFO] Verificando conexão com o módulo em {args.pi_host}:{args.server_port}...")
svc.ensure_alive()
if not svc.is_alive():
raise RuntimeError("Módulo não encontrado ou não respondeu ao ping.")
print("[OK] Módulo conectado e respondendo.")
window_name = "Dataset Capture (C/SPACE=save | A=auto-save | M=preview scale | Q=quit)"
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
processor_core_cam0 = RawProcessorCore(
sensor_width=raw_w,
sensor_height=raw_h,
bayer_pattern=args.bayer,
)
processor_preview_cam0 = RawProcessorPreview(
sensor_width=raw_w,
sensor_height=raw_h,
bayer_pattern=args.bayer,
)
last_frame_id = -1
last_payload_float = None
last_packed_raw = None
last_packed_raw_by_camera = None
last_preview_bgr = None
last_meta_stream = None
try:
receiver.start()
time.sleep(0.5)
svc.connect()
if args.frame_type in ("RAW_BRUTO", "MULTISPEC"):
modes_resp = svc.get_sensor_modes()
if not modes_resp.get("ok"):
print(f"[WARN] Falha ao obter sensor_modes: {modes_resp}")
else:
for mode in modes_resp.get("sensor_modes", []):
print(
f"[cam={mode.get('camera_id')} mode={mode.get('mode_index')}] "
f"size={mode.get('size')} "
f"format={mode.get('format')} "
f"bit_depth={mode.get('bit_depth')} "
f"fps={mode.get('fps')}"
)
else:
print("[INFO] get_sensor_modes pulado para frame_type=RGB")
print("SET CAM0 RES:", svc.set_camera_resolution(0, raw_w, raw_h))
print("SET CAM1 RES:", svc.set_camera_resolution(1, raw_w, raw_h))
print("SET CAM2 RES:", svc.set_camera_resolution(2, raw_w, raw_h))
print("SET CAM0 BAYER:", svc.set_camera_bayer(0, args.bayer))
print("SET CAM1 BAYER:", svc.set_camera_bayer(1, args.bayer))
print("SET FPS:", svc.set_fps(args.fps))
print("SET CAPTURE MODE:", svc.set_capture_mode(effective_capture_mode))
print("SET FRAME TYPE:", svc.set_frame_type(args.frame_type))
print("SET OUTPUT DTYPE:", svc.set_output_dtype(args.output_dtype))
begin_resp = svc.begin(
frame_type=args.frame_type,
output_dtype=args.output_dtype,
capture_mode=effective_capture_mode,
)
print("BEGIN:", begin_resp)
status = svc.get_status()
print("STATUS:", json.dumps({
"status": status.get("status"),
"detected_mode": status.get("detected_mode"),
"camera_count_active": status.get("camera_count_active"),
"active_camera_ids": status.get("active_camera_ids"),
}, ensure_ascii=False))
validate_module_ready(status, args.frame_type, args.raw_policy, effective_capture_mode)
print("START STREAM:", svc.start_stream(args.pc_host, args.stream_port, fps=args.fps))
camera_ctrl = svc.get_camera_controls()
ae_enabled = bool(camera_ctrl.get("ae_enable", True))
awb_enabled = bool(camera_ctrl.get("awb_enable", True))
manual_exposure_us = camera_ctrl.get("exposure_time_us", None)
manual_gain = camera_ctrl.get("analogue_gain", None)
manual_colour_gains = camera_ctrl.get("colour_gains", None)
while True:
t0 = time.time()
meta = receiver.last_meta
frame = receiver.last_frame
if meta is not None and frame is not None and meta.get("frame_id") != last_frame_id:
last_frame_id = meta["frame_id"]
try:
frame_type = meta.get("frame_type", "RAW_BRUTO")
dtype_str = meta.get("dtype") or meta.get("output_dtype", "uint8")
preview_source_id = "cam2"
if frame_type == "RAW_BRUTO":
if isinstance(frame, dict):
packed_by_camera = frame
preview_bgr, raw3_preview, preview_source_id = build_preview_from_raw_payload(
frame=frame,
meta=meta,
processor_core=processor_core_cam0,
processor_preview=processor_preview_cam0,
)
last_packed_raw = None
last_packed_raw_by_camera = {cam_id: arr.copy() for cam_id, arr in packed_by_camera.items()}
last_payload_float = raw3_preview.copy()
else:
preview_bgr, raw3_preview, preview_source_id = build_preview_from_raw_payload(
frame=frame,
meta=meta,
processor_core=processor_core_cam0,
processor_preview=processor_preview_cam0,
)
last_packed_raw = frame.copy()
last_packed_raw_by_camera = None
last_payload_float = raw3_preview.copy()
elif frame_type == "RGB":
rgb_chw = frame
if not isinstance(rgb_chw, np.ndarray) or rgb_chw.ndim != 3:
raise RuntimeError(f"Frame RGB inválido: type={type(rgb_chw)}")
if dtype_str == "uint8":
payload_float = rgb_chw.astype(np.float32) / 255.0
elif dtype_str == "float32":
payload_float = rgb_chw.astype(np.float32)
elif dtype_str == "uint16":
payload_float = rgb_chw.astype(np.float32) / 65535.0
else:
raise RuntimeError(f"dtype RGB não suportado: {dtype_str}")
preview_rgb = np.transpose(payload_float, (1, 2, 0))
preview_bgr = cv2.cvtColor(
np.clip(preview_rgb * 255.0, 0, 255).astype(np.uint8),
cv2.COLOR_RGB2BGR
)
last_payload_float = payload_float.copy()
last_packed_raw = None
last_packed_raw_by_camera = None
elif frame_type == "MULTISPEC":
multispec_chw = frame
if not isinstance(multispec_chw, np.ndarray) or multispec_chw.ndim != 3 or multispec_chw.shape[0] not in (4, 5):
raise RuntimeError(f"Frame MULTISPEC inválido: shape={getattr(multispec_chw, 'shape', None)}")
if dtype_str == "uint8":
payload_float = multispec_chw.astype(np.float32) / 255.0
elif dtype_str == "float32":
payload_float = multispec_chw.astype(np.float32)
elif dtype_str == "uint16":
payload_float = multispec_chw.astype(np.float32) / 65535.0
else:
raise RuntimeError(f"dtype MULTISPEC não suportado: {dtype_str}")
preview_rgb = np.transpose(payload_float[:3], (1, 2, 0))
preview_bgr = cv2.cvtColor(
np.clip(preview_rgb * 255.0, 0, 255).astype(np.uint8),
cv2.COLOR_RGB2BGR
)
last_payload_float = payload_float.copy()
last_packed_raw = None
last_packed_raw_by_camera = None
else:
raise RuntimeError(f"frame_type não suportado neste script: {frame_type}")
if preview_upscale and preview_upscale > 1:
preview_show = cv2.resize(
preview_bgr,
(preview_bgr.shape[1] * preview_upscale, preview_bgr.shape[0] * preview_upscale),
interpolation=cv2.INTER_NEAREST,
)
else:
preview_show = preview_bgr.copy()
curr_frame_id = meta.get("frame_id")
if curr_frame_id is not None:
if last_stream_frame_id != curr_frame_id:
stream_frames_accum += 1
last_stream_frame_id = curr_frame_id
dt_stream = time.time() - t_stream_fps
if dt_stream >= 1.0:
fps_stream = stream_frames_accum / dt_stream
stream_frames_accum = 0
t_stream_fps = time.time()
view_frames += 1
dt_view = time.time() - t_view_fps
if dt_view >= 1.0:
fps_view = view_frames / dt_view
view_frames = 0
t_view_fps = time.time()
active_sources = meta.get("payload_sources")
lines = [
f"CANA: {args.cana} | HORA: {args.horario} | Pasta: {os.path.basename(session_dir)}",
f"Type={meta.get('frame_type')} | CaptureMode={effective_capture_mode} | RAW policy={args.raw_policy}",
f"Sources={active_sources} | FPS_STREAM={fps_stream:.1f} | FPS_VIEW={fps_view:.1f}",
f"frame_id={meta.get('frame_id')} | layout={meta.get('output_layout')} | dtype={meta.get('dtype') or meta.get('output_dtype')}",
f"codec={meta.get('codec_name', meta.get('codec_family', '-'))} | comp={meta.get('dt_comp', 0):.4f}s | send={meta.get('dt_send_payload_prev', 0):.4f}s",
f"AE={'ON' if ae_enabled else 'OFF'} | AWB={'ON' if awb_enabled else 'OFF'} | EXP={manual_exposure_us} | GAIN={manual_gain}",
"Keys: C/SPACE=save | A=auto-save | E=AE | W=AWB | I/K=exp | O/L=gain | R=reset | M=preview | Q/Esc=quit",
]
overlay_hud(preview_show, lines, base_h=raw_h)
if last_msg and (time.time() - last_msg_t) < 2.0:
cv2.putText(preview_show, last_msg, (12, preview_show.shape[0] - 18),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 2, cv2.LINE_AA)
cv2.imshow(window_name, preview_show)
last_preview_bgr = preview_bgr.copy()
last_meta_stream = dict(meta)
except Exception as e:
err = np.zeros((500, 1200, 3), dtype=np.uint8)
cv2.putText(err, f"Erro ao processar frame: {e}", (20, 60),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2, cv2.LINE_AA)
cv2.imshow(window_name, err)
print(f"[ERRO FRAME] {e}")
now = time.time()
can_save = (
last_meta_stream is not None and
last_preview_bgr is not None and
(
(last_meta_stream.get("frame_type") in ("RGB", "MULTISPEC") and last_payload_float is not None) or
(last_meta_stream.get("frame_type") == "RAW_BRUTO" and (last_packed_raw is not None or last_packed_raw_by_camera is not None))
)
)
if auto_save and can_save and (now - last_auto_t) >= args.interval:
frame_type_save = last_meta_stream.get("frame_type")
meta_save = {
"ts": datetime.now().isoformat(timespec="milliseconds"),
"cana": args.cana,
"horario": args.horario,
"sensor_width": raw_w,
"sensor_height": raw_h,
"bayer_pattern": args.bayer,
"fps_target": args.fps,
"frame_type": frame_type_save,
"capture_mode_requested": args.capture_mode,
"capture_mode_effective": effective_capture_mode,
"raw_policy": args.raw_policy,
"stream_meta": last_meta_stream,
"note": "autosave",
"raw_preview_reference_camera": preview_source_id,
}
save_sample(
session_dir,
frame_type=frame_type_save,
preview_bgr=last_preview_bgr,
meta=meta_save,
raw_payload=last_payload_float,
packed_raw=last_packed_raw,
packed_raw_by_camera=last_packed_raw_by_camera,
)
last_msg = "SALVO (auto)"
last_msg_t = now
last_auto_t = now
k = cv2.waitKey(1) & 0xFF
if k in (ord("q"), ord("Q"), 27):
break
elif k in (ord("a"), ord("A")):
auto_save = not auto_save
last_msg = f"AutoSave -> {'ON' if auto_save else 'OFF'}"
last_msg_t = time.time()
elif k in (ord("m"), ord("M")):
preview_upscale = 0 if preview_upscale else args.preview_upscale
last_msg = f"Preview UPSCALE -> {preview_upscale}"
last_msg_t = time.time()
elif k in (ord("e"), ord("E")):
ae_enabled = not ae_enabled
resp = svc.set_ae_enable(ae_enabled)
ae_enabled = bool(resp.get("ae_enable", ae_enabled))
last_msg = f"AE -> {'ON' if ae_enabled else 'OFF'}"
last_msg_t = time.time()
elif k in (ord("w"), ord("W")):
awb_enabled = not awb_enabled
resp = svc.set_awb_enable(awb_enabled)
awb_enabled = bool(resp.get("awb_enable", awb_enabled))
last_msg = f"AWB -> {'ON' if awb_enabled else 'OFF'}"
last_msg_t = time.time()
elif k in (ord("i"), ord("I")):
if manual_exposure_us is None:
manual_exposure_us = 15000
else:
manual_exposure_us = min(int(manual_exposure_us * 1.15), 200000)
if ae_enabled:
ae_enabled = False
svc.set_ae_enable(False)
resp = svc.set_exposure_time(int(manual_exposure_us))
manual_exposure_us = resp.get("exposure_time_us", manual_exposure_us)
last_msg = f"ExposureTime -> {manual_exposure_us} us"
last_msg_t = time.time()
elif k in (ord("k"), ord("K")):
if manual_exposure_us is None:
manual_exposure_us = 15000
else:
manual_exposure_us = max(int(manual_exposure_us / 1.15), 100)
if ae_enabled:
ae_enabled = False
svc.set_ae_enable(False)
resp = svc.set_exposure_time(int(manual_exposure_us))
manual_exposure_us = resp.get("exposure_time_us", manual_exposure_us)
last_msg = f"ExposureTime -> {manual_exposure_us} us"
last_msg_t = time.time()
elif k in (ord("o"), ord("O")):
if manual_gain is None:
manual_gain = 1.0
else:
manual_gain = min(float(manual_gain) * 1.10, 32.0)
if ae_enabled:
ae_enabled = False
svc.set_ae_enable(False)
resp = svc.set_analogue_gain(float(manual_gain))
manual_gain = resp.get("analogue_gain", manual_gain)
last_msg = f"AnalogueGain -> {manual_gain:.2f}"
last_msg_t = time.time()
elif k in (ord("l"), ord("L")):
if manual_gain is None:
manual_gain = 1.0
else:
manual_gain = max(float(manual_gain) / 1.10, 1.0)
if ae_enabled:
ae_enabled = False
svc.set_ae_enable(False)
resp = svc.set_analogue_gain(float(manual_gain))
manual_gain = resp.get("analogue_gain", manual_gain)
last_msg = f"AnalogueGain -> {manual_gain:.2f}"
last_msg_t = time.time()
elif k in (ord("r"), ord("R")):
svc.clear_exposure_time()
svc.clear_analogue_gain()
svc.clear_colour_gains()
manual_exposure_us = None
manual_gain = None
manual_colour_gains = None
last_msg = "Manual controls resetados"
last_msg_t = time.time()
elif k in (ord("c"), ord("C"), 32):
if can_save:
frame_type_save = last_meta_stream.get("frame_type")
meta_save = {
"ts": datetime.now().isoformat(timespec="milliseconds"),
"cana": args.cana,
"horario": args.horario,
"sensor_width": raw_w,
"sensor_height": raw_h,
"bayer_pattern": args.bayer,
"fps_target": args.fps,
"frame_type": frame_type_save,
"capture_mode_requested": args.capture_mode,
"capture_mode_effective": effective_capture_mode,
"raw_policy": args.raw_policy,
"stream_meta": last_meta_stream,
"camera_controls": {
"ae_enable": ae_enabled,
"awb_enable": awb_enabled,
"exposure_time_us": manual_exposure_us,
"analogue_gain": manual_gain,
"colour_gains": manual_colour_gains,
},
"note": "manual",
"raw_preview_reference_camera": preview_source_id,
}
save_sample(
session_dir,
frame_type=frame_type_save,
preview_bgr=last_preview_bgr,
meta=meta_save,
raw_payload=last_payload_float,
packed_raw=last_packed_raw,
packed_raw_by_camera=last_packed_raw_by_camera,
)
last_msg = "SALVO (manual)"
last_msg_t = time.time()
dt_loop = time.time() - t0
if dt_loop < 0.001:
time.sleep(0.001)
finally:
try:
print("STOP STREAM:", svc.stop_stream())
except Exception:
pass
try:
print("STOP:", svc.stop())
except Exception:
pass
svc.disconnect()
receiver.stop()
cv2.destroyAllWindows()
print("Fim da captura.")
if __name__ == "__main__":
main()

218
Python/raspi/cam_3/multispectral_client.py Normal file
View File

@ -0,0 +1,218 @@
import time
import json
from cam_3.multispectral_service import MultiSpectralService
from cam_3.stream_receiver import StreamReceiver
from cam_3.pi.raw_processor_core import RawProcessorCore
from cam_3.pi.raw_processor_preview import RawProcessorPreview
class MultiSpectralClient:
def __init__(
self,
pi_host="192.168.105.6",
pc_host="192.168.105.5",
server_port=5000,
stream_port=6001,
timeout=10,
width=640,
height=480,
bayer="GBRG",
fps=15,
frame_type="RAW_BRUTO",
output_dtype="uint8",
capture_mode="AUTO",
raw_policy="allow_single",
module_calibration_json=None,
):
self.pi_host = pi_host
self.pc_host = pc_host
self.server_port = server_port
self.stream_port = stream_port
self.width = width
self.height = height
self.bayer = bayer
self.fps = fps
self.frame_type = frame_type
self.output_dtype = output_dtype
self.capture_mode = capture_mode
self.raw_policy = raw_policy
self.module_calibration_json = module_calibration_json
self.svc = MultiSpectralService(
host=pi_host,
port=server_port,
timeout=timeout,
)
self.receiver = StreamReceiver(
host="0.0.0.0",
port=stream_port,
)
self.core = RawProcessorCore(
sensor_width=width,
sensor_height=height,
bayer_pattern=bayer,
calibration_json_path=module_calibration_json,
)
self.preview = RawProcessorPreview(
sensor_width=width,
sensor_height=height,
bayer_pattern=bayer,
)
self.last_frame_id = None
self.status = None
self.begin_resp = None
self.applied_params = None
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc, tb):
self.stop()
def start(self, print_debug=True):
if print_debug:
print(f"[INFO] Verificando módulo em {self.pi_host}:{self.server_port}...")
if not self.svc.check_connection(2):
raise RuntimeError("Módulo não encontrado ou não respondeu ao ping.")
self.receiver.start()
time.sleep(0.5)
self.svc.connect()
if print_debug:
print("[OK] Módulo conectado.")
self._configure_module(print_debug=print_debug)
self._begin_module(print_debug=print_debug)
self._apply_module_params(print_debug=print_debug)
self._start_stream(print_debug=print_debug)
return self
def _configure_module(self, print_debug=True):
r0 = self.svc.set_resolution(self.width, self.height)
r1 = self.svc.set_bayer(self.bayer)
r2 = self.svc.set_fps(self.fps)
r3 = self.svc.set_capture_mode(self.capture_mode)
r4 = self.svc.set_frame_type(self.frame_type)
r5 = self.svc.set_output_dtype(self.output_dtype)
if print_debug:
print("SET RES:", r0)
print("SET BAYER:", r1)
print("SET FPS:", r2)
print("SET CAPTURE MODE:", r3)
print("SET FRAME TYPE:", r4)
print("SET OUTPUT DTYPE:", r5)
def _begin_module(self, print_debug=True):
self.begin_resp = self.svc.begin(
frame_type=self.frame_type,
output_dtype=self.output_dtype,
capture_mode=self.capture_mode,
)
self.status = self.svc.get_status()
self.svc.validate_module_ready(
self.status,
self.frame_type,
self.raw_policy,
self.capture_mode,
)
if print_debug:
print("BEGIN:", self.begin_resp)
print("STATUS:", json.dumps({
"status": self.status.get("status"),
"detected_mode": self.status.get("detected_mode"),
"camera_count_active": self.status.get("camera_count_active"),
"active_camera_ids": self.status.get("active_camera_ids"),
}, ensure_ascii=False))
def _apply_module_params(self, print_debug=True):
if not self.module_calibration_json:
return None
self.applied_params = self.svc.apply_camera_params_json(
self.module_calibration_json
)
if print_debug:
print("[OK] Parâmetros do módulo aplicados:")
print(json.dumps(self.applied_params.get("applied"), ensure_ascii=False, indent=2))
return self.applied_params
def _start_stream(self, print_debug=True):
resp = self.svc.start_stream(
self.pc_host,
self.stream_port,
fps=self.fps,
)
if print_debug:
print("START STREAM:", resp)
return resp
def get_latest(self):
return self.receiver.last_frame, self.receiver.last_meta
def get_next_frame(self, timeout=2.0):
t0 = time.perf_counter()
while time.perf_counter() - t0 < timeout:
frame = self.receiver.last_frame
meta = self.receiver.last_meta
if frame is None or meta is None:
time.sleep(0.001)
continue
frame_id = meta.get("frame_id")
if frame_id != self.last_frame_id:
self.last_frame_id = frame_id
return frame, meta
time.sleep(0.001)
raise TimeoutError("Timeout aguardando novo frame do stream.")
def build_infer_tensor(self, frame, meta, channels_expected, target_size=None):
return self.core.build_infer_tensor_from_stream(
frame,
meta,
channels_expected=channels_expected,
target_size=target_size,
)
def stop(self):
try:
self.svc.stop_stream()
except Exception:
pass
try:
self.svc.stop()
except Exception:
pass
try:
self.svc.disconnect()
except Exception:
pass
try:
self.receiver.stop()
except Exception:
pass

96
Python/raspi/cam_3/multispectral_service.py
View File

@ -95,6 +95,68 @@ class MultiSpectralService:
return json.loads(line.strip()) return json.loads(line.strip())
def validate_module_ready(self, status: dict, frame_type: str, raw_policy: str, capture_mode: str):
if not status.get("ok", True):
raise RuntimeError(f"Status inválido retornado pelo módulo: {status}")
active_ids = list(status.get("active_camera_ids", []))
active_count = int(status.get("camera_count_active", 0))
if frame_type == "RGB":
if "cam2" not in active_ids:
raise RuntimeError(
"Modo RGB requer cam2 ativa (USB RGB), mas o módulo não reportou cam2 como ativa."
)
return
if frame_type == "MULTISPEC":
if capture_mode == "TRIPLE":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"Modo MULTISPEC/TRIPLE requer cam0, cam1 e cam2 ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
return
if capture_mode == "DOUBLE":
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not has_rgb or not has_spec:
raise RuntimeError(
f"Modo MULTISPEC/DOUBLE requer cam2 + (cam0 ou cam1). "
f"Ativas atuais: {active_ids}"
)
return
# AUTO ou outros casos
has_rgb = "cam2" in active_ids
has_spec = ("cam0" in active_ids) or ("cam1" in active_ids)
if not (has_rgb and has_spec):
raise RuntimeError(
f"Modo MULTISPEC requer pelo menos RGB + 1 canal espectral. "
f"Ativas atuais: {active_ids}"
)
return
if frame_type == "RAW_BRUTO":
if raw_policy == "require_triple":
missing = [cid for cid in ("cam0", "cam1", "cam2") if cid not in active_ids]
if missing:
raise RuntimeError(
f"RAW_BRUTO com política require_triple exige três câmeras ativas. "
f"Faltando: {missing}. Ativas atuais: {active_ids}"
)
else:
if active_count < 1:
raise RuntimeError("RAW_BRUTO requer ao menos uma câmera ativa, mas nenhuma foi detectada.")
return
raise RuntimeError(f"frame_type desconhecido para validação: {frame_type}")
# ========================================================= # =========================================================
# Helpers numpy # Helpers numpy
# ========================================================= # =========================================================
@ -259,13 +321,20 @@ class MultiSpectralService:
def get_config(self): def get_config(self):
return self._send_command({"cmd": "get_config"}) return self._send_command({"cmd": "get_config"})
def begin(self, frame_type: str = "RAW_BRUTO", output_dtype: str = "uint8", capture_mode: str = "AUTO"): def begin(self, frame_type="RAW_BRUTO", output_dtype="uint8", capture_mode="AUTO", timeout=15):
old_timeout = self.sock.gettimeout() if self.sock else None
if self.sock and timeout is not None:
self.sock.settimeout(timeout)
try:
return self._send_command({ return self._send_command({
"cmd": "begin", "cmd": "begin",
"frame_type": frame_type, "frame_type": frame_type,
"output_dtype": output_dtype, "output_dtype": output_dtype,
"capture_mode": capture_mode, "capture_mode": capture_mode,
}) })
finally:
if self.sock and old_timeout is not None:
self.sock.settimeout(old_timeout)
def stop(self): def stop(self):
return self._send_command({"cmd": "stop"}) return self._send_command({"cmd": "stop"})
@ -307,6 +376,31 @@ class MultiSpectralService:
"height": height "height": height
}) })
def set_resolution(self, width: int, height: int):
res = []
for index in range(0, 3):
res.append(
self._send_command({
"cmd": "set_camera_resolution",
"index": index,
"width": width,
"height": height
})
)
return res
def set_bayer(self, bayer_pattern: str):
res = []
for index in range(0, 2):
res.append(
self._send_command({
"cmd": "set_camera_bayer",
"index": index,
"pattern": bayer_pattern
})
)
return res
# ========================================================= # =========================================================
# Captura # Captura
# ========================================================= # =========================================================

106
Python/raspi/cam_3/pi/camera_manager.py
View File

@ -6,6 +6,7 @@ import numpy as np
import cv2 import cv2
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
import subprocess
class CameraManager: class CameraManager:
@ -87,6 +88,9 @@ class CameraManager:
if not required_ids: if not required_ids:
print("[WARN] Nenhuma câmera requerida para o frame_type/capture_mode atual") print("[WARN] Nenhuma câmera requerida para o frame_type/capture_mode atual")
for cam in self.state.cameras:
cam.attempted = False
cams_to_open = [cam for cam in self.state.cameras if cam.id in required_ids] cams_to_open = [cam for cam in self.state.cameras if cam.id in required_ids]
cams_to_open.sort( cams_to_open.sort(
@ -94,6 +98,8 @@ class CameraManager:
) )
for cam in cams_to_open: for cam in cams_to_open:
cam.attempted = True
if cam.id not in required_ids: if cam.id not in required_ids:
self.state.set_camera_connected(cam.index, False) self.state.set_camera_connected(cam.index, False)
continue continue
@ -128,7 +134,7 @@ class CameraManager:
deadline = time.perf_counter() + 1.0 deadline = time.perf_counter() + 1.0
while time.perf_counter() < deadline: while time.perf_counter() < deadline:
if self.state.camera_count_active > 0: if self.state.camera_count_active > 2:
break break
time.sleep(0.02) time.sleep(0.02)
@ -173,6 +179,9 @@ class CameraManager:
api_preference = cv2.CAP_V4L2 if backend_name == "V4L2" else cv2.CAP_ANY api_preference = cv2.CAP_V4L2 if backend_name == "V4L2" else cv2.CAP_ANY
source = self._resolve_usb_video_path(cam) source = self._resolve_usb_video_path(cam)
device = source or f"/dev/video{cam.index}"
runtime["device"] = device
if source: if source:
cap = cv2.VideoCapture(source, api_preference) cap = cv2.VideoCapture(source, api_preference)
else: else:
@ -189,24 +198,20 @@ class CameraManager:
f"Falha ao abrir câmera USB. device_path={getattr(cam, 'device_path', None)} index={cam.index}" f"Falha ao abrir câmera USB. device_path={getattr(cam, 'device_path', None)} index={cam.index}"
) )
# Configuração desejada device = source or f"/dev/video{cam.index}"
self._v4l2_set_ctrls(device, {
"exposure_dynamic_framerate": 0,
"auto_exposure": 3,
"white_balance_automatic": 1,
"gain": 0,
})
# Configura formato primeiro
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))
cap.set(cv2.CAP_PROP_FRAME_WIDTH, cam.width) cap.set(cv2.CAP_PROP_FRAME_WIDTH, cam.width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, cam.height) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, cam.height)
cap.set(cv2.CAP_PROP_FPS, float(self.state.fps)) cap.set(cv2.CAP_PROP_FPS, float(self.state.fps))
# Tenta reduzir buffer
try:
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
except Exception:
pass
# Tenta MJPG, se a câmera suportar
try:
fourcc = cv2.VideoWriter_fourcc(*"MJPG")
cap.set(cv2.CAP_PROP_FOURCC, fourcc)
except Exception:
pass
# Pequeno warmup # Pequeno warmup
time.sleep(0.25) time.sleep(0.25)
@ -512,39 +517,64 @@ class CameraManager:
cam = self.state.get_camera(camera_id) cam = self.state.get_camera(camera_id)
ctrl = self.state.get_camera_controls(camera_id) ctrl = self.state.get_camera_controls(camera_id)
# FPS device = runtime.get("device")
try: if not device:
cap.set(cv2.CAP_PROP_FPS, float(self.state.fps)) device = getattr(cam, "device_path", None) if cam is not None else None
except Exception: if not device:
pass device = f"/dev/video{runtime['camera_index']}"
# Exposição # Sempre protege FPS
try: self._v4l2_set_ctrls(device, {
"exposure_dynamic_framerate": 0,
})
# AE
if ctrl.ae_enable: if ctrl.ae_enable:
# automático em muitos backends V4L2 self._v4l2_set_ctrls(device, {
cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.75) "auto_exposure": 3,
})
else: else:
cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25) self._v4l2_set_ctrls(device, {
except Exception: "auto_exposure": 1,
pass })
if not ctrl.ae_enable and ctrl.exposure_time_us is not None: if ctrl.exposure_time_us is not None:
try: # V4L2 exposure_time_absolute é em unidades de 100 us
cap.set(cv2.CAP_PROP_EXPOSURE, float(ctrl.exposure_time_us)) exp_abs = int(ctrl.exposure_time_us / 100)
except Exception: exp_abs = max(1, min(5000, exp_abs))
pass
self._v4l2_set_ctrls(device, {
"exposure_time_absolute": exp_abs,
})
# Ganho # Ganho
if ctrl.analogue_gain is not None: if ctrl.analogue_gain is not None:
try: gain = int(max(0, min(100, ctrl.analogue_gain)))
cap.set(cv2.CAP_PROP_GAIN, float(ctrl.analogue_gain)) self._v4l2_set_ctrls(device, {
except Exception: "gain": gain,
})
# AWB
if cam is not None and cam.role == "rgb":
self._v4l2_set_ctrls(device, {
"white_balance_automatic": 1 if ctrl.awb_enable else 0,
})
if not ctrl.awb_enable and ctrl.colour_gains is not None:
# Aqui não dá para aplicar r_gain/b_gain diretamente nessa webcam.
# Ela só tem white_balance_temperature.
pass pass
# AWB / WB - só RGB cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))
if cam is not None and cam.role == "rgb": cap.set(cv2.CAP_PROP_FPS, float(self.state.fps))
def _v4l2_set_ctrls(self, device, controls: dict):
args = ["v4l2-ctl", "-d", device]
for k, v in controls.items():
args += ["-c", f"{k}={v}"]
try: try:
cap.set(cv2.CAP_PROP_AUTO_WB, 1 if ctrl.awb_enable else 0) subprocess.run(args, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except Exception: except Exception:
pass pass

628
Python/raspi/cam_3/pi/raw_processor_core.py
View File

@ -1,13 +1,33 @@
import json
import os
import cv2
import numpy as np import numpy as np
import math import math
from typing import Optional from typing import Optional
class RawProcessorCore: class RawProcessorCore:
def __init__(self, sensor_width: int, sensor_height: int, bayer_pattern: str = "GBRG"): def __init__(self, sensor_width: int, sensor_height: int, bayer_pattern: str = "GBRG", calibration_json_path=None):
self.sensor_width = sensor_width self.sensor_width = sensor_width
self.sensor_height = sensor_height self.sensor_height = sensor_height
self.bayer_pattern = bayer_pattern.upper() self.bayer_pattern = bayer_pattern.upper()
self.fusion_config = {
"alignment_mode": "manual_affine", # identity | manual_offset | manual_affine | homography
"baseline_mm": 75.0,
"manual_offsets": {
"cam0": {"dx": 0, "dy": 0, "theta_deg": 0.0},
"cam1": {"dx": 0, "dy": 0, "theta_deg": 0.0},
},
"homographies": {
"cam0_to_cam2": None,
"cam1_to_cam2": None,
},
"crop_valid_common": True,
"resize_after_crop": True,
"target_size": None, # (w, h) ou None para manter o shape da RGB
}
if calibration_json_path:
self.load_fusion_config_json(calibration_json_path)
def unpack_raw10_packed( def unpack_raw10_packed(
self, self,
@ -21,6 +41,9 @@ class RawProcessorCore:
width = sensor_width if sensor_width is not None else self.sensor_width width = sensor_width if sensor_width is not None else self.sensor_width
height = sensor_height if sensor_height is not None else self.sensor_height height = sensor_height if sensor_height is not None else self.sensor_height
if width % 4 != 0:
raise ValueError(f"Largura {width} não é múltipla de 4 para RAW10 packed")
expected_packed_width = math.ceil(width * 10 / 8) expected_packed_width = math.ceil(width * 10 / 8)
actual_h, actual_w = packed_frame.shape[:2] actual_h, actual_w = packed_frame.shape[:2]
@ -121,3 +144,606 @@ class RawProcessorCore:
return (chw * 65535.0).clip(0, 65535).astype(np.uint16) return (chw * 65535.0).clip(0, 65535).astype(np.uint16)
raise ValueError(f"output_dtype não suportado: {output_dtype}") raise ValueError(f"output_dtype não suportado: {output_dtype}")
def _channel_names_from_decoded(self, decoded):
names = ["R", "G", "B"]
if "cam0" in decoded:
names.append("RE")
if "cam1" in decoded:
names.append("NIR")
return names
def decode_bins_cameras(self, bins_data, bins_meta):
decoded = {}
for data, meta in zip(bins_data, bins_meta):
role = (meta.get("role") or "").strip().lower()
bit_depth = int(meta.get("bit_depth", 10))
max_val = float((1 << bit_depth) - 1)
if role == "rgb":
decoded["cam2"] = {
"name": "RGB",
"image": data.astype(np.float32) / max_val,
"meta": meta,
}
elif role == "re":
decoded["cam0"] = {
"name": "RE",
"image": data.astype(np.float32) / max_val,
"meta": meta,
}
elif role == "nir":
decoded["cam1"] = {
"name": "NIR",
"image": data.astype(np.float32) / max_val,
"meta": meta,
}
return decoded
def build_multispectral_tensor(self, bins_data, bins_meta):
decoded = self.decode_bins_cameras(bins_data, bins_meta)
if "cam2" not in decoded:
raise RuntimeError("RGB obrigatório")
channel_names = self._channel_names_from_decoded(decoded)
tensor = self.fuse_multispec_cameras(decoded, meta=None, channels_expected=len(channel_names))
return tensor, channel_names
def build_infer_tensor_from_stream_old(self, frame, meta, channels_expected):
"""
Converte o frame vindo do stream do Pi em tensor (C,H,W) float32 0..1
compatível com o modelo.
Suporta:
- RGB uint8/float32 já pronto
- MULTISPEC uint8/float32 já pronto
- RAW_BRUTO multi_payload (cam2 RGB + cam0/cam1 packed)
"""
frame_type = meta.get("frame_type")
dtype_str = meta.get("dtype") or meta.get("output_dtype", "uint8")
camera_frames = meta.get("camera_frames", {}) or {}
# -------------------------------------------------
# RAW_BRUTO multi_payload
# -------------------------------------------------
if frame_type == "RAW_BRUTO":
if not isinstance(frame, dict):
raise RuntimeError("RAW_BRUTO esperado como dict de câmeras no modo multi")
arrays = []
channel_names = []
# RGB USB
if "cam2" in frame:
rgb_bgr = frame["cam2"]
if rgb_bgr.ndim != 3 or rgb_bgr.shape[2] != 3:
raise RuntimeError(f"cam2 RGB inválida: shape={rgb_bgr.shape}")
rgb = rgb_bgr[:, :, ::-1].astype(np.float32) / 255.0
rgb_chw = np.transpose(rgb, (2, 0, 1))
arrays.append(rgb_chw)
channel_names.extend(["R", "G", "B"])
else:
raise RuntimeError("RAW_BRUTO para inferência precisa incluir cam2 (RGB)")
# RE / NIR
for cam_id, spec_name in (("cam0", "RE"), ("cam1", "NIR")):
if cam_id not in frame:
continue
packed = frame[cam_id]
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
cam_meta = camera_frames.get(cam_id, {})
packed_width = int(cam_meta.get("width", packed.shape[1]))
height = int(cam_meta.get("height", packed.shape[0]))
bayer = cam_meta.get("bayer_pattern", self.bayer_pattern)
bit_depth = int(cam_meta.get("bit_depth", 10))
if bit_depth == 10:
real_width = int((packed_width * 8) / 10)
else:
real_width = packed_width
rp = RawProcessorCore(
sensor_width=real_width,
sensor_height=height,
bayer_pattern=bayer,
)
raw16 = rp.unpack_raw10_packed(packed)
max_val = float((1 << bit_depth) - 1)
single = np.clip(raw16.astype(np.float32) / max_val, 0.0, 1.0)[None, :, :]
arrays.append(single)
channel_names.append(spec_name)
if len(arrays) < 2:
raise RuntimeError("RAW_BRUTO requer RGB + pelo menos um canal espectral para inferência")
min_h = min(a.shape[1] for a in arrays)
min_w = min(a.shape[2] for a in arrays)
arrays = [a[:, :min_h, :min_w] for a in arrays]
raw_np = np.concatenate(arrays, axis=0)
if raw_np.shape[0] != channels_expected:
raise RuntimeError(
f"Tensor RAW_BRUTO montado com canais inesperados: {raw_np.shape[0]} | esperado={channels_expected} | got={channel_names}"
)
return raw_np
# -------------------------------------------------
# RGB ou MULTISPEC já pronto
# -------------------------------------------------
if frame_type == "RGB" or frame_type == "MULTISPEC":
if not isinstance(frame, np.ndarray):
raise RuntimeError(f"Frame {frame_type} esperado como ndarray")
if frame.ndim != 3:
raise RuntimeError(f"Frame {frame_type} inválido: shape={frame.shape}")
if dtype_str == "uint8":
raw_np = frame.astype(np.float32) / 255.0
elif dtype_str == "float32":
raw_np = frame.astype(np.float32)
elif dtype_str == "uint16":
raw_np = frame.astype(np.float32) / 65535.0
else:
raise RuntimeError(f"dtype {frame_type} não suportado: {dtype_str}")
if raw_np.shape[0] != channels_expected:
raise RuntimeError(
f"Frame {frame_type} com canais inesperados: {raw_np.shape[0]} | esperado={channels_expected}"
)
return raw_np
raise RuntimeError(f"frame_type não suportado para inferência: {frame_type}")
def build_infer_tensor_from_stream(self, frame, meta, channels_expected):
frame_type = meta.get("frame_type")
if frame_type == "RAW_BRUTO":
decoded = self.decode_stream_cameras(frame, meta)
return self.fuse_multispec_cameras(decoded, meta, channels_expected)
if frame_type in ("RGB", "MULTISPEC"):
return self.build_infer_tensor_from_stream_old(frame, meta, channels_expected)
raise RuntimeError(f"frame_type não suportado para inferência: {frame_type}")
def decode_stream_cameras(self, frame, meta):
if not isinstance(frame, dict):
raise RuntimeError("RAW_BRUTO esperado como dict de câmeras no modo multi")
camera_frames = meta.get("camera_frames", {}) or {}
decoded = {}
if "cam2" in frame:
rgb_bgr = frame["cam2"]
if rgb_bgr.ndim != 3 or rgb_bgr.shape[2] != 3:
raise RuntimeError(f"cam2 RGB inválida: shape={rgb_bgr.shape}")
rgb = rgb_bgr[:, :, ::-1].astype(np.float32) / 255.0
decoded["cam2"] = {
"name": "RGB",
"image": rgb,
"meta": camera_frames.get("cam2", {})
}
for cam_id, spec_name in (("cam0", "RE"), ("cam1", "NIR")):
if cam_id not in frame:
continue
packed = frame[cam_id]
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
cam_meta = camera_frames.get(cam_id, {})
packed_width = int(cam_meta.get("width", packed.shape[1]))
height = int(cam_meta.get("height", packed.shape[0]))
bayer = cam_meta.get("bayer_pattern", self.bayer_pattern)
bit_depth = int(cam_meta.get("bit_depth", 10))
real_width = int((packed_width * 8) / 10) if bit_depth == 10 else packed_width
rp = RawProcessorCore(
sensor_width=real_width,
sensor_height=height,
bayer_pattern=bayer,
)
raw16 = rp.unpack_raw10_packed(packed)
max_val = float((1 << bit_depth) - 1)
single = np.clip(raw16.astype(np.float32) / max_val, 0.0, 1.0)
decoded[cam_id] = {
"name": spec_name,
"image": single,
"meta": cam_meta
}
return decoded
def fuse_multispec_cameras(self, decoded, meta, channels_expected):
if "cam2" not in decoded:
raise RuntimeError("Fusão requer cam2 (RGB) como referência")
rgb = decoded["cam2"]["image"]
h, w = rgb.shape[:2]
rgb_chw = np.transpose(rgb, (2, 0, 1))
channels = [rgb_chw]
names = ["R", "G", "B"]
valid_masks = [np.ones((h, w), dtype=np.uint8)]
for cam_id, ch_name in (("cam0", "RE"), ("cam1", "NIR")):
if cam_id not in decoded:
continue
img = decoded[cam_id]["image"]
aligned, valid_mask = self._warp_with_valid_mask(img, cam_id, (h, w), meta)
channels.append(aligned[None, :, :])
names.append(ch_name)
valid_masks.append(valid_mask)
cfg = self.fusion_config
if cfg.get("crop_valid_common", False):
crop_box = self._compute_common_crop_box(valid_masks)
if crop_box is not None:
channels = self._crop_and_resize_channels(channels, crop_box, (h, w))
tensor = np.concatenate(channels, axis=0)
if tensor.shape[0] != channels_expected:
raise RuntimeError(
f"Tensor fundido com canais inesperados: {tensor.shape[0]} | "
f"esperado={channels_expected} | got={names}"
)
return tensor.astype(np.float32, copy=False)
def _shift_image(self, img, dx, dy):
h, w = img.shape[:2]
M = np.float32([[1, 0, dx], [0, 1, dy]])
return cv2.warpAffine(
img, M, (w, h),
flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=0
)
def _affine_image(self, img, dx, dy, theta_deg):
h, w = img.shape[:2]
center = (w * 0.5, h * 0.5)
M = cv2.getRotationMatrix2D(center, theta_deg, 1.0)
M[0, 2] += dx
M[1, 2] += dy
return cv2.warpAffine(
img,
M,
(w, h),
flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=0
)
def _warp_with_valid_mask(self, img, cam_id, ref_shape, meta):
ref_h, ref_w = ref_shape
if img.shape[:2] != (ref_h, ref_w):
img = cv2.resize(img, (ref_w, ref_h), interpolation=cv2.INTER_LINEAR)
cfg = self.fusion_config
mode = cfg.get("alignment_mode", "identity")
mask = np.ones((ref_h, ref_w), dtype=np.uint8) * 255
if mode == "identity":
warped = img
warped_mask = mask
elif mode == "manual_offset":
offs = cfg.get("manual_offsets", {}).get(cam_id, {})
dx = int(offs.get("dx", 0))
dy = int(offs.get("dy", 0))
warped = self._shift_image(img, dx, dy)
warped_mask = self._shift_image(mask, dx, dy)
elif mode == "manual_affine":
offs = cfg.get("manual_offsets", {}).get(cam_id, {})
dx = int(offs.get("dx", 0))
dy = int(offs.get("dy", 0))
theta_deg = float(offs.get("theta_deg", 0.0))
warped = self._affine_image(img, dx, dy, theta_deg)
warped_mask = self._affine_image(mask, dx, dy, theta_deg)
elif mode == "homography":
H = cfg.get("homographies", {}).get(f"{cam_id}_to_cam2")
if H is None:
warped = img
warped_mask = mask
else:
H = np.asarray(H, dtype=np.float32)
if H.shape != (3, 3):
raise RuntimeError(f"Homografia inválida para {cam_id}: shape={H.shape}")
warped = cv2.warpPerspective(
img, H, (ref_w, ref_h),
flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=0
)
warped_mask = cv2.warpPerspective(
mask, H, (ref_w, ref_h),
flags=cv2.INTER_NEAREST,
borderMode=cv2.BORDER_CONSTANT,
borderValue=0
)
else:
raise RuntimeError(f"alignment_mode inválido: {mode}")
warped_mask = (warped_mask > 0).astype(np.uint8)
return warped, warped_mask
def _compute_common_crop_box(self, masks):
if not masks:
return None
common = masks[0].copy()
for m in masks[1:]:
common = np.logical_and(common > 0, m > 0)
ys, xs = np.where(common)
if len(xs) == 0 or len(ys) == 0:
return None
x0 = int(xs.min())
x1 = int(xs.max()) + 1
y0 = int(ys.min())
y1 = int(ys.max()) + 1
return x0, y0, x1, y1
def _crop_and_resize_channels(self, channels, crop_box, ref_shape):
x0, y0, x1, y1 = crop_box
ref_h, ref_w = ref_shape
cropped = [ch[:, y0:y1, x0:x1] for ch in channels]
cfg = self.fusion_config
if not cfg.get("resize_after_crop", False):
return cropped
target_size = cfg.get("target_size", None)
if target_size is None:
target_w, target_h = ref_w, ref_h
else:
target_w, target_h = target_size
resized = []
for ch in cropped:
ch_resized = np.stack([
cv2.resize(
ch_i,
(target_w, target_h),
interpolation=cv2.INTER_LINEAR
)
for ch_i in ch
], axis=0)
resized.append(ch_resized)
return resized
def extract_camera_meta(self, meta_json: dict, cam_id: str) -> dict:
cam_frames = meta_json.get("camera_frames", {}) or meta_json.get("stream_meta", {}).get("camera_frames", {})
cam = cam_frames.get(cam_id)
if not cam:
raise ValueError(f"Camera {cam_id} não encontrada no meta")
# Detecta RAW10 packed mono
if int(cam.get("channels", 1)) == 1 and int(cam.get("bit_depth", 10)) == 10:
packed_width = int(cam.get("width"))
height = int(cam.get("height"))
# 🔥 converte packed → real
real_width = int((packed_width * 8) / 10)
return {
"camera_id": cam_id,
"width": real_width,
"height": height,
"channels": 1,
"bit_depth": 10,
"shape": [height, packed_width], # packed shape
"role": cam.get("role")
}
# RGB
else:
width = int(cam.get("width"))
height = int(cam.get("height"))
channels = int(cam.get("channels", 3))
return {
"camera_id": cam_id,
"width": width,
"height": height,
"channels": channels,
"bit_depth": int(cam.get("bit_depth", 8)),
"shape": [height, width, channels], # 🔥 AQUI está a correção
"role": cam.get("role")
}
# ============================================================
# RAW10 PACKED
# ============================================================
def packed_width_for_raw10(self, sensor_width: int = None) -> int:
width = sensor_width if sensor_width is not None else self.sensor_width
return math.ceil(width * 10 / 8)
def pack_raw10_packed(self, raw16: np.ndarray) -> np.ndarray:
h, w = raw16.shape
if w % 4 != 0:
raise ValueError(f"Width precisa ser múltiplo de 4 para pack otimizado. Veio {w}")
raw16 = np.clip(raw16, 0, 1023).astype(np.uint16)
p0 = raw16[:, 0::4]
p1 = raw16[:, 1::4]
p2 = raw16[:, 2::4]
p3 = raw16[:, 3::4]
b0 = (p0 >> 2).astype(np.uint8)
b1 = (p1 >> 2).astype(np.uint8)
b2 = (p2 >> 2).astype(np.uint8)
b3 = (p3 >> 2).astype(np.uint8)
b4 = (
((p0 & 0x03) << 0) |
((p1 & 0x03) << 2) |
((p2 & 0x03) << 4) |
((p3 & 0x03) << 6)
).astype(np.uint8)
packed = np.empty((h, w // 4, 5), dtype=np.uint8)
packed[:, :, 0] = b0
packed[:, :, 1] = b1
packed[:, :, 2] = b2
packed[:, :, 3] = b3
packed[:, :, 4] = b4
return packed.reshape(h, w // 4 * 5)
def load_raw10_packed_file(self, path: str, width: int, height: int) -> np.ndarray:
packed_width = self.packed_width_for_raw10(width)
expected_size = height * packed_width
actual_size = os.path.getsize(path)
if actual_size != expected_size:
raise ValueError(
f"Tamanho inválido RAW10: {actual_size}, esperado {expected_size} em {path}"
)
packed = np.fromfile(path, dtype=np.uint8).reshape(height, packed_width)
return self.unpack_raw10_packed(packed, sensor_width=width, sensor_height=height)
def save_raw10_packed_file(self, path: str, raw16: np.ndarray):
packed = self.pack_raw10_packed(raw16)
packed.tofile(path)
# ============================================================
# RGB UINT8
# ============================================================
def load_rgb_u8_file(self, path: str, shape) -> np.ndarray:
arr = np.fromfile(path, dtype=np.uint8)
expected = np.prod(shape)
if arr.size != expected:
raise ValueError(
f"Tamanho inválido RGB: {arr.size}, esperado {expected} em {path}"
)
return arr.reshape(shape)
def save_rgb_u8_file(self, path: str, arr: np.ndarray):
arr.astype(np.uint8).tofile(path)
# ============================================================
# DISPATCHER (O MAIS IMPORTANTE)
# ============================================================
def load_native_bin(self, path: str, cam_meta: dict) -> np.ndarray:
"""
Decide automaticamente como carregar o .bin baseado no meta.
"""
channels = int(cam_meta.get("channels", 1))
bit_depth = int(cam_meta.get("bit_depth", 10))
shape = cam_meta.get("shape")
if channels == 1 and bit_depth == 10:
width = int(cam_meta["width"])
height = int(cam_meta["height"])
return self.load_raw10_packed_file(path, width, height)
elif channels == 3 and bit_depth == 8:
return self.load_rgb_u8_file(path, shape)
else:
raise ValueError(
f"Formato não suportado: channels={channels}, bit_depth={bit_depth}"
)
def save_native_bin(self, path: str, arr: np.ndarray, cam_meta: dict):
"""
Salva no formato correto baseado no meta.
"""
channels = int(cam_meta.get("channels", 1))
bit_depth = int(cam_meta.get("bit_depth", 10))
if channels == 1 and bit_depth == 10:
self.save_raw10_packed_file(path, arr)
elif channels == 3 and bit_depth == 8:
self.save_rgb_u8_file(path, arr)
else:
raise ValueError(
f"Formato não suportado para salvar: channels={channels}, bit_depth={bit_depth}"
)
def load_fusion_config_json(self, path: str):
if not path or not os.path.isfile(path):
raise FileNotFoundError(f"Arquivo de calibração não encontrado: {path}")
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
fusion = data.get("fusion_config")
if not isinstance(fusion, dict):
print("[WARN] JSON sem fusion_config. Mantendo config padrão.")
return
self.fusion_config = self._merge_fusion_config(self.fusion_config, fusion)
def _merge_fusion_config(self, default_cfg: dict, loaded_cfg: dict) -> dict:
cfg = json.loads(json.dumps(default_cfg))
for key, value in loaded_cfg.items():
if isinstance(value, dict) and isinstance(cfg.get(key), dict):
cfg[key].update(value)
else:
cfg[key] = value
return cfg

2
Python/raspi/cam_3/pi/server.py
View File

@ -469,7 +469,6 @@ class ModuleServer:
return {"ok": False, "error": "Módulo não inicializado"} return {"ok": False, "error": "Módulo não inicializado"}
self._restart_module_if_needed() self._restart_module_if_needed()
if getattr(self.stream_sender, "is_running", False): if getattr(self.stream_sender, "is_running", False):
return { return {
"ok": True, "ok": True,
@ -487,7 +486,6 @@ class ModuleServer:
self.state.stream_port = port self.state.stream_port = port
self.state.stream_fps = fps self.state.stream_fps = fps
self.state.status = "streaming" self.state.status = "streaming"
return { return {
"ok": True, "ok": True,
"streaming": True, "streaming": True,

1
Python/raspi/cam_3/pi/state.py
View File

@ -450,6 +450,7 @@ class ModuleState:
else: else:
self.detected_mode = "NONE" self.detected_mode = "NONE"
def resolve_capture_mode(self) -> str: def resolve_capture_mode(self) -> str:
connected_re = self.get_active_camera_by_role("re") is not None connected_re = self.get_active_camera_by_role("re") is not None
connected_nir = self.get_active_camera_by_role("nir") is not None connected_nir = self.get_active_camera_by_role("nir") is not None

4
Python/raspi/check_saved_files.py
View File

@ -6,8 +6,8 @@ from pathlib import Path
import cv2 import cv2
import numpy as np import numpy as np
from cam_2.pi.raw_processor_core import RawProcessorCore from cam_3.pi.raw_processor_core import RawProcessorCore
from cam_2.pi.raw_processor_preview import RawProcessorPreview from cam_3.pi.raw_processor_preview import RawProcessorPreview
def load_json(path: Path) -> dict: def load_json(path: Path) -> dict:

177
Python/raspi/manual_fusion_calibrator.py
View File

@ -7,15 +7,7 @@ from datetime import datetime
import cv2 import cv2
import numpy as np import numpy as np
from cam_3.multispectral_service import MultiSpectralService from cam_3.multispectral_client import MultiSpectralClient
from cam_3.stream_receiver import StreamReceiver
from cam_3.pi.raw_processor_core import RawProcessorCore
from cam_3.pi.raw_processor_preview import RawProcessorPreview
STREAM_PORT = 6001
PI_HOST = "192.168.105.6"
PC_HOST = "192.168.105.5"
# ============================================================ # ============================================================
@ -172,7 +164,7 @@ def stack_2x2(a: np.ndarray, b: np.ndarray, c: np.ndarray, d: np.ndarray) -> np.
def build_empty_panel_like(ref_bgr: np.ndarray, title: str) -> np.ndarray: def build_empty_panel_like(ref_bgr: np.ndarray, title: str) -> np.ndarray:
img = np.zeros_like(ref_bgr) img = np.zeros_like(ref_bgr)
overlay_hud(img, [title, "sem frame disponível"], x=18, y=40, font_scale=0.8, line_step=34) overlay_hud(img, [title, "sem frame disponivel"], x=18, y=40, font_scale=0.8, line_step=34)
return img return img
@ -199,76 +191,6 @@ def validate_module_ready(status: dict, frame_type: str, raw_policy: str, captur
raise RuntimeError(f"frame_type desconhecido para validação: {frame_type}") raise RuntimeError(f"frame_type desconhecido para validação: {frame_type}")
def resolve_effective_capture_mode(requested_mode: str) -> str:
if requested_mode in ("SINGLE", "DOUBLE", "TRIPLE"):
return requested_mode
return "AUTO"
# ============================================================
# Decodificação do stream RAW_BRUTO
# ============================================================
class StreamDecoder:
def __init__(self, sensor_width: int, sensor_height: int, bayer_pattern: str = "GBRG"):
self.sensor_width = sensor_width
self.sensor_height = sensor_height
self.bayer_pattern = bayer_pattern
def decode_stream_cameras(self, frame, meta):
if not isinstance(frame, dict):
raise RuntimeError("RAW_BRUTO esperado como dict de câmeras no modo multi")
camera_frames = meta.get("camera_frames", {}) or {}
decoded = {}
if "cam2" in frame:
rgb_bgr = frame["cam2"]
if rgb_bgr.ndim != 3 or rgb_bgr.shape[2] != 3:
raise RuntimeError(f"cam2 RGB inválida: shape={rgb_bgr.shape}")
rgb = rgb_bgr[:, :, ::-1].astype(np.float32) / 255.0
decoded["cam2"] = {
"name": "RGB",
"image": rgb,
"meta": camera_frames.get("cam2", {}),
}
for cam_id, spec_name in (("cam0", "RE"), ("cam1", "NIR")):
if cam_id not in frame:
continue
packed = frame[cam_id]
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
cam_meta = camera_frames.get(cam_id, {})
packed_width = int(cam_meta.get("width", packed.shape[1]))
height = int(cam_meta.get("height", packed.shape[0]))
bayer = cam_meta.get("bayer_pattern", self.bayer_pattern)
bit_depth = int(cam_meta.get("bit_depth", 10))
real_width = int((packed_width * 8) / 10) if bit_depth == 10 else packed_width
rp = RawProcessorCore(
sensor_width=real_width,
sensor_height=height,
bayer_pattern=bayer,
)
raw16 = rp.unpack_raw10_packed(packed)
max_val = float((1 << bit_depth) - 1)
single = np.clip(raw16.astype(np.float32) / max_val, 0.0, 1.0)
decoded[cam_id] = {
"name": spec_name,
"image": single,
"meta": cam_meta,
}
return decoded
# ============================================================ # ============================================================
# Persistência dos offsets # Persistência dos offsets
# ============================================================ # ============================================================
@ -339,9 +261,9 @@ def main():
description="Calibrador manual de offsets para fusão RGB/RE/NIR a partir do stream RAW_BRUTO.", description="Calibrador manual de offsets para fusão RGB/RE/NIR a partir do stream RAW_BRUTO.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter, formatter_class=argparse.ArgumentDefaultsHelpFormatter,
) )
parser.add_argument("--pi_host", default=PI_HOST) parser.add_argument("--pi_host", default="192.168.105.6")
parser.add_argument("--pc_host", default=PC_HOST) parser.add_argument("--pc_host", default="192.168.105.5")
parser.add_argument("--stream_port", type=int, default=STREAM_PORT) parser.add_argument("--stream_port", type=int, default=6001)
parser.add_argument("--server_port", type=int, default=5000) parser.add_argument("--server_port", type=int, default=5000)
parser.add_argument("--fps", type=int, default=20) parser.add_argument("--fps", type=int, default=20)
parser.add_argument("--width", type=int, default=640) parser.add_argument("--width", type=int, default=640)
@ -400,11 +322,7 @@ def main():
last_msg_t = time.time() last_msg_t = time.time()
return return
effective_capture_mode = resolve_effective_capture_mode(args.capture_mode) effective_capture_mode = args.capture_mode
receiver = StreamReceiver(host="0.0.0.0", port=args.stream_port)
svc = MultiSpectralService(host=args.pi_host, port=args.server_port, timeout=10)
decoder = StreamDecoder(sensor_width=args.width, sensor_height=args.height, bayer_pattern=args.bayer)
offsets_data = load_offsets_json(args.load_json, args, effective_capture_mode) offsets_data = load_offsets_json(args.load_json, args, effective_capture_mode)
offsets = offsets_data["manual_offsets"] offsets = offsets_data["manual_offsets"]
@ -444,45 +362,25 @@ def main():
cv2.setMouseCallback(window_name, on_mouse) cv2.setMouseCallback(window_name, on_mouse)
try: try:
receiver.start() with MultiSpectralClient(
time.sleep(0.5) pi_host=args.pi_host,
pc_host=args.pc_host,
print(f"[INFO] Verificando conexão com o módulo em {args.pi_host}:{args.server_port}...") server_port=args.server_port,
if not svc.check_connection(2): stream_port=args.stream_port,
raise RuntimeError("Módulo não encontrado ou não respondeu ao ping.") width=args.width,
print("[OK] Módulo conectado e respondendo.") height=args.height,
bayer=args.bayer,
svc.connect() fps=args.fps,
frame_type="RAW_BRUTO",
print("SET CAM0 RES:", svc.set_camera_resolution(0, args.width, args.height)) output_dtype="uint8",
print("SET CAM1 RES:", svc.set_camera_resolution(1, args.width, args.height)) capture_mode=effective_capture_mode,
print("SET CAM2 RES:", svc.set_camera_resolution(2, args.width, args.height)) raw_policy=args.raw_policy,
print("SET CAM0 BAYER:", svc.set_camera_bayer(0, args.bayer)) module_calibration_json=None,
print("SET CAM1 BAYER:", svc.set_camera_bayer(1, args.bayer)) ) as cam:
print("SET FPS:", svc.set_fps(args.fps))
print("SET CAPTURE MODE:", svc.set_capture_mode(effective_capture_mode))
print("SET FRAME TYPE:", svc.set_frame_type("RAW_BRUTO"))
print("SET OUTPUT DTYPE:", svc.set_output_dtype("float32"))
begin_resp = svc.begin(frame_type="RAW_BRUTO", output_dtype="float32", capture_mode=effective_capture_mode)
print("BEGIN:", begin_resp)
status = svc.get_status()
print("STATUS:", json.dumps({
"status": status.get("status"),
"detected_mode": status.get("detected_mode"),
"camera_count_active": status.get("camera_count_active"),
"active_camera_ids": status.get("active_camera_ids"),
}, ensure_ascii=False))
validate_module_ready(status, "RAW_BRUTO", args.raw_policy, effective_capture_mode)
print("START STREAM:", svc.start_stream(args.pc_host, args.stream_port, fps=args.fps))
while True: while True:
t0 = time.time() t0 = time.time()
meta = receiver.last_meta frame, meta = cam.get_next_frame(timeout=2.0)
frame = receiver.last_frame
if meta is not None and frame is not None and meta.get("frame_id") != last_frame_id: if meta is not None and frame is not None and meta.get("frame_id") != last_frame_id:
last_frame_id = meta["frame_id"] last_frame_id = meta["frame_id"]
@ -490,7 +388,7 @@ def main():
if not isinstance(frame, dict): if not isinstance(frame, dict):
raise RuntimeError("Este calibrador espera RAW_BRUTO multi-payload como dict de câmeras.") raise RuntimeError("Este calibrador espera RAW_BRUTO multi-payload como dict de câmeras.")
decoded = decoder.decode_stream_cameras(frame, meta) decoded = cam.core.decode_stream_cameras(frame, meta)
decoded_last = decoded decoded_last = decoded
curr_frame_id = meta.get("frame_id") curr_frame_id = meta.get("frame_id")
@ -576,8 +474,8 @@ def main():
nir_dx = int(offsets.get("cam1", {}).get("dx", 0)) nir_dx = int(offsets.get("cam1", {}).get("dx", 0))
nir_dy = int(offsets.get("cam1", {}).get("dy", 0)) nir_dy = int(offsets.get("cam1", {}).get("dy", 0))
nir_theta = float(offsets.get("cam1", {}).get("theta_deg", 0.0)) nir_theta = float(offsets.get("cam1", {}).get("theta_deg", 0.0))
overlay_hud(re_panel, [f"RE (cam0) | dx={re_dx} dy={re_dy} th={re_theta:.2f}g", "R seleciona RE"], y=24) overlay_hud(re_panel, [f"RE (cam0) | dx={re_dx} dy={re_dy} th={re_theta:.2f}g", "2 seleciona RE"], y=24)
overlay_hud(nir_panel, [f"NIR (cam1) | dx={nir_dx} dy={nir_dy} th={nir_theta:.2f}g", "N seleciona NIR"], y=24) overlay_hud(nir_panel, [f"NIR (cam1) | dx={nir_dx} dy={nir_dy} th={nir_theta:.2f}g", "3 seleciona NIR"], y=24)
ph = max(fuse_panel.shape[0], rgb_panel.shape[0], re_panel.shape[0], nir_panel.shape[0]) ph = max(fuse_panel.shape[0], rgb_panel.shape[0], re_panel.shape[0], nir_panel.shape[0])
pw = max(fuse_panel.shape[1], rgb_panel.shape[1], re_panel.shape[1], nir_panel.shape[1]) pw = max(fuse_panel.shape[1], rgb_panel.shape[1], re_panel.shape[1], nir_panel.shape[1])
@ -685,19 +583,19 @@ def main():
last_msg = f"{selected_cam}: precisa de >=4 pares e mesmo numero de pontos" last_msg = f"{selected_cam}: precisa de >=4 pares e mesmo numero de pontos"
last_msg_t = time.time() last_msg_t = time.time()
elif k in (ord("r"), ord("R")): elif k == ord("2"):
if "cam0" in decoded_last: if "cam0" in decoded_last:
selected_cam = "cam0" selected_cam = "cam0"
last_msg = "Selecionada: cam0 / RE" last_msg = "Selecionada: cam0 / RE"
else: else:
last_msg = "cam0 / RE não disponível neste frame" last_msg = "cam0 / RE nao disponivel neste frame"
last_msg_t = time.time() last_msg_t = time.time()
elif k in (ord("n"), ord("N")): elif k == ord("3"):
if "cam1" in decoded_last: if "cam1" in decoded_last:
selected_cam = "cam1" selected_cam = "cam1"
last_msg = "Selecionada: cam1 / NIR" last_msg = "Selecionada: cam1 / NIR"
else: else:
last_msg = "cam1 / NIR não disponível neste frame" last_msg = "cam1 / NIR nao disponivel neste frame"
last_msg_t = time.time() last_msg_t = time.time()
elif k == 9: # TAB elif k == 9: # TAB
choices = [cid for cid in ("cam0", "cam1") if cid in decoded_last] choices = [cid for cid in ("cam0", "cam1") if cid in decoded_last]
@ -778,27 +676,12 @@ def main():
selected_points_rgb[selected_cam].pop() selected_points_rgb[selected_cam].pop()
last_msg = f"Removido ultimo ponto RGB de {selected_cam}" last_msg = f"Removido ultimo ponto RGB de {selected_cam}"
last_msg_t = time.time() last_msg_t = time.time()
dt_loop = time.time() - t0 dt_loop = time.time() - t0
if dt_loop < 0.001: if dt_loop < 0.001:
time.sleep(0.001) time.sleep(0.001)
finally: finally:
try:
print("STOP STREAM:", svc.stop_stream())
except Exception:
pass
try:
print("STOP:", svc.stop())
except Exception:
pass
try:
svc.disconnect()
except Exception:
pass
try:
receiver.stop()
except Exception:
pass
cv2.destroyAllWindows() cv2.destroyAllWindows()
print("Fim da calibração manual.") print("Fim da calibração manual.")

366
Python/raspi/sensor_calibration_tool.py
View File

@ -7,15 +7,7 @@ from datetime import datetime
import cv2 import cv2
import numpy as np import numpy as np
from cam_3.multispectral_service import MultiSpectralService from cam_3.multispectral_client import MultiSpectralClient
from cam_3.stream_receiver import StreamReceiver
from cam_3.pi.raw_processor_core import RawProcessorCore
STREAM_PORT = 6001
PI_HOST = "192.168.105.6"
PC_HOST = "192.168.105.5"
# ============================================================ # ============================================================
# Helpers # Helpers
@ -30,17 +22,54 @@ def ensure_dir(path: str):
def overlay_hud( def overlay_hud(
img_bgr: np.ndarray, img_bgr,
lines: list[str], lines,
x: int = 12, x=12,
y: int = 22, y=22,
font_scale: float = 0.55, area_h=None,
line_step: int = 22, max_font_scale=None,
min_font_scale=None,
max_line_step=None,
min_line_step=None,
bottom_margin=12,
): ):
h, w = img_bgr.shape[:2]
if area_h is None:
area_h = h - y - bottom_margin
scale = max(1.0, min(1.45, area_h / 480.0))
if max_font_scale is None:
max_font_scale = 0.62 * scale
if min_font_scale is None:
min_font_scale = 0.34 * scale
if max_line_step is None:
max_line_step = int(22 * scale)
if min_line_step is None:
min_line_step = int(13 * scale)
available_h = max(1, area_h - bottom_margin)
n = max(1, len(lines))
font_scale = max_font_scale
line_step = max_line_step
needed_h = n * line_step
if needed_h > available_h:
shrink = available_h / float(needed_h)
font_scale = max(min_font_scale, max_font_scale * shrink)
line_step = max(min_line_step, int(max_line_step * shrink))
yy = y yy = y
for s in lines: for s in lines:
cv2.putText(img_bgr, s, (x, yy), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 0, 0), 3, cv2.LINE_AA) if yy > y + area_h - bottom_margin:
cv2.putText(img_bgr, s, (x, yy), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (255, 255, 255), 1, cv2.LINE_AA) break
cv2.putText(img_bgr, s, (x, yy), cv2.FONT_HERSHEY_SIMPLEX,
font_scale, (0, 0, 0), 3, cv2.LINE_AA)
cv2.putText(img_bgr, s, (x, yy), cv2.FONT_HERSHEY_SIMPLEX,
font_scale, (255, 255, 255), 1, cv2.LINE_AA)
yy += line_step yy += line_step
@ -87,7 +116,7 @@ def validate_module_ready(status: dict, frame_type: str, raw_policy: str, captur
def build_empty_panel(shape_hw: tuple[int, int], title: str) -> np.ndarray: def build_empty_panel(shape_hw: tuple[int, int], title: str) -> np.ndarray:
h, w = shape_hw h, w = shape_hw
img = np.zeros((h, w, 3), dtype=np.uint8) img = np.zeros((h, w, 3), dtype=np.uint8)
overlay_hud(img, [title, "sem frame disponivel"], x=18, y=40, font_scale=0.8, line_step=34) overlay_hud(img, [title, "sem frame disponivel"])
return img return img
@ -267,70 +296,6 @@ def draw_current_polygon(panel_bgr: np.ndarray, points: list):
cv2.polylines(panel_bgr, [pts], isClosed=False, color=(0, 255, 255), thickness=1) cv2.polylines(panel_bgr, [pts], isClosed=False, color=(0, 255, 255), thickness=1)
# ============================================================
# Decodificação do stream RAW_BRUTO
# ============================================================
class StreamDecoder:
def __init__(self, sensor_width: int, sensor_height: int, bayer_pattern: str = "GBRG"):
self.sensor_width = sensor_width
self.sensor_height = sensor_height
self.bayer_pattern = bayer_pattern
def decode_stream_cameras(self, frame, meta):
if not isinstance(frame, dict):
raise RuntimeError("RAW_BRUTO esperado como dict de câmeras no modo multi")
camera_frames = meta.get("camera_frames", {}) or {}
decoded = {}
if "cam2" in frame:
rgb_bgr = frame["cam2"]
if rgb_bgr.ndim != 3 or rgb_bgr.shape[2] != 3:
raise RuntimeError(f"cam2 RGB inválida: shape={rgb_bgr.shape}")
rgb = rgb_bgr[:, :, ::-1].astype(np.float32) / 255.0
decoded["cam2"] = {
"name": "RGB",
"image": rgb,
"meta": camera_frames.get("cam2", {}),
}
for cam_id, spec_name in (("cam0", "RE"), ("cam1", "NIR")):
if cam_id not in frame:
continue
packed = frame[cam_id]
if packed.ndim == 3 and packed.shape[2] == 1:
packed = packed[:, :, 0]
cam_meta = camera_frames.get(cam_id, {})
packed_width = int(cam_meta.get("width", packed.shape[1]))
height = int(cam_meta.get("height", packed.shape[0]))
bayer = cam_meta.get("bayer_pattern", self.bayer_pattern)
bit_depth = int(cam_meta.get("bit_depth", 10))
real_width = int((packed_width * 8) / 10) if bit_depth == 10 else packed_width
rp = RawProcessorCore(
sensor_width=real_width,
sensor_height=height,
bayer_pattern=bayer,
)
raw16 = rp.unpack_raw10_packed(packed)
max_val = float((1 << bit_depth) - 1)
single = np.clip(raw16.astype(np.float32) / max_val, 0.0, 1.0)
decoded[cam_id] = {
"name": spec_name,
"image": single,
"meta": cam_meta,
}
return decoded
# ============================================================ # ============================================================
# MOCK # MOCK
# ============================================================ # ============================================================
@ -422,7 +387,7 @@ def save_offline_sample(
return png_path, json_path return png_path, json_path
def load_offline_sample_decoded(json_path: str, decoder: StreamDecoder): def load_offline_sample_decoded(json_path: str, cam: MultiSpectralClient):
if not os.path.isfile(json_path): if not os.path.isfile(json_path):
raise FileNotFoundError(f"Sample offline não encontrado: {json_path}") raise FileNotFoundError(f"Sample offline não encontrado: {json_path}")
@ -459,7 +424,7 @@ def load_offline_sample_decoded(json_path: str, decoder: StreamDecoder):
stream_meta.setdefault("camera_frames", meta.get("camera_frames", {})) stream_meta.setdefault("camera_frames", meta.get("camera_frames", {}))
stream_meta.setdefault("frame_type", "RAW_BRUTO") stream_meta.setdefault("frame_type", "RAW_BRUTO")
decoded = decoder.decode_stream_cameras(frame, stream_meta) decoded = cam.core.decode_stream_cameras(frame, stream_meta)
preview_path = meta.get("saved_preview_path") preview_path = meta.get("saved_preview_path")
preview_bgr = None preview_bgr = None
@ -610,7 +575,7 @@ def analyze_spectral_guidance(selected_cam: str, img01: np.ndarray, rois_for_cam
new_ctrl = json.loads(json.dumps(ctrl)) new_ctrl = json.loads(json.dumps(ctrl))
action = "keep" action = "keep"
status = "ok" status = "ok"
reason = "Parâmetros parecem aceitáveis." reason = "Parametros parecem aceitaveis."
if veg_mean is None: if veg_mean is None:
return { return {
@ -646,14 +611,14 @@ def analyze_spectral_guidance(selected_cam: str, img01: np.ndarray, rois_for_cam
new_ctrl["exposure_time_us"] = int(max(exp - exp_step, MIN_EXP_US)) new_ctrl["exposure_time_us"] = int(max(exp - exp_step, MIN_EXP_US))
action = "decrease_exposure" action = "decrease_exposure"
status = "adjust" status = "adjust"
reason = f"Vegetação saturando ({veg_sat:.2f}%). Reduzir exposição." reason = f"Vegetacao saturando ({veg_sat:.2f}%). Reduzir exposicao."
elif gain > MIN_GAIN: elif gain > MIN_GAIN:
new_ctrl["analogue_gain"] = float(max(gain / (1.0 + gain_step), MIN_GAIN)) new_ctrl["analogue_gain"] = float(max(gain / (1.0 + gain_step), MIN_GAIN))
action = "decrease_gain" action = "decrease_gain"
status = "adjust" status = "adjust"
reason = ( reason = (
f"Vegetação saturando ({veg_sat:.2f}%), mas exposição já está no mínimo. " f"Vegetacao saturando ({veg_sat:.2f}%), mas exposicao ja esta no minimo. "
"Reduzir ganho." "Reduzir ganho."
) )
@ -661,8 +626,8 @@ def analyze_spectral_guidance(selected_cam: str, img01: np.ndarray, rois_for_cam
action = "keep" action = "keep"
status = "limit" status = "limit"
reason = ( reason = (
f"Vegetação saturando ({veg_sat:.2f}%), mas exposição e ganho já estão no mínimo. " f"Vegetacao saturando ({veg_sat:.2f}%), mas exposicao e ganho ja estao no minimo. "
"Não há ajuste possível por software." "Nao ha ajuste possivel por software."
) )
# 2) Vegetação pouco iluminada # 2) Vegetação pouco iluminada
@ -670,14 +635,14 @@ def analyze_spectral_guidance(selected_cam: str, img01: np.ndarray, rois_for_cam
new_ctrl["exposure_time_us"] = int(min(exp + exp_step, 200000)) new_ctrl["exposure_time_us"] = int(min(exp + exp_step, 200000))
action = "increase_exposure" action = "increase_exposure"
status = "adjust" status = "adjust"
reason = f"p95 da vegetação baixo ({veg_p95:.3f}). Aumentar exposição." reason = f"p95 da vegetacao baixo ({veg_p95:.3f}). Aumentar exposicao."
# 3) Vegetação muito perto do teto # 3) Vegetação muito perto do teto
elif veg_p95 is not None and veg_p95 > 0.96: elif veg_p95 is not None and veg_p95 > 0.96:
new_ctrl["exposure_time_us"] = int(max(exp - exp_step, 100)) new_ctrl["exposure_time_us"] = int(max(exp - exp_step, 100))
action = "decrease_exposure" action = "decrease_exposure"
status = "adjust" status = "adjust"
reason = f"p95 da vegetação alto ({veg_p95:.3f}). Reduzir exposição." reason = f"p95 da vegetacao alto ({veg_p95:.3f}). Reduzir exposicao."
# 4) Separação ruim # 4) Separação ruim
elif separation is not None and separation < 0.25: elif separation is not None and separation < 0.25:
@ -685,12 +650,12 @@ def analyze_spectral_guidance(selected_cam: str, img01: np.ndarray, rois_for_cam
new_ctrl["exposure_time_us"] = int(min(exp + exp_step, 200000)) new_ctrl["exposure_time_us"] = int(min(exp + exp_step, 200000))
action = "increase_exposure" action = "increase_exposure"
status = "adjust" status = "adjust"
reason = f"Separação baixa ({separation:.3f}) e há margem no p95. Aumentar exposição." reason = f"Separacao baixa ({separation:.3f}) e ha margem no p95. Aumentar exposicao."
else: else:
new_ctrl["analogue_gain"] = float(min(gain * (1.0 + gain_step), 32.0)) new_ctrl["analogue_gain"] = float(min(gain * (1.0 + gain_step), 32.0))
action = "increase_gain" action = "increase_gain"
status = "adjust" status = "adjust"
reason = f"Separação baixa ({separation:.3f}) sem muita margem de exposição. Aumentar ganho levemente." reason = f"Separacao baixa ({separation:.3f}) sem muita margem de exposicao. Aumentar ganho levemente."
return { return {
"status": status, "status": status,
@ -785,9 +750,9 @@ def main():
description="Ferramenta de calibração dos sensores RGB/RE/NIR com controle manual e ROIs em tempo real.", description="Ferramenta de calibração dos sensores RGB/RE/NIR com controle manual e ROIs em tempo real.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter, formatter_class=argparse.ArgumentDefaultsHelpFormatter,
) )
parser.add_argument("--pi_host", default=PI_HOST) parser.add_argument("--pi_host", default="192.168.105.6")
parser.add_argument("--pc_host", default=PC_HOST) parser.add_argument("--pc_host", default="192.168.105.5")
parser.add_argument("--stream_port", type=int, default=STREAM_PORT) parser.add_argument("--stream_port", type=int, default=6001)
parser.add_argument("--server_port", type=int, default=5000) parser.add_argument("--server_port", type=int, default=5000)
parser.add_argument("--fps", type=int, default=20) parser.add_argument("--fps", type=int, default=20)
parser.add_argument("--width", type=int, default=640) parser.add_argument("--width", type=int, default=640)
@ -809,15 +774,27 @@ def main():
parser.add_argument("--offline_save_dir", default="calibration/offline_samples", help="Pasta para salvar frames brutos offline") parser.add_argument("--offline_save_dir", default="calibration/offline_samples", help="Pasta para salvar frames brutos offline")
args = parser.parse_args() args = parser.parse_args()
cam = MultiSpectralClient(
pi_host=args.pi_host,
pc_host=args.pc_host,
server_port=args.server_port,
stream_port=args.stream_port,
width=args.width,
height=args.height,
bayer=args.bayer,
fps=args.fps,
frame_type="RAW_BRUTO",
output_dtype="uint8",
capture_mode=args.capture_mode,
raw_policy=args.raw_policy,
module_calibration_json=None,
)
offline_mode = bool(args.offline_sample_json) offline_mode = bool(args.offline_sample_json)
live_mode = not args.mock and not offline_mode live_mode = not args.mock and not offline_mode
effective_capture_mode = args.capture_mode effective_capture_mode = args.capture_mode
receiver = StreamReceiver(host="0.0.0.0", port=args.stream_port)
svc = MultiSpectralService(host=args.pi_host, port=args.server_port, timeout=10)
decoder = StreamDecoder(sensor_width=args.width, sensor_height=args.height, bayer_pattern=args.bayer)
data_payload = load_payload(args.load_json, args, effective_capture_mode) data_payload = load_payload(args.load_json, args, effective_capture_mode)
selected_cam = "cam2" selected_cam = "cam2"
@ -837,6 +814,10 @@ def main():
last_raw_frame = None last_raw_frame = None
last_preview_bgr = None last_preview_bgr = None
roi_name_input_active = False
roi_name_buffer = ""
roi_name_points_pending = []
guidance_log = data_payload.get("calibration_guidance_log", []) guidance_log = data_payload.get("calibration_guidance_log", [])
last_guidance = guidance_log[-1]["result"] if guidance_log else None last_guidance = guidance_log[-1]["result"] if guidance_log else None
@ -912,31 +893,28 @@ def main():
decoded_last = build_mock_decoded(args) decoded_last = build_mock_decoded(args)
if offline_mode: if offline_mode:
decoded_last, last_meta_stream, last_raw_frame, last_preview_bgr = load_offline_sample_decoded( decoded_last, last_meta_stream, last_raw_frame, last_preview_bgr = load_offline_sample_decoded(args.offline_sample_json, cam)
args.offline_sample_json,
decoder,
)
def apply_controls_to_selected_cam(): def apply_controls_to_selected_cam():
nonlocal last_msg, last_msg_t nonlocal cam, last_msg, last_msg_t
ctrl = camera_controls[selected_cam] ctrl = camera_controls[selected_cam]
try: try:
resp = svc.set_ae_enable(selected_cam, bool(ctrl["ae_enable"])) resp = cam.svc.set_ae_enable(selected_cam, bool(ctrl["ae_enable"]))
ctrl["ae_enable"] = bool(resp.get("ae_enable", ctrl["ae_enable"])) ctrl["ae_enable"] = bool(resp.get("ae_enable", ctrl["ae_enable"]))
if selected_cam == "cam2": if selected_cam == "cam2":
resp = svc.set_awb_enable(selected_cam, bool(ctrl["awb_enable"])) resp = cam.svc.set_awb_enable(selected_cam, bool(ctrl["awb_enable"]))
ctrl["awb_enable"] = bool(resp.get("awb_enable", ctrl["awb_enable"])) ctrl["awb_enable"] = bool(resp.get("awb_enable", ctrl["awb_enable"]))
if not ctrl["ae_enable"]: if not ctrl["ae_enable"]:
if ctrl["exposure_time_us"] is not None: if ctrl["exposure_time_us"] is not None:
resp = svc.set_exposure_time(selected_cam, int(ctrl["exposure_time_us"])) resp = cam.svc.set_exposure_time(selected_cam, int(ctrl["exposure_time_us"]))
exp_val = resp.get("exposure_time_us", ctrl["exposure_time_us"]) exp_val = resp.get("exposure_time_us", ctrl["exposure_time_us"])
ctrl["exposure_time_us"] = int(exp_val) if exp_val is not None else None ctrl["exposure_time_us"] = int(exp_val) if exp_val is not None else None
if ctrl["analogue_gain"] is not None: if ctrl["analogue_gain"] is not None:
resp = svc.set_analogue_gain(selected_cam, float(ctrl["analogue_gain"])) resp = cam.svc.set_analogue_gain(selected_cam, float(ctrl["analogue_gain"]))
gain_val = resp.get("analogue_gain", ctrl["analogue_gain"]) gain_val = resp.get("analogue_gain", ctrl["analogue_gain"])
ctrl["analogue_gain"] = float(gain_val) if gain_val is not None else None ctrl["analogue_gain"] = float(gain_val) if gain_val is not None else None
@ -981,80 +959,51 @@ def main():
} }
return snap return snap
def sync_camera_controls_from_pi():
nonlocal cam, camera_controls
if not live_mode:
return
for cam_id in camera_controls.keys():
try: try:
if live_mode: initial_ctrl = cam.svc.get_camera_controls(cam_id)
receiver.start()
time.sleep(0.5)
print(f"[INFO] Verificando conexão com o módulo em {args.pi_host}:{args.server_port}...")
if not svc.check_connection(2):
raise RuntimeError("Módulo não encontrado ou não respondeu ao ping.")
print("[OK] Módulo conectado e respondendo.")
svc.connect()
print("SET CAM0 RES:", svc.set_camera_resolution(0, args.width, args.height))
print("SET CAM1 RES:", svc.set_camera_resolution(1, args.width, args.height))
print("SET CAM2 RES:", svc.set_camera_resolution(2, args.width, args.height))
print("SET CAM0 BAYER:", svc.set_camera_bayer(0, args.bayer))
print("SET CAM1 BAYER:", svc.set_camera_bayer(1, args.bayer))
print("SET FPS:", svc.set_fps(args.fps))
print("SET CAPTURE MODE:", svc.set_capture_mode(effective_capture_mode))
print("SET FRAME TYPE:", svc.set_frame_type("RAW_BRUTO"))
print("SET OUTPUT DTYPE:", svc.set_output_dtype("float32"))
begin_resp = svc.begin(frame_type="RAW_BRUTO", output_dtype="float32", capture_mode=effective_capture_mode)
print("BEGIN:", begin_resp)
status = svc.get_status()
print("STATUS:", json.dumps({
"status": status.get("status"),
"detected_mode": status.get("detected_mode"),
"camera_count_active": status.get("camera_count_active"),
"active_camera_ids": status.get("active_camera_ids"),
}, ensure_ascii=False))
validate_module_ready(status, "RAW_BRUTO", args.raw_policy, effective_capture_mode)
print("START STREAM:", svc.start_stream(args.pc_host, args.stream_port, fps=args.fps))
else:
last_msg = "MODO OFFLINE ativo" if offline_mode else "MODO MOCK ativo"
last_msg_t = time.time()
if live_mode:
try:
for cam_id, _ in camera_controls.items():
initial_ctrl = svc.get_camera_controls(cam_id)
camera_controls[cam_id]["ae_enable"] = bool( camera_controls[cam_id]["ae_enable"] = bool(
initial_ctrl.get("ae_enable", camera_controls[cam_id]["ae_enable"]) initial_ctrl.get("ae_enable", camera_controls[cam_id]["ae_enable"])
) )
camera_controls[cam_id]["awb_enable"] = bool( camera_controls[cam_id]["awb_enable"] = bool(
initial_ctrl.get("awb_enable", camera_controls[cam_id]["awb_enable"]) initial_ctrl.get("awb_enable", camera_controls[cam_id]["awb_enable"])
) )
exp_val = initial_ctrl.get("exposure_time_us", camera_controls[cam_id]["exposure_time_us"]) exp_val = initial_ctrl.get("exposure_time_us", camera_controls[cam_id]["exposure_time_us"])
if exp_val is not None: camera_controls[cam_id]["exposure_time_us"] = int(exp_val) if exp_val is not None else None
exp_val = int(exp_val)
camera_controls[cam_id]["exposure_time_us"] = exp_val
gain_val = initial_ctrl.get("analogue_gain", camera_controls[cam_id]["analogue_gain"]) gain_val = initial_ctrl.get("analogue_gain", camera_controls[cam_id]["analogue_gain"])
if gain_val is not None: camera_controls[cam_id]["analogue_gain"] = float(gain_val) if gain_val is not None else None
gain_val = float(gain_val)
camera_controls[cam_id]["analogue_gain"] = gain_val
camera_controls[cam_id]["colour_gains"] = initial_ctrl.get( camera_controls[cam_id]["colour_gains"] = initial_ctrl.get(
"colour_gains", "colour_gains",
camera_controls[cam_id]["colour_gains"] camera_controls[cam_id]["colour_gains"]
) )
except Exception as e: except Exception as e:
print(f"[WARN] Falha ao ler controles iniciais: {e}") print(f"[WARN] Falha ao ler controles iniciais de {cam_id}: {e}")
try:
if live_mode:
cam.start(print_debug=True)
sync_camera_controls_from_pi()
else:
last_msg = "MODO OFFLINE ativo" if offline_mode else "MODO MOCK ativo"
last_msg_t = time.time()
while True: while True:
t0 = time.time() t0 = time.time()
if live_mode: if live_mode:
meta = receiver.last_meta frame, meta = cam.get_next_frame(timeout=2.0)
frame = receiver.last_frame
if meta is not None and frame is not None and meta.get("frame_id") != last_frame_id: if meta is not None and frame is not None and meta.get("frame_id") != last_frame_id:
last_frame_id = meta["frame_id"] last_frame_id = meta["frame_id"]
@ -1062,7 +1011,7 @@ def main():
if not isinstance(frame, dict): if not isinstance(frame, dict):
raise RuntimeError("Este calibrador espera RAW_BRUTO multi-payload como dict de câmeras.") raise RuntimeError("Este calibrador espera RAW_BRUTO multi-payload como dict de câmeras.")
decoded_last = decoder.decode_stream_cameras(frame, meta) decoded_last = cam.core.decode_stream_cameras(frame, meta)
last_meta_stream = dict(meta) last_meta_stream = dict(meta)
last_raw_frame = {cam_id: arr.copy() for cam_id, arr in frame.items()} last_raw_frame = {cam_id: arr.copy() for cam_id, arr in frame.items()}
@ -1172,6 +1121,14 @@ def main():
if sep is not None: if sep is not None:
lines.append(f"sep_veg_solo={sep:.3f}") lines.append(f"sep_veg_solo={sep:.3f}")
if roi_name_input_active:
lines.extend([
"-",
"NOME DA ROI:",
f"> {roi_name_buffer}_",
"ENTER confirma | ESC cancela | BACKSPACE apaga",
])
lines.append("-") lines.append("-")
lines.append(f"ROIs: {len(rois[selected_cam])}") lines.append(f"ROIs: {len(rois[selected_cam])}")
for idx, roi in enumerate(rois[selected_cam][:6]): for idx, roi in enumerate(rois[selected_cam][:6]):
@ -1189,8 +1146,14 @@ def main():
"F salva frame bruto | SPACE salva PARAMS | S snapshot | Q sai", "F salva frame bruto | SPACE salva PARAMS | S snapshot | Q sai",
]) ])
x0, y0, _, _ = panel_rects["data"] x0, y0, x1, y1 = panel_rects["data"]
overlay_hud(board, lines, x=x0 + 12, y=y0 + 22, font_scale=0.52, line_step=20) overlay_hud(
board,
lines,
x=x0 + 12,
y=y0 + 22,
area_h=(y1 - y0) - 22,
)
if last_msg and (time.time() - last_msg_t) < 2.5: if last_msg and (time.time() - last_msg_t) < 2.5:
cv2.putText(board, last_msg, (12, board.shape[0] - 16), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 255, 0), 2, cv2.LINE_AA) cv2.putText(board, last_msg, (12, board.shape[0] - 16), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 255, 0), 2, cv2.LINE_AA)
@ -1205,10 +1168,49 @@ def main():
cv2.imshow(window_name, board) cv2.imshow(window_name, board)
else: else:
blank = np.zeros((720, 1280, 3), dtype=np.uint8) blank = np.zeros((720, 1280, 3), dtype=np.uint8)
overlay_hud(blank, ["Aguardando frames do módulo..."], x=40, y=80, font_scale=1.0, line_step=34) overlay_hud(blank, ["Aguardando frames do módulo..."], x=40, y=80)
cv2.imshow(window_name, blank) cv2.imshow(window_name, blank)
k = cv2.waitKey(1) & 0xFF k = cv2.waitKey(1) & 0xFF
if roi_name_input_active:
if k in (13, 10): # ENTER
name = roi_name_buffer.strip()
if not name:
name = f"roi_{len(rois[selected_cam]) + 1}"
roi = {
"name": name,
"type": "polygon",
"points": list(roi_name_points_pending),
"color": color_for_index(len(rois[selected_cam])),
}
rois[selected_cam].append(roi)
current_polygon_points = []
roi_name_points_pending = []
roi_name_buffer = ""
roi_name_input_active = False
last_msg = f"ROI criada em {selected_cam}: {name}"
last_msg_t = time.time()
elif k in (27,): # ESC
roi_name_input_active = False
roi_name_buffer = ""
roi_name_points_pending = []
last_msg = "Criacao de ROI cancelada"
last_msg_t = time.time()
elif k in (8, 127): # BACKSPACE
roi_name_buffer = roi_name_buffer[:-1]
elif 32 <= k <= 126:
roi_name_buffer += chr(k)
continue
if k in (ord("q"), ord("Q"), 27): if k in (ord("q"), ord("Q"), 27):
break break
elif k == ord("1"): elif k == ord("1"):
@ -1386,21 +1388,10 @@ def main():
last_msg = "ROI poligonal precisa de pelo menos 3 pontos" last_msg = "ROI poligonal precisa de pelo menos 3 pontos"
last_msg_t = time.time() last_msg_t = time.time()
else: else:
name = input(f"Nome da ROI para {selected_cam}: ").strip() roi_name_input_active = True
if not name: roi_name_buffer = ""
name = f"roi_{len(rois[selected_cam]) + 1}" roi_name_points_pending = list(current_polygon_points)
last_msg = "Digite o nome da ROI na tela"
roi = {
"name": name,
"type": "polygon",
"points": list(current_polygon_points),
"color": color_for_index(len(rois[selected_cam])),
}
rois[selected_cam].append(roi)
current_polygon_points = []
last_msg = f"ROI criada em {selected_cam}: {name}"
last_msg_t = time.time() last_msg_t = time.time()
dt_loop = time.time() - t0 dt_loop = time.time() - t0
@ -1409,22 +1400,7 @@ def main():
finally: finally:
if live_mode: if live_mode:
try: cam.stop()
print("STOP STREAM:", svc.stop_stream())
except Exception:
pass
try:
print("STOP:", svc.stop())
except Exception:
pass
try:
svc.disconnect()
except Exception:
pass
try:
receiver.stop()
except Exception:
pass
cv2.destroyAllWindows() cv2.destroyAllWindows()
print("Fim da calibração dos sensores.") print("Fim da calibração dos sensores.")

74
Python/raspi/test_service.py
View File

@ -1,29 +1,77 @@
import time import time
import cv2
import numpy as np
from cam_3.multispectral_service import MultiSpectralService from cam_3.multispectral_service import MultiSpectralService
svc = MultiSpectralService(host="192.168.105.6", port=5000)
svc = MultiSpectralService(host="192.168.105.6", port=5000, timeout=10)
try:
svc.connect() svc.connect()
print("PING:", svc.ping()) print("PING:", svc.ping())
print("STATUS:", svc.get_status()) print("STATUS:", svc.get_status())
print("SET FPS:", svc.set_fps(15)) print("SET FPS:", svc.set_fps(15))
print("SET JPG:", svc.set_jpeg_quality(85)) print("SET RES:", svc.set_resolution(640, 480))
print("SET RES:", svc.set_resolution(1280, 720)) print("SET CAPTURE MODE:", svc.set_capture_mode("AUTO"))
print("BEGIN:", svc.begin()) print("SET FRAME TYPE:", svc.set_frame_type("RAW_BRUTO"))
jpg = None print("SET OUTPUT DTYPE:", svc.set_output_dtype("uint8"))
for i in range(1, 6): # Começa em 1 e vai até 5
print("BEGIN:", svc.begin(
frame_type="RAW_BRUTO",
output_dtype="uint8",
capture_mode="AUTO",
))
last_frame = None
last_meta = None
for i in range(1, 6):
t0 = time.time() t0 = time.time()
jpg = svc.capture_jpg_base64() frame, meta = svc.capture_frame()
status = "OK" if jpg is not None else "Falha"
tempo = time.time() - t0 tempo = time.time() - t0
print(f"CAPTURE {i} JPG: {status}, Tempo: {tempo:.4f}")
print("STOP:", svc.stop()) last_frame = frame
last_meta = meta
print(
f"CAPTURE {i}: OK, Tempo={tempo:.4f}s, "
f"type={type(frame)}, frame_type={meta.get('frame_type')}, "
f"sources={meta.get('payload_sources')}"
)
print("CONFIG:", svc.get_config()) print("CONFIG:", svc.get_config())
print("STATUS FINAL:", svc.get_status()) print("STATUS FINAL:", svc.get_status())
finally:
try:
print("STOP:", svc.stop())
except Exception:
pass
svc.disconnect() svc.disconnect()
if jpg is not None:
with open("capture.jpg", "wb") as f: # Salva uma imagem simples de preview
f.write(jpg) if last_frame is not None:
if isinstance(last_frame, dict) and "cam2" in last_frame:
# cam2 vem BGR do OpenCV
cv2.imwrite("capture_cam2.jpg", last_frame["cam2"])
print("[OK] Salvo: capture_cam2.jpg")
elif isinstance(last_frame, np.ndarray):
if last_frame.ndim == 3:
# Pode ser CHW ou HWC
img = last_frame
if img.shape[0] in (3, 4, 5):
img = np.transpose(img[:3], (1, 2, 0))
if img.dtype != np.uint8:
img = np.clip(img * 255.0, 0, 255).astype(np.uint8)
cv2.imwrite("calibration/capture.jpg", img)
print("[OK] Salvo: calibration/capture.jpg")
else:
cv2.imwrite("calibration/capture_gray.jpg", last_frame)
print("[OK] Salvo: calibration/capture_gray.jpg")