Zendion/agrobot_base
Zendion
/
agrobot_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ajustado ferramenta de calibragem dos sensores multiespectrais

This commit is contained in:
Diego Freitas 2026-04-23 22:42:43 -03:00
parent 97c8de9196
commit cd1f777415
15 changed files with 9274 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
Python/OAK/datasets/multiespec_module/_0_capture.py
View File

@ -17,6 +17,13 @@ STREAM_PORT = 6001
PI_HOST = "192.168.105.6"
PC_HOST = "192.168.105.5"
with open("config.json", "r", encoding="utf-8") as f:
config = json.load(f)
MODELO = config.get("camera", ".")
RAW_SIZE = config.get("raw_size", [1296, 1028]) # [W, H]
CAMERA_PARAMS = config.get("camera_params_json")
# =========================
# Helpers gerais
@ -280,22 +287,6 @@ def build_preview_from_raw_payload(
raise RuntimeError(f"Tipo de frame não suportado para preview: {type(frame)}")
def resolve_effective_capture_mode(frame_type: str, raw_policy: str, requested_mode: str) -> str:
"""
Decide o modo real que será pedido ao módulo.
Nova regra:
- Se o usuário pediu explicitamente SINGLE/DOUBLE/TRIPLE, respeitamos.
- Se pediu AUTO, deixamos AUTO ir para o módulo.
- A única exceção opcional é MULTISPEC + require_triple implícito,
mas mesmo assim podemos deixar o módulo resolver se preferirmos.
"""
if requested_mode in ("SINGLE", "DOUBLE", "TRIPLE"):
return requested_mode
# requested_mode == AUTO
return "AUTO"
# =========================
# MAIN
# =========================
@ -314,8 +305,8 @@ def main():
parser.add_argument("--stream_port", type=int, default=STREAM_PORT, help="Porta TCP do receiver de stream.")
parser.add_argument("--server_port", type=int, default=5000, help="Porta TCP do servidor de comandos no Pi.")
parser.add_argument("--fps", type=int, default=20, help="FPS desejado.")
parser.add_argument("--width", type=int, default=640, help="Largura óptica da câmera.")
parser.add_argument("--height", type=int, default=480, help="Altura óptica da câmera.")
parser.add_argument("--width", type=int, default=RAW_SIZE[0], help="Largura óptica da câmera.")
parser.add_argument("--height", type=int, default=RAW_SIZE[1], help="Altura óptica da câmera.")
parser.add_argument("--interval", type=float, default=1.0, help="Intervalo em segundos para auto-save quando ligado.")
parser.add_argument("--preview_upscale", type=int, default=2, help="Fator de upscale visual do preview.")
parser.add_argument("--bayer", default="GBRG", choices=["GBRG", "GRBG", "RGGB", "BGGR"], help="Padrão Bayer das câmeras.")
@ -323,14 +314,11 @@ def main():
parser.add_argument("--frame_type", default="RAW_BRUTO", choices=["RAW_BRUTO", "RGB", "MULTISPEC"], help="Tipo de payload pedido ao Pi.")
parser.add_argument("--capture_mode", default="AUTO", choices=["AUTO", "SINGLE", "DOUBLE", "TRIPLE"], help="Modo de captura desejado no módulo.")
parser.add_argument("--raw_policy", default="allow_single", choices=["allow_single", "require_triple"], help="Quando frame_type=RAW_BRUTO, define se o script aceita 1 câmera ou exige 3.")
parser.add_argument("--camera_params_json", default=CAMERA_PARAMS, help="JSON salvo pelo calibrador de sensores com parâmetros fixos por câmera.")
args = parser.parse_args()
effective_capture_mode = resolve_effective_capture_mode(
frame_type=args.frame_type,
raw_policy=args.raw_policy,
requested_mode=args.capture_mode,
)
effective_capture_mode = args.capture_mode
raw_w = args.width
raw_h = args.height
@ -451,44 +439,17 @@ def main():
validate_module_ready(status, args.frame_type, args.raw_policy, effective_capture_mode)
params_resp = svc.apply_camera_params_json(args.camera_params_json)
if params_resp is not None:
camera_settings = params_resp["camera_settings"]
applied_camera_controls = params_resp["applied"]
print("[OK] Parâmetros fixos das câmeras aplicados:")
print(json.dumps(applied_camera_controls, ensure_ascii=False, indent=2))
else:
print("[OK] Parâmetros fixos das câmeras não aplicados")
print("START STREAM:", svc.start_stream(args.pc_host, args.stream_port, fps=args.fps))
selected_cam = "cam2" # padrão: RGB
camera_controls = {
"cam0": {
"ae_enable": False,
"awb_enable": False,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": None,
},
"cam1": {
"ae_enable": False,
"awb_enable": False,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": None,
},
"cam2": {
"ae_enable": True,
"awb_enable": True,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": [1.0, 1.0],
},
}
for cam_id in ("cam0", "cam1", "cam2"):
try:
camera_ctrl = svc.get_camera_controls(cam_id)
camera_controls[cam_id]["ae_enable"] = bool(camera_ctrl.get("ae_enable", camera_controls[cam_id]["ae_enable"]))
camera_controls[cam_id]["awb_enable"] = bool(camera_ctrl.get("awb_enable", camera_controls[cam_id]["awb_enable"]))
camera_controls[cam_id]["exposure_time_us"] = camera_ctrl.get("exposure_time_us", camera_controls[cam_id]["exposure_time_us"])
camera_controls[cam_id]["analogue_gain"] = camera_ctrl.get("analogue_gain", camera_controls[cam_id]["analogue_gain"])
camera_controls[cam_id]["colour_gains"] = camera_ctrl.get("colour_gains", camera_controls[cam_id]["colour_gains"])
except Exception as e:
print(f"[WARN] Falha ao ler controles de {cam_id}: {e}")
while True:
t0 = time.time()
@ -612,15 +573,14 @@ def main():
t_view_fps = time.time()
active_sources = meta.get("payload_sources")
ctrl = camera_controls[selected_cam]
lines = [
f"CANA: {args.cana} | HORA: {args.horario} | Pasta: {os.path.basename(session_dir)}",
f"Type={meta.get('frame_type')} | CaptureMode={effective_capture_mode} | RAW policy={args.raw_policy}",
f"Sources={active_sources} | FPS_STREAM={fps_stream:.1f} | FPS_VIEW={fps_view:.1f}",
f"frame_id={meta.get('frame_id')} | layout={meta.get('output_layout')} | dtype={meta.get('dtype') or meta.get('output_dtype')}",
f"codec={meta.get('codec_name', meta.get('codec_family', '-'))} | comp={meta.get('dt_comp', 0):.4f}s | send={meta.get('dt_send_payload_prev', 0):.4f}s",
f"CTRL_CAM={selected_cam} | AE={'ON' if ctrl['ae_enable'] else 'OFF'} | AWB={'ON' if ctrl['awb_enable'] else 'OFF'} | EXP={ctrl['exposure_time_us']} | GAIN={ctrl['analogue_gain']}",
"Keys: 1=cam0(RE) | 2=cam1(NIR) | 3=cam2(RGB) | C/SPACE=save | A=auto-save | E=AE | W=AWB | I/K=exp | O/L=gain | R=reset | M=preview | Q/Esc=quit",
f"CAM_PARAMS={os.path.basename(args.camera_params_json)} | controles fixos aplicados",
"Keys: C/SPACE=save | A=auto-save | M=preview | Q/Esc=quit"
]
overlay_hud(preview_show, lines, base_h=raw_h)
@ -666,8 +626,8 @@ def main():
"capture_mode_effective": effective_capture_mode,
"raw_policy": args.raw_policy,
"stream_meta": last_meta_stream,
"camera_controls": camera_controls,
"selected_control_camera": selected_cam,
"applied_camera_controls": applied_camera_controls,
"camera_params_json": args.camera_params_json,
"note": "autosave",
"raw_preview_reference_camera": preview_source_id,
}
@ -700,115 +660,6 @@ def main():
last_msg = f"Preview UPSCALE -> {preview_upscale}"
last_msg_t = time.time()
elif k in (ord("e"), ord("E")):
ctrl = camera_controls[selected_cam]
ctrl["ae_enable"] = not ctrl["ae_enable"]
resp = svc.set_ae_enable(selected_cam, ctrl["ae_enable"])
ctrl["ae_enable"] = bool(resp.get("ae_enable", ctrl["ae_enable"]))
last_msg = f"{selected_cam} AE -> {'ON' if ctrl['ae_enable'] else 'OFF'}"
last_msg_t = time.time()
elif k in (ord("w"), ord("W")):
ctrl = camera_controls[selected_cam]
if selected_cam != "cam2":
last_msg = "AWB só se aplica à cam2 / RGB"
last_msg_t = time.time()
else:
ctrl["awb_enable"] = not ctrl["awb_enable"]
resp = svc.set_awb_enable(selected_cam, ctrl["awb_enable"])
ctrl["awb_enable"] = bool(resp.get("awb_enable", ctrl["awb_enable"]))
last_msg = f"{selected_cam} AWB -> {'ON' if ctrl['awb_enable'] else 'OFF'}"
last_msg_t = time.time()
elif k in (ord("i"), ord("I")):
ctrl = camera_controls[selected_cam]
if ctrl["exposure_time_us"] is None:
ctrl["exposure_time_us"] = 15000
else:
ctrl["exposure_time_us"] = min(int(ctrl["exposure_time_us"] * 1.15), 200000)
if ctrl["ae_enable"]:
ctrl["ae_enable"] = False
svc.set_ae_enable(selected_cam, False)
resp = svc.set_exposure_time(selected_cam, int(ctrl["exposure_time_us"]))
ctrl["exposure_time_us"] = resp.get("exposure_time_us", ctrl["exposure_time_us"])
last_msg = f"{selected_cam} ExposureTime -> {ctrl['exposure_time_us']} us"
last_msg_t = time.time()
elif k in (ord("k"), ord("K")):
ctrl = camera_controls[selected_cam]
if ctrl["exposure_time_us"] is None:
ctrl["exposure_time_us"] = 15000
else:
ctrl["exposure_time_us"] = max(int(ctrl["exposure_time_us"] / 1.15), 100)
if ctrl["ae_enable"]:
ctrl["ae_enable"] = False
svc.set_ae_enable(selected_cam, False)
resp = svc.set_exposure_time(selected_cam, int(ctrl["exposure_time_us"]))
ctrl["exposure_time_us"] = resp.get("exposure_time_us", ctrl["exposure_time_us"])
last_msg = f"{selected_cam} ExposureTime -> {ctrl['exposure_time_us']} us"
last_msg_t = time.time()
elif k in (ord("o"), ord("O")):
ctrl = camera_controls[selected_cam]
if ctrl["analogue_gain"] is None:
ctrl["analogue_gain"] = 1.0
else:
ctrl["analogue_gain"] = min(float(ctrl["analogue_gain"]) * 1.10, 32.0)
if ctrl["ae_enable"]:
ctrl["ae_enable"] = False
svc.set_ae_enable(selected_cam, False)
resp = svc.set_analogue_gain(selected_cam, float(ctrl["analogue_gain"]))
ctrl["analogue_gain"] = resp.get("analogue_gain", ctrl["analogue_gain"])
last_msg = f"{selected_cam} AnalogueGain -> {ctrl['analogue_gain']:.2f}"
last_msg_t = time.time()
elif k in (ord("l"), ord("L")):
ctrl = camera_controls[selected_cam]
if ctrl["analogue_gain"] is None:
ctrl["analogue_gain"] = 1.0
else:
ctrl["analogue_gain"] = max(float(ctrl["analogue_gain"]) / 1.10, 1.0)
if ctrl["ae_enable"]:
ctrl["ae_enable"] = False
svc.set_ae_enable(selected_cam, False)
resp = svc.set_analogue_gain(selected_cam, float(ctrl["analogue_gain"]))
ctrl["analogue_gain"] = resp.get("analogue_gain", ctrl["analogue_gain"])
last_msg = f"{selected_cam} AnalogueGain -> {ctrl['analogue_gain']:.2f}"
last_msg_t = time.time()
elif k in (ord("r"), ord("R")):
ctrl = camera_controls[selected_cam]
svc.clear_exposure_time(selected_cam)
svc.clear_analogue_gain(selected_cam)
if selected_cam == "cam2":
svc.clear_colour_gains(selected_cam)
ctrl["exposure_time_us"] = None
ctrl["analogue_gain"] = None
ctrl["colour_gains"] = None if selected_cam == "cam2" else ctrl["colour_gains"]
last_msg = f"{selected_cam} manual controls resetados"
last_msg_t = time.time()
elif k in (ord("c"), ord("C"), 32):
if can_save:
frame_type_save = last_meta_stream.get("frame_type")
@ -825,8 +676,8 @@ def main():
"capture_mode_effective": effective_capture_mode,
"raw_policy": args.raw_policy,
"stream_meta": last_meta_stream,
"camera_controls": camera_controls,
"selected_control_camera": selected_cam,
"applied_camera_controls": applied_camera_controls,
"camera_params_json": args.camera_params_json,
"note": "manual",
"raw_preview_reference_camera": preview_source_id,
}
@ -844,21 +695,6 @@ def main():
last_msg = "SALVO (manual)"
last_msg_t = time.time()
elif k == ord("1"):
selected_cam = "cam0"
last_msg = "Controle ativo -> cam0 / RE"
last_msg_t = time.time()
elif k == ord("2"):
selected_cam = "cam1"
last_msg = "Controle ativo -> cam1 / NIR"
last_msg_t = time.time()
elif k == ord("3"):
selected_cam = "cam2"
last_msg = "Controle ativo -> cam2 / RGB"
last_msg_t = time.time()
dt_loop = time.time() - t0
if dt_loop < 0.001:
time.sleep(0.001)

13
Python/OAK/datasets/multiespec_module/_9_test_segformer.py
View File

@ -99,6 +99,7 @@ def main():
parser.add_argument("--alpha", type=float, default=0.45, help="Alpha do overlay da máscara")
parser.add_argument("--camera_frame_type", type=str, default="MULTISPEC", choices=["RAW_BRUTO", "RGB", "MULTISPEC"])
parser.add_argument("--camera_capture_mode", type=str, default="AUTO", choices=["AUTO", "SINGLE", "DOUBLE", "TRIPLE"])
parser.add_argument("--camera_params_json", default=None, help="JSON salvo pelo calibrador de sensores com parâmetros fixos por câmera.")
args = parser.parse_args()
@ -113,6 +114,9 @@ def main():
MODEL_NAME = config["model_name"]
modelo_folder = config["modelo"]
CAMERA_PARAMS = config.get("camera_params_json")
if args.camera_params_json is not None:
CAMERA_PARAMS = args.camera_params_json
CHANNELS = int(config.get("channels", 5))
FUSION_MODE = config.get("fusion_mode", "stacked")
RESOLUCAO = config["resolucao"]
@ -235,6 +239,15 @@ def main():
capture_mode=camera_capture_mode,
))
params_resp = svc.apply_camera_params_json(CAMERA_PARAMS)
if params_resp is not None:
camera_settings = params_resp["camera_settings"]
applied_camera_controls = params_resp["applied"]
print("[OK] Parâmetros fixos das câmeras aplicados:")
print(json.dumps(applied_camera_controls, ensure_ascii=False, indent=2))
else:
print("[OK] Parâmetros fixos das câmeras não aplicados")
print("START STREAM:", svc.start_stream(PC_HOST, STREAM_PORT, fps=15))
win = "CAMERA MULTISPEC + SEGFORMER (Q=quit)"

3
Python/OAK/datasets/multiespec_module/config.json
View File

@ -15,5 +15,6 @@
"use_ndvi": false,
"backbone": "nvidia/mit-b1",
"fusion_mode": "stacked",
"stats_source_tag": "stacked_raw4"
"stats_source_tag": "stacked_raw4",
"camera_params_json": "camera_params.json"
}

78
Python/OAK/datasets/multiespec_module/multispectral_service.py
View File

@ -3,6 +3,7 @@ import json
import numpy as np
import base64
import time
import os
from typing import Optional
@ -428,4 +429,79 @@ class MultiSpectralService:
})
def get_sensor_modes(self):
return self._send_command({"cmd": "get_sensor_modes"})
return self._send_command({"cmd": "get_sensor_modes"})
def load_camera_params_json(self, path: str) -> dict:
if not path or not os.path.isfile(path):
raise FileNotFoundError(f"Arquivo de parâmetros das câmeras não encontrado: {path}")
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
settings = data.get("camera_settings")
if not isinstance(settings, dict):
raise RuntimeError("JSON de calibração inválido: chave camera_settings ausente")
return settings
def apply_camera_settings(self, camera_settings: dict) -> dict:
applied = {}
for cam_id in ("cam0", "cam1", "cam2"):
ctrl = camera_settings.get(cam_id)
if not ctrl:
applied[cam_id] = {
"ok": False,
"error": "sem parâmetros no JSON",
}
continue
ae = bool(ctrl.get("ae_enable", False))
awb = bool(ctrl.get("awb_enable", False))
exp = ctrl.get("exposure_time_us")
gain = ctrl.get("analogue_gain")
colour_gains = ctrl.get("colour_gains")
result = {}
result["ae"] = self.set_ae_enable(cam_id, ae)
if cam_id == "cam2":
result["awb"] = self.set_awb_enable(cam_id, awb)
if not ae:
if exp is not None:
result["exposure"] = self.set_exposure_time(cam_id, int(exp))
if gain is not None:
result["gain"] = self.set_analogue_gain(cam_id, float(gain))
if cam_id == "cam2" and not awb and colour_gains is not None:
r_gain, b_gain = colour_gains
result["colour_gains"] = self.set_colour_gains(cam_id, float(r_gain), float(b_gain))
applied[cam_id] = {
"ok": True,
"requested": ctrl,
"responses": result,
}
return applied
def apply_camera_params_json(self, path: str) -> dict:
if path is None:
return {
"ok": False,
"path": None,
"camera_settings": None,
"applied": None,
}
camera_settings = self.load_camera_params_json(path)
applied = self.apply_camera_settings(camera_settings)
return {
"ok": True,
"path": path,
"camera_settings": camera_settings,
"applied": applied,
}

BIN
Python/raspi/calibration/mock/nir.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.0 MiB

BIN
Python/raspi/calibration/mock/re.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.9 MiB

BIN
Python/raspi/calibration/mock/rgb.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.0 MiB

107
Python/raspi/calibration/offline_samples/20260423_210500_000.json Normal file
View File

@ -0,0 +1,107 @@
{
"ts": "2026-04-24T00:27:10.376",
"schema": "multispec_offline_sample_v1",
"frame_type": "RAW_BRUTO",
"sensor_width": 2048,
"sensor_height": 1365,
"bayer_pattern": "GBRG",
"capture_mode_requested": "MOCK",
"capture_mode_effective": "MOCK",
"raw_policy": "allow_single",
"saved_payload_type": "raw_native_multi",
"saved_payload_paths": {
"cam0": "20260423_210500_000_cam0.bin",
"cam1": "20260423_210500_000_cam1.bin",
"cam2": "20260423_210500_000_cam2.bin"
},
"saved_payload_shapes": {
"cam0": [
1365,
2560
],
"cam1": [
1365,
2560
],
"cam2": [
1365,
2048,
3
]
},
"saved_payload_dtypes": {
"cam0": "uint8",
"cam1": "uint8",
"cam2": "uint8"
},
"saved_preview_path": "20260423_210500_000.png",
"stream_meta": {
"frame_id": 1,
"frame_type": "RAW_BRUTO",
"output_layout": "multi_payload",
"payload_sources": [
"cam0",
"cam1",
"cam2"
],
"camera_frames": {
"cam0": {
"id": "cam0",
"role": "re",
"width": 2560,
"height": 1365,
"real_width": 2048,
"bit_depth": 10,
"bayer_pattern": "GBRG",
"mock": true
},
"cam1": {
"id": "cam1",
"role": "nir",
"width": 2560,
"height": 1365,
"real_width": 2048,
"bit_depth": 10,
"bayer_pattern": "GBRG",
"mock": true
},
"cam2": {
"id": "cam2",
"role": "rgb",
"width": 2048,
"height": 1365,
"bit_depth": 8,
"mock": true
}
},
"codec_name": "offline_mock",
"dtype": "uint8"
},
"camera_settings": {
"cam0": {
"ae_enable": false,
"awb_enable": false,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": null
},
"cam1": {
"ae_enable": false,
"awb_enable": false,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": null
},
"cam2": {
"ae_enable": true,
"awb_enable": true,
"exposure_time_us": 15000,
"analogue_gain": 1.0,
"colour_gains": [
1.0,
1.0
]
}
},
"note": "offline mock sample generated from RGB/RE/NIR PNGs for sensor_calibration_tool testing"
}

BIN
Python/raspi/calibration/offline_samples/20260423_210500_000.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.5 MiB

BIN
Python/raspi/calibration/offline_samples/20260423_210500_000_cam0.bin Normal file
View File

Binary file not shown.

BIN
Python/raspi/calibration/offline_samples/20260423_210500_000_cam1.bin Normal file
View File

Binary file not shown.

BIN
Python/raspi/calibration/offline_samples/20260423_210500_000_cam2.bin Normal file
View File

Binary file not shown.

8209
Python/raspi/calibration/sensor_calibration.json Normal file
View File

File diff suppressed because it is too large Load Diff

79
Python/raspi/cam_3/multispectral_service.py
View File

@ -3,6 +3,7 @@ import json
import numpy as np
import base64
import time
import os
from typing import Optional
@ -428,4 +429,80 @@ class MultiSpectralService:
})
def get_sensor_modes(self):
return self._send_command({"cmd": "get_sensor_modes"})
return self._send_command({"cmd": "get_sensor_modes"})
def load_camera_params_json(self, path: str) -> dict:
if not path or not os.path.isfile(path):
raise FileNotFoundError(f"Arquivo de parâmetros das câmeras não encontrado: {path}")
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
settings = data.get("camera_settings")
if not isinstance(settings, dict):
raise RuntimeError("JSON de calibração inválido: chave camera_settings ausente")
return settings
def apply_camera_settings(self, camera_settings: dict) -> dict:
applied = {}
for cam_id in ("cam0", "cam1", "cam2"):
ctrl = camera_settings.get(cam_id)
if not ctrl:
applied[cam_id] = {
"ok": False,
"error": "sem parâmetros no JSON",
}
continue
ae = bool(ctrl.get("ae_enable", False))
awb = bool(ctrl.get("awb_enable", False))
exp = ctrl.get("exposure_time_us")
gain = ctrl.get("analogue_gain")
colour_gains = ctrl.get("colour_gains")
result = {}
result["ae"] = self.set_ae_enable(cam_id, ae)
if cam_id == "cam2":
result["awb"] = self.set_awb_enable(cam_id, awb)
if not ae:
if exp is not None:
result["exposure"] = self.set_exposure_time(cam_id, int(exp))
if gain is not None:
result["gain"] = self.set_analogue_gain(cam_id, float(gain))
if cam_id == "cam2" and not awb and colour_gains is not None:
r_gain, b_gain = colour_gains
result["colour_gains"] = self.set_colour_gains(cam_id, float(r_gain), float(b_gain))
applied[cam_id] = {
"ok": True,
"requested": ctrl,
"responses": result,
}
return applied
def apply_camera_params_json(self, path: str) -> dict:
if path is None:
return {
"ok": False,
"path": None,
"camera_settings": None,
"applied": None,
}
camera_settings = self.load_camera_params_json(path)
applied = self.apply_camera_settings(camera_settings)
return {
"ok": True,
"path": path,
"camera_settings": camera_settings,
"applied": applied,
}

919
Python/raspi/sensor_calibration_tool.py
View File

File diff suppressed because it is too large Load Diff