1945 lines
79 KiB
Python
1945 lines
79 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
audit_multispec_dataset_bulletproof.py
|
|
|
|
Auditoria parruda para dataset multiespectral OAK-FCC-3 / OAK-FCC-3P.
|
|
|
|
Objetivo:
|
|
Avaliar se o tensor final usado para treino/inferência está saudável o bastante
|
|
para campo, com rastreabilidade por amostra, por grupo, por classe e por canal.
|
|
|
|
Suporta:
|
|
1) Dataset original RAW multi-câmera:
|
|
group/<grupo>/metas/*.json
|
|
group/<grupo>/bins/*_CAM_A.bin, *_CAM_B.bin, *_CAM_C.bin
|
|
group/<grupo>/masks/*.png/.npy
|
|
|
|
2) Dataset normalizado tensor_npy:
|
|
dataset/<WxH>/group/<grupo>/metas/*.json
|
|
dataset/<WxH>/group/<grupo>/tensors/*.npy
|
|
dataset/<WxH>/group/<grupo>/masks/*.npy/.png
|
|
|
|
3) Dataset final com payload único .bin/.raw, quando meta traz saved_payload_path.
|
|
|
|
Gera:
|
|
- audit_summary.json
|
|
- audit_health.json
|
|
- audit_warnings.json
|
|
- audit_samples.csv
|
|
- audit_by_group.csv
|
|
- audit_by_class_channel.csv
|
|
- audit_by_sample_class_feature.csv
|
|
- audit_core_telemetry.csv
|
|
- audit_manifest_readme.txt
|
|
- visuals/*.png
|
|
- fixed/group/*, opcionalmente, com amostras aprovadas
|
|
|
|
Exemplos:
|
|
# Auditoria visual rápida no dataset normalizado
|
|
python -m utils.audit_multispec_dataset_bulletproof ^
|
|
--input_path dataset/1024x640/group ^
|
|
--out_dir audit_out ^
|
|
--save-visuals --visual-every 20
|
|
|
|
# Revisão manual e criação do fixed/group
|
|
python -m utils.audit_multispec_dataset_bulletproof ^
|
|
--input_path dataset/1024x640/group ^
|
|
--out_dir audit_out_manual ^
|
|
--manual-review --build-fixed-dataset --save-rejected-previews
|
|
|
|
# Auditoria do RAW original reconstruindo o tensor com RawProcessorCore
|
|
python -m utils.audit_multispec_dataset_bulletproof ^
|
|
--input_path dataset/original/group ^
|
|
--out_dir audit_raw_out ^
|
|
--save-visuals --visual-every 10
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import unicodedata
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
|
|
import cv2
|
|
import numpy as np
|
|
|
|
try:
|
|
from core.raw_processor_core import RawProcessorCore
|
|
except Exception:
|
|
RawProcessorCore = None
|
|
|
|
|
|
# ============================================================
|
|
# Constantes do contrato atual
|
|
# ============================================================
|
|
|
|
CHANNELS = ["R", "G", "B", "RE", "NIR"]
|
|
DERIVED = [
|
|
"NDVI",
|
|
"NDRE",
|
|
"NIR_minus_RE",
|
|
"NIR_over_RE",
|
|
"NIR_over_R",
|
|
"RE_over_R",
|
|
"NIR_over_G",
|
|
"RE_over_G",
|
|
"G_minus_R",
|
|
]
|
|
ALL_FEATURES = CHANNELS + DERIVED
|
|
|
|
DEFAULT_CLASS_MAP = {
|
|
0: "chao",
|
|
1: "cana",
|
|
2: "erva",
|
|
}
|
|
|
|
DEFAULT_MASK_COLOR_MAP_RGB = {
|
|
(128, 0, 0): 0, # chao
|
|
(0, 0, 128): 1, # cana
|
|
(0, 128, 0): 2, # erva
|
|
}
|
|
|
|
DEFAULT_VIS_PALETTE_BGR = {
|
|
0: (0, 0, 128),
|
|
1: (128, 0, 0),
|
|
2: (0, 128, 0),
|
|
255: (0, 0, 0),
|
|
}
|
|
|
|
EPS = 1e-6
|
|
|
|
|
|
# ============================================================
|
|
# Helpers gerais
|
|
# ============================================================
|
|
|
|
|
|
def ensure_dir(path: Path | str) -> Path:
|
|
p = Path(path)
|
|
p.mkdir(parents=True, exist_ok=True)
|
|
return p
|
|
|
|
|
|
def load_json(path: Path) -> dict:
|
|
with path.open("r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def write_json(path: Path, data: Any):
|
|
ensure_dir(path.parent)
|
|
with path.open("w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
def write_csv(path: Path, rows: List[Dict[str, Any]]):
|
|
ensure_dir(path.parent)
|
|
if not rows:
|
|
path.write_text("", encoding="utf-8")
|
|
return
|
|
|
|
keys: List[str] = []
|
|
seen = set()
|
|
for row in rows:
|
|
for k in row.keys():
|
|
if k not in seen:
|
|
seen.add(k)
|
|
keys.append(k)
|
|
|
|
with path.open("w", encoding="utf-8", newline="") as f:
|
|
w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore")
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
|
|
|
|
def safe_float(x: Any, default: float = 0.0) -> float:
|
|
try:
|
|
if x is None:
|
|
return default
|
|
v = float(x)
|
|
if math.isnan(v) or math.isinf(v):
|
|
return default
|
|
return v
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def safe_int(x: Any, default: int = 0) -> int:
|
|
try:
|
|
if x is None:
|
|
return default
|
|
return int(x)
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def cv_text(text: Any) -> str:
|
|
s = str(text)
|
|
s = unicodedata.normalize("NFKD", s)
|
|
return s.encode("ascii", "ignore").decode("ascii")
|
|
|
|
|
|
def parse_class_map(text: Optional[str]) -> Dict[int, str]:
|
|
if not text:
|
|
return dict(DEFAULT_CLASS_MAP)
|
|
out: Dict[int, str] = {}
|
|
for item in str(text).split(","):
|
|
item = item.strip()
|
|
if not item:
|
|
continue
|
|
if ":" in item:
|
|
k, v = item.split(":", 1)
|
|
elif "=" in item:
|
|
k, v = item.split("=", 1)
|
|
else:
|
|
raise ValueError(f"Classe inválida em --class-map: {item}")
|
|
out[int(k.strip())] = v.strip()
|
|
return out
|
|
|
|
|
|
def parse_csv_set(text: Optional[str]) -> set:
|
|
if not text:
|
|
return set()
|
|
return {x.strip().lower() for x in str(text).split(",") if x.strip()}
|
|
|
|
|
|
def sha1_short(path: Path, max_bytes: int = 1024 * 1024) -> str:
|
|
try:
|
|
h = hashlib.sha1()
|
|
with path.open("rb") as f:
|
|
h.update(f.read(max_bytes))
|
|
return h.hexdigest()[:12]
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def safe_rel(path: Path, root: Path) -> str:
|
|
try:
|
|
return str(path.resolve().relative_to(root.resolve())).replace("\\", "/")
|
|
except Exception:
|
|
return str(path).replace("\\", "/")
|
|
|
|
|
|
def flatten_dict(d: Any, prefix: str = "", max_depth: int = 4) -> Dict[str, Any]:
|
|
out: Dict[str, Any] = {}
|
|
if max_depth <= 0 or not isinstance(d, dict):
|
|
return out
|
|
for k, v in d.items():
|
|
key = f"{prefix}{k}" if not prefix else f"{prefix}.{k}"
|
|
if isinstance(v, dict):
|
|
out.update(flatten_dict(v, key, max_depth - 1))
|
|
elif isinstance(v, (str, int, float, bool)) or v is None:
|
|
out[key] = v
|
|
elif isinstance(v, (list, tuple)):
|
|
if len(v) <= 8 and all(isinstance(x, (str, int, float, bool)) or x is None for x in v):
|
|
out[key] = json.dumps(list(v), ensure_ascii=False)
|
|
else:
|
|
out[key] = f"list[{len(v)}]"
|
|
else:
|
|
out[key] = str(type(v).__name__)
|
|
return out
|
|
|
|
|
|
# ============================================================
|
|
# Visualização
|
|
# ============================================================
|
|
|
|
|
|
def normalize_to_u8(x: np.ndarray, p_low: float = 1.0, p_high: float = 99.0) -> np.ndarray:
|
|
arr = x.astype(np.float32, copy=False)
|
|
finite = np.isfinite(arr)
|
|
if not np.any(finite):
|
|
return np.zeros(arr.shape, dtype=np.uint8)
|
|
vals = arr[finite]
|
|
lo = np.percentile(vals, p_low)
|
|
hi = np.percentile(vals, p_high)
|
|
if hi <= lo + EPS:
|
|
hi = lo + 1.0
|
|
y = (arr - lo) / (hi - lo)
|
|
return np.clip(y * 255.0, 0, 255).astype(np.uint8)
|
|
|
|
|
|
def float01_to_u8(x: np.ndarray) -> np.ndarray:
|
|
return np.clip(x.astype(np.float32) * 255.0, 0, 255).astype(np.uint8)
|
|
|
|
|
|
def rgb_from_tensor(tensor: np.ndarray, stretch: bool = False) -> np.ndarray:
|
|
rgb = np.transpose(tensor[:3], (1, 2, 0)).astype(np.float32)
|
|
if stretch:
|
|
rgb_u8 = np.dstack([normalize_to_u8(rgb[:, :, i]) for i in range(3)])
|
|
else:
|
|
rgb_u8 = float01_to_u8(rgb)
|
|
return cv2.cvtColor(rgb_u8, cv2.COLOR_RGB2BGR)
|
|
|
|
|
|
def apply_colormap_gray(x: np.ndarray, stretch: bool = True) -> np.ndarray:
|
|
u8 = normalize_to_u8(x) if stretch else float01_to_u8(x)
|
|
return cv2.applyColorMap(u8, cv2.COLORMAP_VIRIDIS)
|
|
|
|
|
|
def put_label(img: np.ndarray, title: str, subtitle: str = "") -> np.ndarray:
|
|
out = img.copy()
|
|
title = cv_text(title)
|
|
subtitle = cv_text(subtitle)
|
|
header_h = 62 if subtitle else 38
|
|
cv2.rectangle(out, (0, 0), (out.shape[1], header_h), (0, 0, 0), -1)
|
|
cv2.putText(out, title[:90], (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.63, (0, 255, 255), 2, cv2.LINE_AA)
|
|
if subtitle:
|
|
cv2.putText(out, subtitle[:130], (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.43, (255, 255, 255), 1, cv2.LINE_AA)
|
|
return out
|
|
|
|
|
|
def make_grid(panels: List[Tuple[str, np.ndarray, str]], panel_w: int = 380, cols: int = 3) -> np.ndarray:
|
|
rendered: List[np.ndarray] = []
|
|
for title, img, subtitle in panels:
|
|
if img.ndim == 2:
|
|
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
|
|
scale = panel_w / max(1, img.shape[1])
|
|
panel_h = max(1, int(img.shape[0] * scale))
|
|
small = cv2.resize(img, (panel_w, panel_h), interpolation=cv2.INTER_AREA)
|
|
rendered.append(put_label(small, title, subtitle))
|
|
if not rendered:
|
|
return np.zeros((240, 480, 3), dtype=np.uint8)
|
|
|
|
max_h = max(x.shape[0] for x in rendered)
|
|
padded = []
|
|
for img in rendered:
|
|
if img.shape[0] < max_h:
|
|
pad = np.zeros((max_h - img.shape[0], img.shape[1], 3), dtype=np.uint8)
|
|
img = np.vstack([img, pad])
|
|
padded.append(img)
|
|
|
|
gap_w = np.full((max_h, 12, 3), 25, dtype=np.uint8)
|
|
row_w = cols * panel_w + (cols - 1) * 12
|
|
gap_h = np.full((12, row_w, 3), 25, dtype=np.uint8)
|
|
|
|
rows = []
|
|
for i in range(0, len(padded), cols):
|
|
row_imgs = padded[i:i + cols]
|
|
while len(row_imgs) < cols:
|
|
row_imgs.append(np.zeros_like(padded[0]))
|
|
row = row_imgs[0]
|
|
for img in row_imgs[1:]:
|
|
row = np.hstack([row, gap_w, img])
|
|
rows.append(row)
|
|
canvas = rows[0]
|
|
for r in rows[1:]:
|
|
canvas = np.vstack([canvas, gap_h, r])
|
|
return canvas
|
|
|
|
|
|
# ============================================================
|
|
# Dataset discovery / leitura
|
|
# ============================================================
|
|
|
|
|
|
def is_dataset_root(path: Path) -> bool:
|
|
# original RAW: metas + bins
|
|
if (path / "metas").is_dir() and (path / "bins").is_dir():
|
|
return True
|
|
# normalizado novo: metas + tensors
|
|
if (path / "metas").is_dir() and (path / "tensors").is_dir():
|
|
return True
|
|
# dataset com metas + masks pode ser parcialmente auditável, mas precisa payload no meta
|
|
if (path / "metas").is_dir() and ((path / "masks").is_dir() or (path / "previews").is_dir()):
|
|
return True
|
|
return False
|
|
|
|
|
|
def find_dataset_roots(path: Path) -> List[Path]:
|
|
p = path.resolve()
|
|
candidates: List[Path] = []
|
|
if p.is_file():
|
|
candidates.extend([p.parent, p.parent.parent])
|
|
else:
|
|
candidates.extend([p, p.parent])
|
|
|
|
roots: List[Path] = []
|
|
for c in candidates:
|
|
root = c.parent if c.name.lower() in ("metas", "bins", "masks", "previews", "tensors") else c
|
|
if is_dataset_root(root):
|
|
roots.append(root)
|
|
|
|
search_base = p if p.is_dir() else p.parent
|
|
if not roots and search_base.exists():
|
|
for metas_dir in search_base.rglob("metas"):
|
|
root = metas_dir.parent
|
|
if is_dataset_root(root):
|
|
roots.append(root)
|
|
|
|
unique: List[Path] = []
|
|
seen = set()
|
|
for r in roots:
|
|
rr = r.resolve()
|
|
if rr not in seen:
|
|
seen.add(rr)
|
|
unique.append(rr)
|
|
if not unique:
|
|
raise FileNotFoundError(
|
|
f"Não consegui detectar dataset_root a partir de {path}. "
|
|
"Esperado root/metas + root/bins ou root/tensors."
|
|
)
|
|
return unique
|
|
|
|
|
|
def list_meta_files(dataset_root: Path) -> List[Path]:
|
|
metas = sorted((dataset_root / "metas").glob("*.json"))
|
|
if not metas:
|
|
raise RuntimeError(f"Nenhum .json encontrado em {dataset_root / 'metas'}")
|
|
return metas
|
|
|
|
|
|
def resolve_mask_path(dataset_root: Path, meta_path: Path, meta: Optional[dict] = None) -> Optional[Path]:
|
|
meta = meta or {}
|
|
candidates: List[Path] = []
|
|
|
|
# Caminho explícito do normalizador novo.
|
|
for key in ("saved_mask_path", "mask_path"):
|
|
val = meta.get(key)
|
|
if val:
|
|
p = Path(str(val))
|
|
candidates.extend([dataset_root / p, meta_path.parent / p, Path.cwd() / p])
|
|
|
|
head_masks = meta.get("head_masks", {}) if isinstance(meta.get("head_masks"), dict) else {}
|
|
sem = head_masks.get("semantic", {}) if isinstance(head_masks.get("semantic"), dict) else {}
|
|
if sem.get("path"):
|
|
p = Path(str(sem.get("path")))
|
|
candidates.extend([dataset_root / p, meta_path.parent / p, Path.cwd() / p])
|
|
|
|
real_stem = meta_path.stem
|
|
masks_dir = dataset_root / "masks"
|
|
for ext in (".npy", ".png", ".tif", ".tiff"):
|
|
candidates.append(masks_dir / f"{real_stem}{ext}")
|
|
for suffix in ("_mask", "_gt", "_label", "_labels", "_seg"):
|
|
for ext in (".npy", ".png", ".tif", ".tiff"):
|
|
candidates.append(masks_dir / f"{real_stem}{suffix}{ext}")
|
|
if masks_dir.is_dir():
|
|
candidates.extend(sorted(masks_dir.glob(f"{real_stem}*.*")))
|
|
|
|
for p in candidates:
|
|
p = Path(p)
|
|
if p.exists() and p.suffix.lower() in (".npy", ".png", ".tif", ".tiff"):
|
|
return p
|
|
return None
|
|
|
|
|
|
def resolve_tensor_npy_path(meta_path: Path, dataset_root: Path, meta: dict) -> Optional[Path]:
|
|
candidates: List[Path] = []
|
|
for key in ("saved_tensor_path", "tensor_path"):
|
|
val = meta.get(key)
|
|
if val:
|
|
p = Path(str(val))
|
|
candidates.extend([dataset_root / p, meta_path.parent / p, Path.cwd() / p])
|
|
candidates.append(dataset_root / "tensors" / f"{meta_path.stem}.npy")
|
|
for p in candidates:
|
|
if p.exists() and p.suffix.lower() == ".npy":
|
|
return p
|
|
return None
|
|
|
|
|
|
def resolve_tensor_payload(meta_path: Path, dataset_root: Path, meta: dict) -> Optional[Path]:
|
|
stem = meta_path.stem
|
|
candidates: List[Path] = []
|
|
bins = dataset_root / "bins"
|
|
val = meta.get("saved_payload_path")
|
|
if val:
|
|
p = Path(str(val))
|
|
candidates.extend([bins / p.name, dataset_root / p, meta_path.parent / p, Path.cwd() / p])
|
|
candidates.extend([
|
|
bins / f"{stem}.raw",
|
|
bins / f"{stem}.bin",
|
|
bins / f"{stem}_multispec.raw",
|
|
bins / f"{stem}_multispec.bin",
|
|
bins / f"{stem}_offline_multispec.raw",
|
|
bins / f"{stem}_offline_multispec.bin",
|
|
])
|
|
for c in candidates:
|
|
if c.exists():
|
|
return c
|
|
return None
|
|
|
|
|
|
def resolve_camera_payloads(meta_path: Path, dataset_root: Path, meta: dict) -> Dict[str, Path]:
|
|
stem = meta_path.stem
|
|
bins = dataset_root / "bins"
|
|
paths: Dict[str, Path] = {}
|
|
saved_payload_paths = meta.get("saved_payload_paths", {}) or {}
|
|
if not isinstance(saved_payload_paths, dict) or not saved_payload_paths:
|
|
# Fallback pelo padrão de nome.
|
|
for cam_id in ("CAM_A", "CAM_B", "CAM_C"):
|
|
for ext in (".bin", ".raw"):
|
|
p = bins / f"{stem}_{cam_id}{ext}"
|
|
if p.exists():
|
|
paths[cam_id] = p
|
|
break
|
|
if paths:
|
|
return paths
|
|
raise FileNotFoundError(f"saved_payload_paths ausente e bins por câmera não encontrados para {meta_path.name}")
|
|
|
|
for cam_id, fname in saved_payload_paths.items():
|
|
fp = Path(str(fname))
|
|
candidates = [
|
|
bins / fp.name,
|
|
meta_path.parent / fp,
|
|
dataset_root / fp,
|
|
Path.cwd() / fp,
|
|
bins / f"{stem}_{cam_id}.bin",
|
|
bins / f"{stem}_{cam_id}.raw",
|
|
bins / f"{stem}_{str(cam_id).lower()}.bin",
|
|
bins / f"{stem}_{str(cam_id).lower()}.raw",
|
|
]
|
|
found = None
|
|
for c in candidates:
|
|
if Path(c).exists():
|
|
found = Path(c)
|
|
break
|
|
if found is None:
|
|
raise FileNotFoundError(f"Payload bruto não encontrado para {cam_id} em {meta_path.name}: {fname}")
|
|
paths[str(cam_id)] = found
|
|
return paths
|
|
|
|
|
|
def resolve_module_params_path(meta_path: Path, dataset_root: Path, meta: dict) -> Optional[str]:
|
|
candidates: List[Path] = []
|
|
for key in ("camera_params_json", "module_params_json"):
|
|
val = meta.get(key)
|
|
if val:
|
|
p = Path(str(val))
|
|
if p.is_absolute():
|
|
candidates.append(p)
|
|
else:
|
|
candidates.extend([Path.cwd() / p, meta_path.parent / p, dataset_root / p, dataset_root.parent / p, dataset_root.parent.parent / p])
|
|
candidates.append(Path.cwd() / "calibration" / "module_params.json")
|
|
for c in candidates:
|
|
if c.exists():
|
|
return str(c)
|
|
return None
|
|
|
|
|
|
@dataclass
|
|
class TensorLoadResult:
|
|
tensor: np.ndarray
|
|
meta: dict
|
|
payload_path: Path
|
|
source_kind: str
|
|
core_telemetry: Dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
def get_raw_processor_core(core_cache: Dict[Tuple[int, int, str, str], Any], sensor_width: int, sensor_height: int, bayer: str, calib_path: str):
|
|
key = (int(sensor_width), int(sensor_height), str(bayer).upper(), str(Path(calib_path).resolve()))
|
|
if key not in core_cache:
|
|
if RawProcessorCore is None:
|
|
raise RuntimeError("RawProcessorCore não importado. Rode a partir da raiz do projeto e confira core/raw_processor_core.py.")
|
|
core_cache[key] = RawProcessorCore(
|
|
sensor_width=int(sensor_width),
|
|
sensor_height=int(sensor_height),
|
|
bayer_pattern=str(bayer).upper(),
|
|
calibration_json_path=str(calib_path),
|
|
)
|
|
return core_cache[key]
|
|
|
|
|
|
def build_processing_meta_for_core(meta: dict) -> dict:
|
|
stream_meta = dict(meta.get("stream_meta", {}) or {})
|
|
stream_meta["frame_type"] = "RAW_BRUTO"
|
|
if "camera_info" not in stream_meta and isinstance(meta.get("camera_info"), dict):
|
|
stream_meta["camera_info"] = meta.get("camera_info")
|
|
for key in ("actual_camera_controls", "startup_camera_controls", "camera_controls", "radiometric_last_result"):
|
|
if meta.get(key) is not None:
|
|
stream_meta[key] = meta.get(key)
|
|
return stream_meta
|
|
|
|
|
|
def capture_core_telemetry(core: Any, normalized_meta: Optional[dict] = None) -> Dict[str, Any]:
|
|
out: Dict[str, Any] = {}
|
|
if core is not None:
|
|
for name, attr in (
|
|
("radiometric_normalization", "last_radiometric_normalization_result"),
|
|
("patch_normalization", "last_patch_normalization_result"),
|
|
("frame_quality", "last_frame_quality_result"),
|
|
("fusion", "last_fusion_result"),
|
|
("decode_perf", "last_decode_perf"),
|
|
):
|
|
val = getattr(core, attr, None)
|
|
if val is not None:
|
|
try:
|
|
json.dumps(val, default=str)
|
|
out[name] = val
|
|
except Exception:
|
|
out[name] = str(val)
|
|
if normalized_meta:
|
|
processing = normalized_meta.get("processing", {}) if isinstance(normalized_meta.get("processing"), dict) else {}
|
|
# Meta normalizado antigo/novo já pode carregar essas telemetrias.
|
|
for src_key, dst_key in (
|
|
("radiometric_normalization_result", "radiometric_normalization"),
|
|
("patch_normalization_result", "patch_normalization"),
|
|
("frame_quality", "frame_quality"),
|
|
("fusion_result", "fusion"),
|
|
):
|
|
if src_key in processing and dst_key not in out:
|
|
out[dst_key] = processing.get(src_key)
|
|
return out
|
|
|
|
|
|
def build_multispec_from_raw_native_multi(meta_path: Path, dataset_root: Path, meta: dict, core_cache: Dict[Tuple[int, int, str, str], Any]) -> TensorLoadResult:
|
|
saved_dtypes = meta.get("saved_payload_dtypes", {}) or {}
|
|
saved_shapes = meta.get("saved_payload_shapes", {}) or {}
|
|
cam_paths = resolve_camera_payloads(meta_path, dataset_root, meta)
|
|
|
|
frame: Dict[str, np.ndarray] = {}
|
|
for cam_id, payload_path in cam_paths.items():
|
|
dtype = saved_dtypes.get(cam_id)
|
|
shape = saved_shapes.get(cam_id)
|
|
if dtype is None or shape is None:
|
|
raise RuntimeError(f"Faltam saved_payload_dtypes/shapes para {cam_id} em {meta_path.name}")
|
|
frame[cam_id] = np.fromfile(str(payload_path), dtype=np.dtype(dtype)).reshape(tuple(shape))
|
|
|
|
sensor_width = int(meta.get("sensor_width", 1280))
|
|
sensor_height = int(meta.get("sensor_height", 800))
|
|
bayer = str(meta.get("bayer_pattern", "BGGR")).upper()
|
|
calib_path = resolve_module_params_path(meta_path, dataset_root, meta)
|
|
if calib_path is None:
|
|
raise FileNotFoundError(f"module_params.json não encontrado para {meta_path.name}")
|
|
|
|
core = get_raw_processor_core(core_cache, sensor_width, sensor_height, bayer, calib_path)
|
|
processing_meta = build_processing_meta_for_core(meta)
|
|
tensor = core.build_infer_tensor_from_stream(frame, processing_meta, 5)
|
|
if tensor is None:
|
|
raise RuntimeError(f"RawProcessorCore retornou None para {meta_path.name}")
|
|
|
|
tensor = np.asarray(tensor, dtype=np.float32)
|
|
if tensor.ndim != 3:
|
|
raise RuntimeError(f"Tensor reconstruído inválido: {meta_path.name} shape={tensor.shape}")
|
|
if tensor.shape[0] != 5 and tensor.shape[-1] == 5:
|
|
tensor = np.transpose(tensor, (2, 0, 1))
|
|
if tensor.shape[0] < 5:
|
|
raise RuntimeError(f"Tensor precisa de 5 canais, recebido {tensor.shape}")
|
|
|
|
first_payload = next(iter(cam_paths.values()))
|
|
return TensorLoadResult(
|
|
tensor=np.ascontiguousarray(tensor[:5]),
|
|
meta=meta,
|
|
payload_path=first_payload,
|
|
source_kind="raw_native_multi_reconstructed",
|
|
core_telemetry=capture_core_telemetry(core),
|
|
)
|
|
|
|
|
|
def load_multispec_tensor(meta_path: Path, dataset_root: Path, core_cache: Dict[Tuple[int, int, str, str], Any]) -> TensorLoadResult:
|
|
meta = load_json(meta_path)
|
|
saved_type = str(meta.get("saved_payload_type") or "").lower()
|
|
|
|
if saved_type == "raw_native_multi" or "saved_payload_paths" in meta:
|
|
return build_multispec_from_raw_native_multi(meta_path, dataset_root, meta, core_cache)
|
|
|
|
if saved_type == "tensor_npy" or meta.get("saved_tensor_path") or (dataset_root / "tensors" / f"{meta_path.stem}.npy").exists():
|
|
p = resolve_tensor_npy_path(meta_path, dataset_root, meta)
|
|
if p is None:
|
|
raise FileNotFoundError(f"Tensor .npy não encontrado para {meta_path.name}")
|
|
arr = np.load(str(p)).astype(np.float32, copy=False)
|
|
if arr.ndim != 3:
|
|
raise RuntimeError(f"Tensor NPY inválido: {p} shape={arr.shape}")
|
|
if arr.shape[0] != 5 and arr.shape[-1] == 5:
|
|
arr = np.transpose(arr, (2, 0, 1))
|
|
if arr.shape[0] < 5:
|
|
raise RuntimeError(f"Tensor NPY precisa de 5 canais: {p} shape={arr.shape}")
|
|
return TensorLoadResult(
|
|
tensor=np.ascontiguousarray(arr[:5]),
|
|
meta=meta,
|
|
payload_path=p,
|
|
source_kind="tensor_npy",
|
|
core_telemetry=capture_core_telemetry(None, normalized_meta=meta),
|
|
)
|
|
|
|
payload_path = resolve_tensor_payload(meta_path, dataset_root, meta)
|
|
if payload_path is None:
|
|
raise FileNotFoundError(f"Payload tensor final não encontrado para {meta_path.name}")
|
|
|
|
dtype = meta.get("saved_payload_dtype", "float32")
|
|
shape = meta.get("saved_payload_shape") or meta.get("tensor_shape") or meta.get("shape")
|
|
if shape is None:
|
|
raise RuntimeError(f"Shape do tensor não encontrado em {meta_path.name}")
|
|
arr = np.fromfile(str(payload_path), dtype=np.dtype(dtype)).reshape(tuple(shape)).astype(np.float32, copy=False)
|
|
if arr.ndim != 3:
|
|
raise RuntimeError(f"Tensor inválido em {payload_path}: shape={arr.shape}")
|
|
if arr.shape[0] != 5 and arr.shape[-1] == 5:
|
|
arr = np.transpose(arr, (2, 0, 1))
|
|
if arr.shape[0] < 5:
|
|
raise RuntimeError(f"Tensor precisa de 5 canais [R,G,B,RE,NIR], recebido shape={arr.shape}")
|
|
return TensorLoadResult(
|
|
tensor=np.ascontiguousarray(arr[:5]),
|
|
meta=meta,
|
|
payload_path=payload_path,
|
|
source_kind="tensor_binary_payload",
|
|
core_telemetry=capture_core_telemetry(None, normalized_meta=meta),
|
|
)
|
|
|
|
|
|
# ============================================================
|
|
# Máscaras
|
|
# ============================================================
|
|
|
|
|
|
def decode_color_mask(mask_img: np.ndarray, ignore_index: int) -> np.ndarray:
|
|
if mask_img.ndim == 2:
|
|
return mask_img.astype(np.int32, copy=False)
|
|
bgr = mask_img[:, :, :3]
|
|
out = np.full(mask_img.shape[:2], ignore_index, dtype=np.int32)
|
|
for rgb_color, cls_id in DEFAULT_MASK_COLOR_MAP_RGB.items():
|
|
r, g, b = rgb_color
|
|
bgr_color = np.array([b, g, r], dtype=np.uint8)
|
|
hit = np.all(bgr == bgr_color, axis=2)
|
|
out[hit] = int(cls_id)
|
|
# fallback caso algum pipeline já entregue RGB
|
|
for rgb_color, cls_id in DEFAULT_MASK_COLOR_MAP_RGB.items():
|
|
rgb_arr = np.array(rgb_color, dtype=np.uint8)
|
|
hit = np.all(bgr == rgb_arr, axis=2)
|
|
out[(out == ignore_index) & hit] = int(cls_id)
|
|
return out
|
|
|
|
|
|
def load_mask(mask_path: Optional[Path], target_hw: Tuple[int, int], ignore_index: int) -> Optional[np.ndarray]:
|
|
if mask_path is None:
|
|
return None
|
|
if mask_path.suffix.lower() == ".npy":
|
|
mask = np.load(str(mask_path))
|
|
if mask.ndim == 3:
|
|
mask = decode_color_mask(mask.astype(np.uint8), ignore_index)
|
|
else:
|
|
mask_img = cv2.imread(str(mask_path), cv2.IMREAD_UNCHANGED)
|
|
if mask_img is None:
|
|
raise RuntimeError(f"Falha ao ler máscara: {mask_path}")
|
|
mask = decode_color_mask(mask_img, ignore_index)
|
|
mask = mask.astype(np.int32, copy=False)
|
|
h, w = target_hw
|
|
if mask.shape[:2] != (h, w):
|
|
mask = cv2.resize(mask, (w, h), interpolation=cv2.INTER_NEAREST)
|
|
return mask
|
|
|
|
|
|
def mask_unique_summary(mask: Optional[np.ndarray], class_map: Dict[int, str], max_items: int = 20) -> str:
|
|
if mask is None:
|
|
return "none"
|
|
vals, counts = np.unique(mask, return_counts=True)
|
|
parts = []
|
|
for v, c in zip(vals[:max_items], counts[:max_items]):
|
|
vi = int(v)
|
|
name = class_map.get(vi, "ignore" if vi == 255 else "unk")
|
|
parts.append(f"{vi}:{name}:{int(c)}")
|
|
if len(vals) > max_items:
|
|
parts.append("...")
|
|
return ",".join(parts)
|
|
|
|
|
|
def colorize_mask(mask: Optional[np.ndarray], class_map: Dict[int, str], target_hw: Tuple[int, int]) -> np.ndarray:
|
|
h, w = target_hw
|
|
out = np.zeros((h, w, 3), dtype=np.uint8)
|
|
if mask is None:
|
|
return out
|
|
palette = dict(DEFAULT_VIS_PALETTE_BGR)
|
|
for cls_id in np.unique(mask):
|
|
ci = int(cls_id)
|
|
if ci in palette:
|
|
out[mask == ci] = palette[ci]
|
|
elif ci in class_map:
|
|
rng = np.random.default_rng(ci)
|
|
out[mask == ci] = rng.integers(40, 220, size=3)
|
|
return out
|
|
|
|
|
|
def mask_edges_on_rgb(rgb_bgr: np.ndarray, mask: Optional[np.ndarray]) -> np.ndarray:
|
|
out = rgb_bgr.copy()
|
|
if mask is None:
|
|
return out
|
|
m = mask.astype(np.uint8)
|
|
edges = cv2.Canny(m, 0, 1)
|
|
out[edges > 0] = (0, 255, 255)
|
|
return out
|
|
|
|
|
|
# ============================================================
|
|
# Features / estatísticas
|
|
# ============================================================
|
|
|
|
|
|
def compute_feature_maps(tensor: np.ndarray) -> Dict[str, np.ndarray]:
|
|
r, g, b, re, nir = [tensor[i].astype(np.float32, copy=False) for i in range(5)]
|
|
features = {
|
|
"R": r,
|
|
"G": g,
|
|
"B": b,
|
|
"RE": re,
|
|
"NIR": nir,
|
|
"NDVI": (nir - r) / (nir + r + EPS),
|
|
"NDRE": (nir - re) / (nir + re + EPS),
|
|
"NIR_minus_RE": nir - re,
|
|
"NIR_over_RE": nir / (re + EPS),
|
|
"NIR_over_R": nir / (r + EPS),
|
|
"RE_over_R": re / (r + EPS),
|
|
"NIR_over_G": nir / (g + EPS),
|
|
"RE_over_G": re / (g + EPS),
|
|
"G_minus_R": g - r,
|
|
}
|
|
for k in list(features.keys()):
|
|
features[k] = np.nan_to_num(features[k], nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32)
|
|
return features
|
|
|
|
|
|
def calc_stats(values: np.ndarray, raw01: bool = False) -> Dict[str, float]:
|
|
v = values.astype(np.float32, copy=False)
|
|
v = v[np.isfinite(v)]
|
|
if v.size == 0:
|
|
return {k: 0.0 for k in (
|
|
"count", "mean", "std", "min", "p01", "p05", "p25", "p50", "p75", "p95", "p99", "max", "iqr", "p95_p05", "dark_pct", "sat_pct", "over_1_pct", "under_0_pct", "nan_pct"
|
|
)}
|
|
p = np.percentile(v, [1, 5, 25, 50, 75, 95, 99])
|
|
out = {
|
|
"count": int(v.size),
|
|
"mean": float(np.mean(v)),
|
|
"std": float(np.std(v)),
|
|
"min": float(np.min(v)),
|
|
"p01": float(p[0]),
|
|
"p05": float(p[1]),
|
|
"p25": float(p[2]),
|
|
"p50": float(p[3]),
|
|
"p75": float(p[4]),
|
|
"p95": float(p[5]),
|
|
"p99": float(p[6]),
|
|
"max": float(np.max(v)),
|
|
"iqr": float(p[4] - p[2]),
|
|
"p95_p05": float(p[5] - p[1]),
|
|
"dark_pct": 0.0,
|
|
"sat_pct": 0.0,
|
|
"over_1_pct": 0.0,
|
|
"under_0_pct": 0.0,
|
|
"nan_pct": 0.0,
|
|
}
|
|
if raw01:
|
|
out["dark_pct"] = float(np.mean(v <= 0.01) * 100.0)
|
|
out["sat_pct"] = float(np.mean(v >= 0.99) * 100.0)
|
|
out["over_1_pct"] = float(np.mean(v > 1.0) * 100.0)
|
|
out["under_0_pct"] = float(np.mean(v < 0.0) * 100.0)
|
|
return out
|
|
|
|
|
|
@dataclass
|
|
class RunningFeatureStats:
|
|
values: Dict[str, List[float]] = field(default_factory=lambda: {f: [] for f in ALL_FEATURES})
|
|
counts: Dict[str, int] = field(default_factory=lambda: {f: 0 for f in ALL_FEATURES})
|
|
|
|
def add(self, feature_name: str, values: np.ndarray, max_samples: int = 25000):
|
|
v = values.astype(np.float32, copy=False)
|
|
v = v[np.isfinite(v)]
|
|
if v.size == 0:
|
|
return
|
|
self.counts[feature_name] += int(v.size)
|
|
if v.size > max_samples:
|
|
idx = np.random.choice(v.size, size=max_samples, replace=False)
|
|
v = v[idx]
|
|
self.values[feature_name].extend(v.tolist())
|
|
|
|
def summarize(self) -> Dict[str, Dict[str, float]]:
|
|
out = {}
|
|
for f, vals in self.values.items():
|
|
arr = np.asarray(vals, dtype=np.float32)
|
|
st = calc_stats(arr, raw01=(f in CHANNELS))
|
|
st["total_pixels_seen"] = int(self.counts.get(f, 0))
|
|
st["sampled_values"] = int(arr.size)
|
|
out[f] = st
|
|
return out
|
|
|
|
|
|
# ============================================================
|
|
# Alinhamento / geometria
|
|
# ============================================================
|
|
|
|
|
|
def gradient_mag(x: np.ndarray) -> np.ndarray:
|
|
u8 = normalize_to_u8(x)
|
|
gx = cv2.Sobel(u8, cv2.CV_32F, 1, 0, ksize=3)
|
|
gy = cv2.Sobel(u8, cv2.CV_32F, 0, 1, ksize=3)
|
|
return cv2.magnitude(gx, gy).astype(np.float32)
|
|
|
|
|
|
def estimate_shift_phase(a: np.ndarray, b: np.ndarray) -> Tuple[float, float, float]:
|
|
aa = normalize_to_u8(a).astype(np.float32)
|
|
bb = normalize_to_u8(b).astype(np.float32)
|
|
try:
|
|
(dx, dy), response = cv2.phaseCorrelate(aa, bb)
|
|
return float(dx), float(dy), float(response)
|
|
except Exception:
|
|
return 0.0, 0.0, 0.0
|
|
|
|
|
|
def edge_agreement(a: np.ndarray, b: np.ndarray) -> Dict[str, float]:
|
|
ga = gradient_mag(a)
|
|
gb = gradient_mag(b)
|
|
va = ga.reshape(-1)
|
|
vb = gb.reshape(-1)
|
|
if np.std(va) < EPS or np.std(vb) < EPS:
|
|
corr = 0.0
|
|
else:
|
|
corr = float(np.corrcoef(va, vb)[0, 1])
|
|
dx, dy, resp = estimate_shift_phase(ga, gb)
|
|
return {
|
|
"edge_corr": corr,
|
|
"phase_dx": dx,
|
|
"phase_dy": dy,
|
|
"phase_mag": float(math.hypot(dx, dy)),
|
|
"phase_response": resp,
|
|
}
|
|
|
|
|
|
def make_edge_overlay(tensor: np.ndarray) -> np.ndarray:
|
|
r, g, b, re, nir = [tensor[i] for i in range(5)]
|
|
rgb_gray = (0.299 * r + 0.587 * g + 0.114 * b).astype(np.float32)
|
|
e_rgb = normalize_to_u8(gradient_mag(rgb_gray), 5, 99)
|
|
e_re = normalize_to_u8(gradient_mag(re), 5, 99)
|
|
e_nir = normalize_to_u8(gradient_mag(nir), 5, 99)
|
|
overlay = np.zeros((tensor.shape[1], tensor.shape[2], 3), dtype=np.uint8)
|
|
overlay[:, :, 1] = e_rgb # verde
|
|
overlay[:, :, 2] = e_re # vermelho
|
|
overlay[:, :, 0] = e_nir # azul
|
|
return overlay
|
|
|
|
|
|
# ============================================================
|
|
# Core telemetry extraction
|
|
# ============================================================
|
|
|
|
|
|
def get_nested(d: Any, path: str, default=None):
|
|
cur = d
|
|
for part in path.split("."):
|
|
if not isinstance(cur, dict) or part not in cur:
|
|
return default
|
|
cur = cur[part]
|
|
return cur
|
|
|
|
|
|
def extract_core_metrics(core_telemetry: Dict[str, Any]) -> Dict[str, Any]:
|
|
row: Dict[str, Any] = {}
|
|
rad = core_telemetry.get("radiometric_normalization") or {}
|
|
patch = core_telemetry.get("patch_normalization") or {}
|
|
quality = core_telemetry.get("frame_quality") or {}
|
|
fusion = core_telemetry.get("fusion") or {}
|
|
|
|
if isinstance(rad, dict):
|
|
row["radnorm_present"] = True
|
|
row["radnorm_enabled"] = bool(rad.get("enabled", False))
|
|
row["radnorm_applied"] = bool(rad.get("applied", False))
|
|
row["radnorm_warning_count"] = len(rad.get("warnings", []) or [])
|
|
row["radnorm_warnings"] = " ; ".join([str(x) for x in (rad.get("warnings", []) or [])])
|
|
summary = rad.get("summary", {}) if isinstance(rad.get("summary"), dict) else {}
|
|
row["radnorm_scale_min"] = safe_float(summary.get("scale_min"), 0.0)
|
|
row["radnorm_scale_max"] = safe_float(summary.get("scale_max"), 0.0)
|
|
row["radnorm_scale_mean"] = safe_float(summary.get("scale_mean"), 0.0)
|
|
by_role = rad.get("by_role", {}) if isinstance(rad.get("by_role"), dict) else {}
|
|
for role in ("rgb", "re", "nir"):
|
|
rr = by_role.get(role, {}) if isinstance(by_role.get(role), dict) else {}
|
|
row[f"radnorm_{role}_scale"] = safe_float(rr.get("scale_applied"), 0.0)
|
|
row[f"radnorm_{role}_actual_factor"] = safe_float(rr.get("actual_factor"), 0.0)
|
|
row[f"radnorm_{role}_reference_factor"] = safe_float(rr.get("reference_factor"), 0.0)
|
|
else:
|
|
row["radnorm_present"] = False
|
|
|
|
if isinstance(patch, dict):
|
|
row["patchnorm_present"] = True
|
|
row["patchnorm_enabled"] = bool(patch.get("enabled", False))
|
|
row["patchnorm_applied"] = bool(patch.get("applied", False))
|
|
row["patchnorm_warning_count"] = len(patch.get("warnings", []) or [])
|
|
row["patchnorm_warnings"] = " ; ".join([str(x) for x in (patch.get("warnings", []) or [])])
|
|
summ = patch.get("summary", {}) if isinstance(patch.get("summary"), dict) else {}
|
|
row["patchnorm_valid_channel_count"] = safe_int(summ.get("valid_channel_count"), 0)
|
|
row["patchnorm_max_white_sat_pct"] = safe_float(summ.get("max_white_sat_pct"), 0.0)
|
|
row["patchnorm_max_would_clip_pct"] = safe_float(summ.get("max_would_clip_pct"), 0.0)
|
|
row["patchnorm_scale_min_applied"] = safe_float(summ.get("scale_min_applied"), 0.0)
|
|
row["patchnorm_scale_max_applied"] = safe_float(summ.get("scale_max_applied"), 0.0)
|
|
else:
|
|
row["patchnorm_present"] = False
|
|
|
|
if isinstance(quality, dict):
|
|
row["core_quality_present"] = True
|
|
row["core_quality_status"] = quality.get("status", "")
|
|
row["core_usable_for_training"] = bool(quality.get("usable_for_training", True))
|
|
row["core_quality_reasons"] = " ; ".join([str(x) for x in (quality.get("reasons", []) or [])])
|
|
metrics = quality.get("metrics", {}) if isinstance(quality.get("metrics"), dict) else {}
|
|
row["core_max_tensor_sat_pct"] = safe_float(metrics.get("max_tensor_sat_pct"), 0.0)
|
|
row["core_max_tensor_dark_pct"] = safe_float(metrics.get("max_tensor_dark_pct"), 0.0)
|
|
row["core_max_tensor_over_1_pct"] = safe_float(metrics.get("max_tensor_over_1_pct"), 0.0)
|
|
row["core_max_tensor_under_0_pct"] = safe_float(metrics.get("max_tensor_under_0_pct"), 0.0)
|
|
else:
|
|
row["core_quality_present"] = False
|
|
|
|
if isinstance(fusion, dict):
|
|
row["fusion_present"] = True
|
|
row["fusion_ref_shape"] = json.dumps(fusion.get("ref_shape"), ensure_ascii=False)
|
|
row["fusion_output_shape"] = json.dumps(fusion.get("output_shape"), ensure_ascii=False)
|
|
row["fusion_crop_box"] = json.dumps(fusion.get("crop_box"), ensure_ascii=False)
|
|
row["fusion_crop_applied"] = bool(fusion.get("crop_applied", False))
|
|
row["fusion_direct_fast"] = bool(fusion.get("direct_fusion_fast", False))
|
|
profiles = fusion.get("homography_profiles_used", {})
|
|
if isinstance(profiles, dict):
|
|
for role in ("re", "nir"):
|
|
pr = profiles.get(role, {}) if isinstance(profiles.get(role), dict) else {}
|
|
row[f"homography_{role}_profile"] = pr.get("profile", "")
|
|
row[f"homography_{role}_calib_size"] = json.dumps(pr.get("calib_size"), ensure_ascii=False)
|
|
perf = fusion.get("perf", {}) if isinstance(fusion.get("perf"), dict) else {}
|
|
row["fusion_total_ms"] = safe_float(perf.get("total_ms"), 0.0)
|
|
row["fusion_radnorm_ms"] = safe_float(perf.get("radnorm_ms"), 0.0)
|
|
row["fusion_flat_ms"] = safe_float(perf.get("flat_ms"), 0.0)
|
|
row["fusion_spatial_direct_ms"] = safe_float(perf.get("spatial_direct_ms"), 0.0)
|
|
else:
|
|
row["fusion_present"] = False
|
|
|
|
return row
|
|
|
|
|
|
# ============================================================
|
|
# Saúde por amostra / gates
|
|
# ============================================================
|
|
|
|
|
|
@dataclass
|
|
class HealthResult:
|
|
score: float
|
|
status: str
|
|
approved: bool
|
|
warnings: List[str]
|
|
errors: List[str]
|
|
notes: List[str]
|
|
|
|
|
|
def evaluate_sample_health(
|
|
row: Dict[str, Any],
|
|
class_map: Dict[int, str],
|
|
args,
|
|
) -> HealthResult:
|
|
score = 100.0
|
|
warnings: List[str] = []
|
|
errors: List[str] = []
|
|
notes: List[str] = []
|
|
|
|
def warn(msg: str, penalty: float):
|
|
nonlocal score
|
|
warnings.append(msg)
|
|
score -= penalty
|
|
|
|
def err(msg: str, penalty: float):
|
|
nonlocal score
|
|
errors.append(msg)
|
|
score -= penalty
|
|
|
|
if not bool(row.get("tensor_valid", True)):
|
|
err("tensor_invalid", 60)
|
|
|
|
if not bool(row.get("mask_found", False)):
|
|
err("missing_mask", 35)
|
|
elif safe_float(row.get("mask_valid_pct"), 0.0) < args.min_valid_mask_pct:
|
|
err(f"mask_valid_pct_low:{safe_float(row.get('mask_valid_pct')):.2f}%", 25)
|
|
|
|
for ch in CHANNELS:
|
|
sat = safe_float(row.get(f"{ch}_sat_pct"), 0.0)
|
|
dark = safe_float(row.get(f"{ch}_dark_pct"), 0.0)
|
|
dyn = safe_float(row.get(f"{ch}_p95_p05"), 0.0)
|
|
mean = safe_float(row.get(f"{ch}_mean"), 0.0)
|
|
over = safe_float(row.get(f"{ch}_over_1_pct"), 0.0)
|
|
under = safe_float(row.get(f"{ch}_under_0_pct"), 0.0)
|
|
|
|
if sat >= args.bad_sat_pct:
|
|
err(f"{ch}:sat_bad:{sat:.2f}%", 18)
|
|
elif sat >= args.warn_sat_pct:
|
|
warn(f"{ch}:sat_warn:{sat:.2f}%", 5)
|
|
|
|
if dark >= args.bad_dark_pct:
|
|
err(f"{ch}:dark_bad:{dark:.2f}%", 18)
|
|
elif dark >= args.warn_dark_pct:
|
|
warn(f"{ch}:dark_warn:{dark:.2f}%", 5)
|
|
|
|
if dyn <= args.bad_low_dynamic:
|
|
err(f"{ch}:dynamic_bad:{dyn:.4f}", 14)
|
|
elif dyn <= args.warn_low_dynamic:
|
|
warn(f"{ch}:dynamic_warn:{dyn:.4f}", 4)
|
|
|
|
if mean <= args.bad_mean_low or mean >= args.bad_mean_high:
|
|
warn(f"{ch}:mean_extreme:{mean:.4f}", 4)
|
|
|
|
if over > 0.01:
|
|
warn(f"{ch}:over_1:{over:.3f}%", 4)
|
|
if under > 0.01:
|
|
warn(f"{ch}:under_0:{under:.3f}%", 4)
|
|
|
|
# Alinhamento: alerta, não juiz absoluto. Canais espectrais podem ter textura diferente.
|
|
for role in ("re", "nir"):
|
|
mag = safe_float(row.get(f"{role}_phase_mag"), 0.0)
|
|
corr = safe_float(row.get(f"{role}_edge_corr"), 0.0)
|
|
resp = safe_float(row.get(f"{role}_phase_response"), 0.0)
|
|
if mag >= args.bad_shift_px and resp >= args.min_phase_response_for_shift_gate:
|
|
err(f"{role}:shift_bad:{mag:.2f}px resp={resp:.3f}", 16)
|
|
elif mag >= args.warn_shift_px and resp >= args.min_phase_response_for_shift_gate:
|
|
warn(f"{role}:shift_warn:{mag:.2f}px resp={resp:.3f}", 5)
|
|
if corr < args.bad_edge_corr:
|
|
warn(f"{role}:edge_corr_low:{corr:.3f}", 4)
|
|
elif corr < args.warn_edge_corr:
|
|
notes.append(f"{role}:edge_corr_warn:{corr:.3f}")
|
|
|
|
# Core telemetry gates.
|
|
if row.get("radnorm_present") is True:
|
|
if bool(row.get("radnorm_enabled")) and not bool(row.get("radnorm_applied")):
|
|
if args.require_radnorm_applied:
|
|
err("radnorm_enabled_not_applied", 25)
|
|
else:
|
|
warn("radnorm_enabled_not_applied", 8)
|
|
if safe_int(row.get("radnorm_warning_count"), 0) > 0:
|
|
warn(f"radnorm_warnings:{row.get('radnorm_warnings', '')}", 6)
|
|
for role in ("rgb", "re", "nir"):
|
|
s = safe_float(row.get(f"radnorm_{role}_scale"), 0.0)
|
|
if s > 0:
|
|
if s < args.radnorm_scale_min_ok or s > args.radnorm_scale_max_ok:
|
|
warn(f"radnorm_{role}_scale_out:{s:.3f}", 6)
|
|
|
|
if row.get("patchnorm_present") is True:
|
|
if bool(row.get("patchnorm_enabled")) and not bool(row.get("patchnorm_applied")):
|
|
warn("patchnorm_enabled_not_applied", 6)
|
|
if safe_int(row.get("patchnorm_warning_count"), 0) > 0:
|
|
warn(f"patchnorm_warnings:{row.get('patchnorm_warnings', '')}", 5)
|
|
if safe_float(row.get("patchnorm_max_would_clip_pct"), 0.0) >= args.bad_patch_clip_pct:
|
|
err(f"patchnorm_clip_bad:{safe_float(row.get('patchnorm_max_would_clip_pct')):.2f}%", 15)
|
|
|
|
if row.get("core_quality_present") is True:
|
|
if str(row.get("core_quality_status", "")).lower() == "bad":
|
|
err(f"core_quality_bad:{row.get('core_quality_reasons','')}", 35)
|
|
elif str(row.get("core_quality_status", "")).lower() == "warning":
|
|
warn(f"core_quality_warning:{row.get('core_quality_reasons','')}", 6)
|
|
if row.get("core_usable_for_training") is False:
|
|
err("core_not_usable_for_training", 35)
|
|
|
|
# Tamanho mínimo de classes relevantes, quando existir no grupo/nome.
|
|
total_px = max(1, safe_int(row.get("H"), 0) * safe_int(row.get("W"), 0))
|
|
group = str(row.get("group", "")).lower()
|
|
if args.clean_min_target_pct > 0:
|
|
if "cana" in group:
|
|
pct = 100.0 * safe_float(row.get("pixels_cana"), 0.0) / total_px
|
|
if pct < args.clean_min_target_pct * 100.0:
|
|
warn(f"cana_pixels_low:{pct:.4f}%", 4)
|
|
if "erva" in group:
|
|
pct = 100.0 * safe_float(row.get("pixels_erva"), 0.0) / total_px
|
|
if pct < args.clean_min_target_pct * 100.0:
|
|
warn(f"erva_pixels_low:{pct:.4f}%", 4)
|
|
|
|
score = float(max(0.0, min(100.0, score)))
|
|
if errors:
|
|
status = "bad"
|
|
elif score < args.health_warning_score or warnings:
|
|
status = "warning"
|
|
else:
|
|
status = "good"
|
|
|
|
approved = status != "bad"
|
|
if args.reject_warnings:
|
|
approved = status == "good"
|
|
|
|
return HealthResult(score=score, status=status, approved=approved, warnings=warnings, errors=errors, notes=notes)
|
|
|
|
|
|
# ============================================================
|
|
# Visuais por amostra
|
|
# ============================================================
|
|
|
|
|
|
def make_sample_visual(
|
|
sample_name: str,
|
|
tensor: np.ndarray,
|
|
mask: Optional[np.ndarray],
|
|
class_map: Dict[int, str],
|
|
sample_stats: Dict[str, Any],
|
|
health: Optional[HealthResult] = None,
|
|
) -> np.ndarray:
|
|
features = compute_feature_maps(tensor)
|
|
h, w = tensor.shape[1], tensor.shape[2]
|
|
|
|
rgb = rgb_from_tensor(tensor, stretch=False)
|
|
rgb_stretch = rgb_from_tensor(tensor, stretch=True)
|
|
re = apply_colormap_gray(features["RE"], stretch=True)
|
|
nir = apply_colormap_gray(features["NIR"], stretch=True)
|
|
ndvi = apply_colormap_gray(features["NDVI"], stretch=True)
|
|
ndre = apply_colormap_gray(features["NDRE"], stretch=True)
|
|
diff = apply_colormap_gray(features["NIR_minus_RE"], stretch=True)
|
|
ratio = apply_colormap_gray(features["NIR_over_RE"], stretch=True)
|
|
edge = make_edge_overlay(tensor)
|
|
mask_bgr = colorize_mask(mask, class_map, (h, w))
|
|
overlay_mask = rgb.copy()
|
|
if mask is not None:
|
|
overlay_mask = cv2.addWeighted(rgb, 0.65, mask_bgr, 0.35, 0)
|
|
mask_edges = mask_edges_on_rgb(rgb, mask)
|
|
|
|
align = sample_stats.get("alignment", {})
|
|
re_align = align.get("RGBgray_vs_RE", {})
|
|
nir_align = align.get("RGBgray_vs_NIR", {})
|
|
score_txt = ""
|
|
if health is not None:
|
|
score_txt = f"health={health.status} score={health.score:.1f}"
|
|
|
|
panels = [
|
|
("RGB tensor", rgb, f"{sample_name} | {score_txt}"),
|
|
("RGB stretch", rgb_stretch, "visual p1-p99"),
|
|
("GT mask", mask_bgr, f"unique={mask_unique_summary(mask, class_map)}"),
|
|
("Mask overlay", overlay_mask, "GT sobre RGB"),
|
|
("Mask edges", mask_edges, "bordas GT sobre RGB"),
|
|
("RE", re, "canal 3 | stretch p1-p99"),
|
|
("NIR", nir, "canal 4 | stretch p1-p99"),
|
|
("NDVI", ndvi, "(NIR-R)/(NIR+R)"),
|
|
("NDRE", ndre, "(NIR-RE)/(NIR+RE)"),
|
|
("NIR - RE", diff, "diferença direta"),
|
|
("NIR / RE", ratio, "razão com eps"),
|
|
(
|
|
"Edge overlay",
|
|
edge,
|
|
f"G=RGB R=RE B=NIR | REcorr={safe_float(re_align.get('edge_corr')):.3f} NIRcorr={safe_float(nir_align.get('edge_corr')):.3f}",
|
|
),
|
|
]
|
|
return make_grid(panels, panel_w=380, cols=3)
|
|
|
|
|
|
# ============================================================
|
|
# Fixed dataset e revisão manual
|
|
# ============================================================
|
|
|
|
|
|
def resolve_fixed_out_root(dataset_roots: List[Path], args) -> Path:
|
|
if args.fixed_out_root:
|
|
return Path(args.fixed_out_root).resolve()
|
|
first = dataset_roots[0].resolve()
|
|
if first.parent.name == "group":
|
|
# dataset/original/group/chao -> dataset/fixed/group
|
|
dataset_base = first.parent.parent.parent if first.parent.parent.name in ("original", "1024x640", "640x400") else first.parent.parent
|
|
return dataset_base / "fixed" / "group"
|
|
return first.parent / "fixed" / "group"
|
|
|
|
|
|
def resolve_rejected_preview_root(fixed_root: Path) -> Path:
|
|
return fixed_root.parent / "rejected_previews" if fixed_root.name == "group" else fixed_root / "_rejected_previews"
|
|
|
|
|
|
def safe_copy_file(src: Path, dst: Path):
|
|
if not src or not Path(src).exists():
|
|
return
|
|
ensure_dir(dst.parent)
|
|
shutil.copy2(str(src), str(dst))
|
|
|
|
|
|
def copy_sample_to_fixed(row: Dict[str, Any], fixed_root: Path, copy_previews: bool):
|
|
dataset_root = Path(str(row["dataset_root"]))
|
|
group = str(row["group"])
|
|
real_stem = str(row["real_stem"])
|
|
meta_path = Path(str(row["meta_path"]))
|
|
dst_group = fixed_root / group
|
|
|
|
safe_copy_file(meta_path, dst_group / "metas" / meta_path.name)
|
|
|
|
mask_path = Path(str(row.get("mask_path", "")))
|
|
if str(mask_path) and mask_path.exists():
|
|
safe_copy_file(mask_path, dst_group / "masks" / mask_path.name)
|
|
|
|
payload_path = Path(str(row.get("payload_path", "")))
|
|
if str(payload_path) and payload_path.exists():
|
|
if payload_path.suffix.lower() == ".npy" or "tensors" in payload_path.parts:
|
|
safe_copy_file(payload_path, dst_group / "tensors" / payload_path.name)
|
|
else:
|
|
safe_copy_file(payload_path, dst_group / "bins" / payload_path.name)
|
|
|
|
# Copia todos payloads RAW multi do meta, se existirem.
|
|
try:
|
|
meta = load_json(meta_path)
|
|
saved_payload_paths = meta.get("saved_payload_paths", {}) or {}
|
|
if isinstance(saved_payload_paths, dict):
|
|
for _, fname in saved_payload_paths.items():
|
|
src = dataset_root / "bins" / Path(str(fname)).name
|
|
safe_copy_file(src, dst_group / "bins" / src.name)
|
|
saved_tensor_path = meta.get("saved_tensor_path")
|
|
if saved_tensor_path:
|
|
src = dataset_root / Path(str(saved_tensor_path))
|
|
if src.exists():
|
|
safe_copy_file(src, dst_group / "tensors" / src.name)
|
|
except Exception:
|
|
pass
|
|
|
|
if copy_previews:
|
|
for ext in (".png", ".jpg", ".jpeg", ".webp"):
|
|
src = dataset_root / "previews" / f"{real_stem}{ext}"
|
|
if src.exists():
|
|
safe_copy_file(src, dst_group / "previews" / src.name)
|
|
|
|
|
|
def find_original_preview(dataset_root: Path, real_stem: str) -> Optional[Path]:
|
|
previews_dir = dataset_root / "previews"
|
|
for ext in (".png", ".jpg", ".jpeg", ".webp"):
|
|
p = previews_dir / f"{real_stem}{ext}"
|
|
if p.exists():
|
|
return p
|
|
if previews_dir.is_dir():
|
|
matches: List[Path] = []
|
|
for ext in (".png", ".jpg", ".jpeg", ".webp"):
|
|
matches.extend(previews_dir.glob(f"{real_stem}*{ext}"))
|
|
if matches:
|
|
return matches[0]
|
|
return None
|
|
|
|
|
|
def resize_preview_max_width(img: np.ndarray, max_width: int) -> np.ndarray:
|
|
if img is None or img.size == 0 or max_width <= 0 or img.shape[1] <= max_width:
|
|
return img
|
|
scale = max_width / img.shape[1]
|
|
return cv2.resize(img, (int(img.shape[1] * scale), int(img.shape[0] * scale)), interpolation=cv2.INTER_AREA)
|
|
|
|
|
|
def draw_rejected_header(img: np.ndarray, row: Dict[str, Any]) -> np.ndarray:
|
|
out = img.copy()
|
|
h, w = out.shape[:2]
|
|
header_h = 116
|
|
canvas = np.zeros((h + header_h, w, 3), dtype=np.uint8)
|
|
canvas[:header_h, :] = (20, 20, 20)
|
|
canvas[header_h:, :] = out
|
|
line1 = cv_text(str(row.get("sample", "")))[:140]
|
|
line2 = f"status={row.get('health_status','')} score={safe_float(row.get('health_score')):.1f} approved={row.get('approved_for_training','')}"
|
|
line3 = cv_text(str(row.get("reject_reasons", row.get("health_errors", ""))))[:170]
|
|
line4 = cv_text(str(row.get("health_warnings", "")))[:170]
|
|
cv2.putText(canvas, line1, (10, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.58, (0, 255, 255), 2, cv2.LINE_AA)
|
|
cv2.putText(canvas, line2, (10, 52), cv2.FONT_HERSHEY_SIMPLEX, 0.50, (255, 255, 255), 1, cv2.LINE_AA)
|
|
cv2.putText(canvas, line3, (10, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.43, (120, 220, 255), 1, cv2.LINE_AA)
|
|
cv2.putText(canvas, line4, (10, 105), cv2.FONT_HERSHEY_SIMPLEX, 0.40, (160, 160, 255), 1, cv2.LINE_AA)
|
|
return canvas
|
|
|
|
|
|
def save_rejected_preview(row: Dict[str, Any], fixed_root: Path, args, tensor=None, mask=None, class_map=None, audit_panel=None):
|
|
rejected_root = resolve_rejected_preview_root(fixed_root)
|
|
group = str(row.get("group", "unknown"))
|
|
dataset_root = Path(str(row.get("dataset_root", ".")))
|
|
real_stem = str(row.get("real_stem", "sample"))
|
|
dst_dir = ensure_dir(rejected_root / group)
|
|
reason_tag = str(row.get("health_status", "rejected"))
|
|
dst_path = dst_dir / f"{row.get('sample', real_stem)}__{reason_tag}.png"
|
|
|
|
img = None
|
|
mode = str(args.rejected_preview_source)
|
|
if mode in ("auto", "audit_panel") and audit_panel is not None:
|
|
img = audit_panel
|
|
if img is None and mode in ("auto", "preview"):
|
|
p = find_original_preview(dataset_root, real_stem)
|
|
if p is not None:
|
|
img = cv2.imread(str(p), cv2.IMREAD_COLOR)
|
|
if img is None and tensor is not None and mode in ("auto", "rgb_tensor", "audit_panel"):
|
|
img = rgb_from_tensor(tensor, stretch=False)
|
|
if mask is not None and class_map is not None:
|
|
mask_bgr = colorize_mask(mask, class_map, (tensor.shape[1], tensor.shape[2]))
|
|
img = cv2.addWeighted(img, 0.70, mask_bgr, 0.30, 0)
|
|
if img is None:
|
|
img = np.zeros((360, 640, 3), dtype=np.uint8)
|
|
img = resize_preview_max_width(img, int(args.rejected_preview_max_width))
|
|
img = draw_rejected_header(img, row)
|
|
cv2.imwrite(str(dst_path), img)
|
|
|
|
|
|
def manual_fit_to_width(img: np.ndarray, max_width: int) -> np.ndarray:
|
|
if max_width <= 0 or img.shape[1] <= max_width:
|
|
return img
|
|
scale = max_width / img.shape[1]
|
|
return cv2.resize(img, (int(img.shape[1] * scale), int(img.shape[0] * scale)), interpolation=cv2.INTER_AREA)
|
|
|
|
|
|
def draw_manual_review_bar(canvas: np.ndarray, row: Dict[str, Any], idx: int, total: int) -> np.ndarray:
|
|
h, w = canvas.shape[:2]
|
|
bar_h = 112
|
|
out = np.zeros((h + bar_h, w, 3), dtype=np.uint8)
|
|
out[:bar_h, :] = (18, 18, 18)
|
|
out[bar_h:, :] = canvas
|
|
line1 = f"[{idx}/{total}] group={row.get('group')} | status={row.get('health_status')} score={safe_float(row.get('health_score')):.1f} | {row.get('sample')}"
|
|
line2 = "A/ENTER aprova | R rejeita | S pula | Q/ESC finaliza | 1 alinhamento | 2 mascara | 3 saturacao | 4 escuro | 5 classe | 6 duplicado"
|
|
line3 = cv_text(str(row.get("health_errors", "")))[:180]
|
|
line4 = cv_text(str(row.get("health_warnings", "")))[:180]
|
|
cv2.putText(out, cv_text(line1)[:180], (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.56, (0, 255, 255), 2, cv2.LINE_AA)
|
|
cv2.putText(out, cv_text(line2), (10, 53), cv2.FONT_HERSHEY_SIMPLEX, 0.49, (255, 255, 255), 1, cv2.LINE_AA)
|
|
if line3:
|
|
cv2.putText(out, line3, (10, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.42, (120, 220, 255), 1, cv2.LINE_AA)
|
|
if line4:
|
|
cv2.putText(out, line4, (10, 104), cv2.FONT_HERSHEY_SIMPLEX, 0.38, (160, 160, 255), 1, cv2.LINE_AA)
|
|
return out
|
|
|
|
|
|
def manual_review_decision(canvas: np.ndarray, row: Dict[str, Any], idx: int, total: int, args) -> Tuple[str, str]:
|
|
window_name = str(args.manual_window_name)
|
|
view = draw_manual_review_bar(canvas, row, idx, total)
|
|
view = manual_fit_to_width(view, int(args.manual_window_width))
|
|
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
|
|
cv2.imshow(window_name, view)
|
|
reason_map = {
|
|
ord("1"): "manual_reject_alignment",
|
|
ord("2"): "manual_reject_mask",
|
|
ord("3"): "manual_reject_saturation",
|
|
ord("4"): "manual_reject_dark_dynamic",
|
|
ord("5"): "manual_reject_wrong_class",
|
|
ord("6"): "manual_reject_duplicate_or_bad_frame",
|
|
}
|
|
while True:
|
|
key = cv2.waitKey(0) & 0xFF
|
|
if key in (ord("a"), ord("A"), 13, 32):
|
|
return "approved", "manual_approved"
|
|
if key in (ord("r"), ord("R"), 8, 127):
|
|
return "rejected", "manual_rejected"
|
|
if key in reason_map:
|
|
return "rejected", reason_map[key]
|
|
if key in (ord("s"), ord("S")):
|
|
return "skipped", "manual_skipped"
|
|
if key in (ord("q"), ord("Q"), 27):
|
|
return "quit", "manual_quit"
|
|
|
|
|
|
# ============================================================
|
|
# Diagnóstico global
|
|
# ============================================================
|
|
|
|
|
|
def summarize_by_group(sample_rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
groups: Dict[str, List[Dict[str, Any]]] = {}
|
|
for r in sample_rows:
|
|
groups.setdefault(str(r.get("group", "unknown")), []).append(r)
|
|
rows = []
|
|
for g, items in sorted(groups.items()):
|
|
n = len(items)
|
|
approved = sum(1 for r in items if bool(r.get("approved_for_training")))
|
|
bad = sum(1 for r in items if r.get("health_status") == "bad")
|
|
warning = sum(1 for r in items if r.get("health_status") == "warning")
|
|
row = {
|
|
"group": g,
|
|
"samples": n,
|
|
"approved": approved,
|
|
"approval_pct": 100.0 * approved / max(1, n),
|
|
"bad": bad,
|
|
"warning": warning,
|
|
"mean_health_score": float(np.mean([safe_float(r.get("health_score")) for r in items])) if items else 0.0,
|
|
}
|
|
for cls_name in ("chao", "cana", "erva"):
|
|
row[f"pixels_{cls_name}"] = int(sum(safe_int(r.get(f"pixels_{cls_name}"), 0) for r in items))
|
|
for ch in CHANNELS:
|
|
row[f"{ch}_sat_pct_mean"] = float(np.mean([safe_float(r.get(f"{ch}_sat_pct")) for r in items]))
|
|
row[f"{ch}_dark_pct_mean"] = float(np.mean([safe_float(r.get(f"{ch}_dark_pct")) for r in items]))
|
|
row[f"{ch}_dyn_mean"] = float(np.mean([safe_float(r.get(f"{ch}_p95_p05")) for r in items]))
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def build_separability(class_summary: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
|
|
separability: Dict[str, Any] = {}
|
|
class_items = list(class_summary.items())
|
|
for feature in ALL_FEATURES:
|
|
vals = []
|
|
for cls_id, item in class_items:
|
|
st = item.get("features", {}).get(feature, {})
|
|
vals.append((item.get("class_name", str(cls_id)), safe_float(st.get("mean")), safe_float(st.get("std"))))
|
|
rows = []
|
|
for i in range(len(vals)):
|
|
for j in range(i + 1, len(vals)):
|
|
a_name, a_mean, a_std = vals[i]
|
|
b_name, b_mean, b_std = vals[j]
|
|
pooled = math.sqrt((a_std * a_std + b_std * b_std) / 2.0) + EPS
|
|
d = abs(a_mean - b_mean) / pooled
|
|
rows.append({"pair": f"{a_name}_vs_{b_name}", "effect_size_d": float(d), "mean_a": a_mean, "mean_b": b_mean})
|
|
separability[feature] = rows
|
|
ranking = []
|
|
for feature, rows in separability.items():
|
|
if rows:
|
|
ranking.append({"feature": feature, "avg_effect_size_d": float(np.mean([r["effect_size_d"] for r in rows]))})
|
|
ranking.sort(key=lambda x: x["avg_effect_size_d"], reverse=True)
|
|
return separability, ranking
|
|
|
|
|
|
def build_global_health(sample_rows: List[Dict[str, Any]], global_summary: Dict[str, Any], class_summary: Dict[str, Any], warnings: List[Dict[str, Any]], args) -> Dict[str, Any]:
|
|
n = len(sample_rows)
|
|
if n <= 0:
|
|
return {"status": "bad", "score": 0.0, "verdict": "Nenhuma amostra processada."}
|
|
approved = sum(1 for r in sample_rows if bool(r.get("approved_for_training")))
|
|
bad = sum(1 for r in sample_rows if r.get("health_status") == "bad")
|
|
warning = sum(1 for r in sample_rows if r.get("health_status") == "warning")
|
|
mean_score = float(np.mean([safe_float(r.get("health_score")) for r in sample_rows]))
|
|
approval_pct = 100.0 * approved / n
|
|
bad_pct = 100.0 * bad / n
|
|
warning_pct = 100.0 * warning / n
|
|
|
|
issues = []
|
|
positives = []
|
|
score = mean_score
|
|
|
|
if approval_pct < args.dataset_min_approval_pct:
|
|
issues.append(f"approval_pct abaixo do mínimo: {approval_pct:.1f}% < {args.dataset_min_approval_pct:.1f}%")
|
|
score -= 15
|
|
else:
|
|
positives.append(f"approval_pct bom: {approval_pct:.1f}%")
|
|
if bad_pct > args.dataset_max_bad_pct:
|
|
issues.append(f"bad_pct alto: {bad_pct:.1f}% > {args.dataset_max_bad_pct:.1f}%")
|
|
score -= 15
|
|
if len(warnings) > 0:
|
|
positives.append(f"auditoria gerou rastreabilidade com {len(warnings)} avisos/exceções")
|
|
|
|
for ch in CHANNELS:
|
|
st = global_summary.get(ch, {})
|
|
dyn = safe_float(st.get("p95_p05"), 0.0)
|
|
sat = safe_float(st.get("sat_pct"), 0.0)
|
|
dark = safe_float(st.get("dark_pct"), 0.0)
|
|
if dyn < args.warn_low_dynamic:
|
|
issues.append(f"{ch}: dinâmica global baixa {dyn:.4f}")
|
|
score -= 4
|
|
if sat > args.warn_sat_pct:
|
|
issues.append(f"{ch}: saturação global {sat:.2f}%")
|
|
score -= 4
|
|
if dark > args.warn_dark_pct:
|
|
issues.append(f"{ch}: dark global {dark:.2f}%")
|
|
score -= 4
|
|
|
|
separability, ranking = build_separability(class_summary)
|
|
if ranking:
|
|
top = ranking[0]
|
|
if top["avg_effect_size_d"] >= args.min_top_feature_effect_size:
|
|
positives.append(f"separabilidade espectral útil: top={top['feature']} d≈{top['avg_effect_size_d']:.2f}")
|
|
else:
|
|
issues.append(f"separabilidade espectral fraca: top={top['feature']} d≈{top['avg_effect_size_d']:.2f}")
|
|
score -= 8
|
|
|
|
score = float(max(0.0, min(100.0, score)))
|
|
if bad_pct > args.dataset_max_bad_pct or approval_pct < args.dataset_min_approval_pct:
|
|
status = "bad"
|
|
verdict = "Dataset NÃO está pronto para campo/treino oficial sem limpeza ou revisão."
|
|
elif issues or score < args.dataset_good_score:
|
|
status = "warning"
|
|
verdict = "Dataset utilizável com cautela; recomenda-se revisar avisos e rejeitados antes do treino oficial."
|
|
else:
|
|
status = "good"
|
|
verdict = "Dataset saudável para treino/campo dentro dos critérios configurados."
|
|
|
|
return {
|
|
"status": status,
|
|
"score": score,
|
|
"verdict": verdict,
|
|
"samples_total": n,
|
|
"samples_approved": approved,
|
|
"approval_pct": approval_pct,
|
|
"bad_pct": bad_pct,
|
|
"warning_pct": warning_pct,
|
|
"mean_sample_health_score": mean_score,
|
|
"issues": issues,
|
|
"positives": positives,
|
|
"feature_separability": separability,
|
|
"feature_separability_ranking": ranking,
|
|
}
|
|
|
|
|
|
# ============================================================
|
|
# Auditoria principal
|
|
# ============================================================
|
|
|
|
|
|
def audit_dataset(args):
|
|
np.random.seed(args.seed)
|
|
t_start = time.time()
|
|
|
|
dataset_roots = find_dataset_roots(Path(args.input_path))
|
|
groups_except = parse_csv_set(args.groups_except)
|
|
if groups_except:
|
|
dataset_roots = [r for r in dataset_roots if r.name.lower() not in groups_except]
|
|
|
|
out_dir = ensure_dir(args.out_dir)
|
|
visuals_dir = ensure_dir(out_dir / "visuals")
|
|
class_map = parse_class_map(args.class_map)
|
|
|
|
entries: List[Tuple[Path, Path]] = []
|
|
for root in dataset_roots:
|
|
for meta_path in list_meta_files(root):
|
|
entries.append((root, meta_path))
|
|
if args.manual_start_index and args.manual_start_index > 1:
|
|
entries = entries[int(args.manual_start_index) - 1:]
|
|
if args.limit and args.limit > 0:
|
|
entries = entries[:args.limit]
|
|
|
|
global_acc = RunningFeatureStats()
|
|
class_acc: Dict[int, RunningFeatureStats] = {cls: RunningFeatureStats() for cls in class_map.keys()}
|
|
sample_rows: List[Dict[str, Any]] = []
|
|
by_sample_class_feature_rows: List[Dict[str, Any]] = []
|
|
core_rows: List[Dict[str, Any]] = []
|
|
warnings: List[Dict[str, Any]] = []
|
|
core_cache: Dict[Tuple[int, int, str, str], Any] = {}
|
|
|
|
print(f"[INFO] dataset_roots={len(dataset_roots)}")
|
|
for r in dataset_roots:
|
|
print(f" - {r}")
|
|
print(f"[INFO] amostras={len(entries)}")
|
|
print(f"[INFO] out_dir={out_dir}")
|
|
if args.manual_review:
|
|
print("[MANUAL] A/ENTER aprova | R rejeita | S pula | Q sai | 1..6 rejeita com motivo")
|
|
|
|
manual_stop = False
|
|
|
|
for idx, (dataset_root, meta_path) in enumerate(entries, start=1):
|
|
group_name = dataset_root.name
|
|
stem = f"{group_name}__{meta_path.stem}"
|
|
canvas = None
|
|
try:
|
|
load_result = load_multispec_tensor(meta_path, dataset_root, core_cache)
|
|
tensor = load_result.tensor
|
|
meta = load_result.meta
|
|
payload_path = load_result.payload_path
|
|
core_telemetry = load_result.core_telemetry
|
|
|
|
h, w = int(tensor.shape[1]), int(tensor.shape[2])
|
|
features = compute_feature_maps(tensor)
|
|
mask_path = resolve_mask_path(dataset_root, meta_path, meta)
|
|
mask = load_mask(mask_path, (h, w), args.ignore_index)
|
|
|
|
# Tensor base health.
|
|
tensor_valid = tensor.ndim == 3 and tensor.shape[0] == 5 and np.all(np.isfinite(tensor))
|
|
tensor_nan_pct = float(np.mean(~np.isfinite(tensor)) * 100.0)
|
|
|
|
sample_feature_stats: Dict[str, Dict[str, float]] = {}
|
|
for fname, fmap in features.items():
|
|
st = calc_stats(fmap.reshape(-1), raw01=(fname in CHANNELS))
|
|
sample_feature_stats[fname] = st
|
|
global_acc.add(fname, fmap.reshape(-1), max_samples=args.max_pixels_per_feature)
|
|
|
|
class_pixel_counts: Dict[str, int] = {name: 0 for name in class_map.values()}
|
|
mask_valid_pct = 0.0
|
|
if mask is not None:
|
|
valid_mask = mask != args.ignore_index
|
|
mask_valid_pct = float(np.mean(valid_mask) * 100.0)
|
|
for cls_id, cls_name in class_map.items():
|
|
cm = (mask == cls_id) & valid_mask
|
|
n = int(np.sum(cm))
|
|
class_pixel_counts[cls_name] = n
|
|
if n < args.min_class_pixels:
|
|
continue
|
|
for fname, fmap in features.items():
|
|
vals = fmap[cm]
|
|
class_acc[cls_id].add(fname, vals, max_samples=args.max_pixels_per_feature)
|
|
st = calc_stats(vals, raw01=(fname in CHANNELS))
|
|
by_sample_class_feature_rows.append({
|
|
"sample": stem,
|
|
"group": group_name,
|
|
"class_id": cls_id,
|
|
"class_name": cls_name,
|
|
"feature": fname,
|
|
**st,
|
|
})
|
|
else:
|
|
warnings.append({"sample": stem, "type": "missing_mask", "message": "Máscara ausente."})
|
|
|
|
r, g, b, re, nir = [tensor[i] for i in range(5)]
|
|
rgb_gray = (0.299 * r + 0.587 * g + 0.114 * b).astype(np.float32)
|
|
alignment = {
|
|
"RGBgray_vs_RE": edge_agreement(rgb_gray, re),
|
|
"RGBgray_vs_NIR": edge_agreement(rgb_gray, nir),
|
|
"RE_vs_NIR": edge_agreement(re, nir),
|
|
}
|
|
core_metrics = extract_core_metrics(core_telemetry)
|
|
|
|
row: Dict[str, Any] = {
|
|
"idx": idx,
|
|
"sample": stem,
|
|
"group": group_name,
|
|
"real_stem": meta_path.stem,
|
|
"dataset_root": str(dataset_root),
|
|
"meta_path": str(meta_path),
|
|
"payload_path": str(payload_path),
|
|
"payload_sha1_head": sha1_short(payload_path),
|
|
"source_kind": load_result.source_kind,
|
|
"mask_path": str(mask_path) if mask_path else "",
|
|
"mask_found": bool(mask_path is not None),
|
|
"mask_shape": str(mask.shape) if mask is not None else "",
|
|
"mask_unique": mask_unique_summary(mask, class_map),
|
|
"mask_valid_pct": mask_valid_pct,
|
|
"H": h,
|
|
"W": w,
|
|
"tensor_shape": str(tensor.shape),
|
|
"tensor_dtype": str(tensor.dtype),
|
|
"tensor_valid": bool(tensor_valid),
|
|
"tensor_nan_pct": tensor_nan_pct,
|
|
"meta_ts": meta.get("ts") or get_nested(meta, "source_capture_meta.ts", ""),
|
|
"sync_ok": get_nested(meta, "stream_meta.sync_ok", get_nested(meta, "source_capture_meta.stream_meta.sync_ok", "")),
|
|
"sync_dt_ms": get_nested(meta, "stream_meta.sync_dt_ms", get_nested(meta, "source_capture_meta.stream_meta.sync_dt_ms", "")),
|
|
"camera_params_json": meta.get("camera_params_json", ""),
|
|
"schema": meta.get("schema", ""),
|
|
}
|
|
row.update(core_metrics)
|
|
|
|
row["re_edge_corr"] = alignment["RGBgray_vs_RE"]["edge_corr"]
|
|
row["nir_edge_corr"] = alignment["RGBgray_vs_NIR"]["edge_corr"]
|
|
row["renir_edge_corr"] = alignment["RE_vs_NIR"]["edge_corr"]
|
|
for key, al in (("re", alignment["RGBgray_vs_RE"]), ("nir", alignment["RGBgray_vs_NIR"]), ("renir", alignment["RE_vs_NIR"])):
|
|
row[f"{key}_phase_dx"] = al["phase_dx"]
|
|
row[f"{key}_phase_dy"] = al["phase_dy"]
|
|
row[f"{key}_phase_mag"] = al["phase_mag"]
|
|
row[f"{key}_phase_response"] = al["phase_response"]
|
|
|
|
for fname in ALL_FEATURES:
|
|
st = sample_feature_stats[fname]
|
|
for k in ("mean", "std", "min", "p01", "p05", "p50", "p95", "p99", "max", "p95_p05", "iqr"):
|
|
row[f"{fname}_{k}"] = st.get(k, 0.0)
|
|
if fname in CHANNELS:
|
|
for k in ("dark_pct", "sat_pct", "over_1_pct", "under_0_pct"):
|
|
row[f"{fname}_{k}"] = st.get(k, 0.0)
|
|
|
|
for cls_name, count in class_pixel_counts.items():
|
|
row[f"pixels_{cls_name}"] = count
|
|
row[f"pct_{cls_name}"] = 100.0 * count / max(1, h * w)
|
|
|
|
health = evaluate_sample_health(row, class_map, args)
|
|
row["health_score"] = health.score
|
|
row["health_status"] = health.status
|
|
row["approved_for_training"] = bool(health.approved)
|
|
row["health_warnings"] = " ; ".join(health.warnings)
|
|
row["health_errors"] = " ; ".join(health.errors)
|
|
row["health_notes"] = " ; ".join(health.notes)
|
|
row["warnings_count"] = len(health.warnings)
|
|
row["errors_count"] = len(health.errors)
|
|
|
|
if health.warnings or health.errors:
|
|
warnings.append({
|
|
"sample": stem,
|
|
"type": "sample_health",
|
|
"status": health.status,
|
|
"score": health.score,
|
|
"warnings": health.warnings,
|
|
"errors": health.errors,
|
|
})
|
|
|
|
sample_stats_for_visual = {"alignment": alignment, "features": sample_feature_stats}
|
|
should_save_visual = args.save_visuals and (
|
|
args.visual_every <= 1 or idx % args.visual_every == 0 or health.status != "good"
|
|
)
|
|
if should_save_visual or args.manual_review:
|
|
canvas = make_sample_visual(stem, tensor, mask, class_map, sample_stats_for_visual, health=health)
|
|
if should_save_visual and canvas is not None:
|
|
cv2.imwrite(str(visuals_dir / f"{idx:05d}_{stem}.png"), canvas)
|
|
|
|
if args.manual_review and canvas is not None:
|
|
decision, reason = manual_review_decision(canvas, row, idx, len(entries), args)
|
|
if decision == "quit":
|
|
manual_stop = True
|
|
row["manual_decision"] = "skipped"
|
|
row["manual_reason"] = reason
|
|
else:
|
|
row["manual_decision"] = decision
|
|
row["manual_reason"] = reason
|
|
if decision == "approved":
|
|
row["approved_for_training"] = True
|
|
elif decision == "rejected":
|
|
row["approved_for_training"] = False
|
|
row["health_status"] = "bad"
|
|
row["reject_reasons"] = reason
|
|
print(f"[MANUAL] {idx}/{len(entries)} | {stem} | decisão={row.get('manual_decision')} | motivo={row.get('manual_reason')}")
|
|
|
|
if args.save_rejected_previews and not bool(row.get("approved_for_training")):
|
|
fixed_root_tmp = resolve_fixed_out_root(dataset_roots, args)
|
|
row.setdefault("reject_reasons", row.get("health_errors") or row.get("health_warnings") or "rejected")
|
|
save_rejected_preview(row, fixed_root_tmp, args, tensor=tensor, mask=mask, class_map=class_map, audit_panel=canvas)
|
|
|
|
sample_rows.append(row)
|
|
|
|
core_row = {
|
|
"idx": idx,
|
|
"sample": stem,
|
|
"group": group_name,
|
|
"source_kind": load_result.source_kind,
|
|
**core_metrics,
|
|
}
|
|
core_rows.append(core_row)
|
|
|
|
if idx % args.print_every == 0 or idx == len(entries):
|
|
print(f"[OK] {idx}/{len(entries)} | {stem} | health={row['health_status']} score={row['health_score']:.1f}")
|
|
if manual_stop:
|
|
break
|
|
|
|
except Exception as e:
|
|
msg = str(e)
|
|
warnings.append({"sample": stem, "type": "exception", "message": msg})
|
|
print(f"[ERRO] {idx}/{len(entries)} | {stem}: {msg}")
|
|
if args.stop_on_error:
|
|
raise
|
|
|
|
if args.manual_review:
|
|
cv2.destroyAllWindows()
|
|
|
|
global_summary = global_acc.summarize()
|
|
class_summary = {
|
|
str(cls_id): {"class_name": class_map[cls_id], "features": acc.summarize()}
|
|
for cls_id, acc in class_acc.items()
|
|
}
|
|
by_group_rows = summarize_by_group(sample_rows)
|
|
global_health = build_global_health(sample_rows, global_summary, class_summary, warnings, args)
|
|
|
|
summary = {
|
|
"schema": "multispec_dataset_audit_bulletproof_v1",
|
|
"created_at_unix": time.time(),
|
|
"elapsed_s": time.time() - t_start,
|
|
"input_path": str(args.input_path),
|
|
"out_dir": str(out_dir),
|
|
"dataset_roots": [str(r) for r in dataset_roots],
|
|
"samples_processed": len(sample_rows),
|
|
"channels": CHANNELS,
|
|
"derived_features": DERIVED,
|
|
"class_map": {str(k): v for k, v in class_map.items()},
|
|
"thresholds": vars(args),
|
|
"global_health": global_health,
|
|
"global_summary": global_summary,
|
|
"class_summary": class_summary,
|
|
}
|
|
|
|
write_json(out_dir / "audit_summary.json", summary)
|
|
write_json(out_dir / "audit_health.json", global_health)
|
|
write_json(out_dir / "audit_warnings.json", warnings)
|
|
write_csv(out_dir / "audit_samples.csv", sample_rows)
|
|
write_csv(out_dir / "audit_by_group.csv", by_group_rows)
|
|
write_csv(out_dir / "audit_by_sample_class_feature.csv", by_sample_class_feature_rows)
|
|
write_csv(out_dir / "audit_core_telemetry.csv", core_rows)
|
|
|
|
class_feature_rows: List[Dict[str, Any]] = []
|
|
for cls_id, item in class_summary.items():
|
|
for feature, st in item["features"].items():
|
|
class_feature_rows.append({"class_id": cls_id, "class_name": item["class_name"], "feature": feature, **st})
|
|
write_csv(out_dir / "audit_by_class_channel.csv", class_feature_rows)
|
|
|
|
readme = f"""Auditoria Multiespectral - Bulletproof
|
|
|
|
Veredito: {global_health.get('status')} | score={safe_float(global_health.get('score')):.1f}
|
|
{global_health.get('verdict')}
|
|
|
|
Arquivos principais:
|
|
- audit_summary.json: resumo completo, thresholds e estatísticas globais.
|
|
- audit_health.json: veredito direto do dataset.
|
|
- audit_samples.csv: saúde e métricas por amostra.
|
|
- audit_by_group.csv: resumo por grupo.
|
|
- audit_by_class_channel.csv: estatísticas por classe e feature.
|
|
- audit_by_sample_class_feature.csv: estatísticas por amostra/classe/feature.
|
|
- audit_core_telemetry.csv: telemetria extraída do RawProcessorCore/meta normalizado.
|
|
- audit_warnings.json: avisos e exceções detalhados.
|
|
- visuals/: painéis visuais de amostras selecionadas ou com problemas.
|
|
|
|
Observação:
|
|
A métrica de phase correlation entre RGB/RE/NIR é tratada como alerta, não verdade absoluta.
|
|
Canais espectrais podem ter textura diferente do RGB mesmo quando a homografia está boa.
|
|
"""
|
|
(out_dir / "audit_manifest_readme.txt").write_text(readme, encoding="utf-8")
|
|
|
|
if args.build_fixed_dataset:
|
|
fixed_root = resolve_fixed_out_root(dataset_roots, args)
|
|
ensure_dir(fixed_root)
|
|
approved_rows = [r for r in sample_rows if bool(r.get("approved_for_training"))]
|
|
rejected_rows = [r for r in sample_rows if not bool(r.get("approved_for_training"))]
|
|
for r in approved_rows:
|
|
copy_sample_to_fixed(r, fixed_root, copy_previews=args.clean_copy_previews)
|
|
write_csv(fixed_root / "fixed_manifest.csv", sample_rows)
|
|
write_csv(fixed_root / "fixed_approved.csv", approved_rows)
|
|
write_csv(fixed_root / "fixed_rejected.csv", rejected_rows)
|
|
write_json(fixed_root / "fixed_summary.json", {
|
|
"schema": "multispec_fixed_dataset_bulletproof_v1",
|
|
"fixed_root": str(fixed_root),
|
|
"samples_total": len(sample_rows),
|
|
"samples_approved": len(approved_rows),
|
|
"samples_rejected": len(rejected_rows),
|
|
"approval_pct": 100.0 * len(approved_rows) / max(1, len(sample_rows)),
|
|
"global_health": global_health,
|
|
})
|
|
print(f"[FIXED] fixed/group: {fixed_root} | aprovadas={len(approved_rows)} rejeitadas={len(rejected_rows)}")
|
|
|
|
print("\n========== AUDITORIA FINALIZADA ==========")
|
|
print(f"Amostras processadas : {len(sample_rows)}")
|
|
print(f"Warnings/exceções : {len(warnings)}")
|
|
print(f"Status dataset : {global_health.get('status')} | score={safe_float(global_health.get('score')):.1f}")
|
|
print(f"Veredito : {global_health.get('verdict')}")
|
|
print(f"Resumo : {out_dir / 'audit_summary.json'}")
|
|
print(f"CSV amostras : {out_dir / 'audit_samples.csv'}")
|
|
print(f"CSV grupos : {out_dir / 'audit_by_group.csv'}")
|
|
print(f"Visuais : {visuals_dir}")
|
|
print("==========================================\n")
|
|
|
|
|
|
# ============================================================
|
|
# CLI
|
|
# ============================================================
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Auditoria parruda do tensor multiespectral [R,G,B,RE,NIR].")
|
|
parser.add_argument("--input_path", required=True)
|
|
parser.add_argument("--out_dir", default="audit_multispec_out")
|
|
parser.add_argument("--class-map", default="0:chao,1:cana,2:erva")
|
|
parser.add_argument("--ignore-index", type=int, default=255)
|
|
parser.add_argument("--limit", type=int, default=0)
|
|
parser.add_argument("--seed", type=int, default=42)
|
|
parser.add_argument("--stop-on-error", action="store_true")
|
|
parser.add_argument("--print-every", type=int, default=10)
|
|
parser.add_argument("--groups-except", default="")
|
|
|
|
# Visual/manual
|
|
parser.add_argument("--save-visuals", action="store_true")
|
|
parser.add_argument("--visual-every", type=int, default=10)
|
|
parser.add_argument("--manual-review", action="store_true")
|
|
parser.add_argument("--manual-window-width", type=int, default=1500)
|
|
parser.add_argument("--manual-window-name", default="Audit Multispec - Revisao Manual")
|
|
parser.add_argument("--manual-start-index", type=int, default=1)
|
|
|
|
# Stats/performance
|
|
parser.add_argument("--min-class-pixels", type=int, default=50)
|
|
parser.add_argument("--max-pixels-per-feature", type=int, default=25000)
|
|
|
|
# Thresholds amostra
|
|
parser.add_argument("--warn-sat-pct", type=float, default=1.0)
|
|
parser.add_argument("--bad-sat-pct", type=float, default=5.0)
|
|
parser.add_argument("--warn-dark-pct", type=float, default=35.0)
|
|
parser.add_argument("--bad-dark-pct", type=float, default=75.0)
|
|
parser.add_argument("--warn-low-dynamic", type=float, default=0.03)
|
|
parser.add_argument("--bad-low-dynamic", type=float, default=0.015)
|
|
parser.add_argument("--bad-mean-low", type=float, default=0.01)
|
|
parser.add_argument("--bad-mean-high", type=float, default=0.99)
|
|
parser.add_argument("--warn-shift-px", type=float, default=3.0)
|
|
parser.add_argument("--bad-shift-px", type=float, default=22.0)
|
|
parser.add_argument("--warn-edge-corr", type=float, default=0.08)
|
|
parser.add_argument("--bad-edge-corr", type=float, default=0.03)
|
|
parser.add_argument("--min-phase-response-for-shift-gate", type=float, default=0.05)
|
|
parser.add_argument("--min-valid-mask-pct", type=float, default=95.0)
|
|
parser.add_argument("--bad-patch-clip-pct", type=float, default=5.0)
|
|
parser.add_argument("--health-warning-score", type=float, default=85.0)
|
|
parser.add_argument("--reject-warnings", action="store_true")
|
|
|
|
# Radiometria
|
|
parser.add_argument("--require-radnorm-applied", action="store_true")
|
|
parser.add_argument("--radnorm-scale-min-ok", type=float, default=0.15)
|
|
parser.add_argument("--radnorm-scale-max-ok", type=float, default=3.0)
|
|
|
|
# Dataset global
|
|
parser.add_argument("--dataset-min-approval-pct", type=float, default=85.0)
|
|
parser.add_argument("--dataset-max-bad-pct", type=float, default=10.0)
|
|
parser.add_argument("--dataset-good-score", type=float, default=88.0)
|
|
parser.add_argument("--min-top-feature-effect-size", type=float, default=0.25)
|
|
|
|
# Fixed dataset
|
|
parser.add_argument("--build-fixed-dataset", action="store_true")
|
|
parser.add_argument("--fixed-out-root", default="")
|
|
parser.add_argument("--clean-copy-previews", action="store_true")
|
|
parser.add_argument("--clean-min-target-pct", type=float, default=0.0025)
|
|
parser.add_argument("--save-rejected-previews", action="store_true")
|
|
parser.add_argument("--rejected-preview-source", default="auto", choices=["auto", "preview", "audit_panel", "rgb_tensor"])
|
|
parser.add_argument("--rejected-preview-max-width", type=int, default=900)
|
|
|
|
args = parser.parse_args()
|
|
audit_dataset(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|