diff --git a/AgroBase/AgroBase/Services/GPSService.cs b/AgroBase/AgroBase/Services/GPSService.cs index 17415f667..26cb7ac4a 100644 --- a/AgroBase/AgroBase/Services/GPSService.cs +++ b/AgroBase/AgroBase/Services/GPSService.cs @@ -1857,7 +1857,7 @@ namespace AgroBase.Services // "AGRO_NTRIP_USERNAME" //) ?? string.Empty; - string password = "c3pc7*9N"; + string password = "kY3zd$*5"; //Environment.GetEnvironmentVariable( // "AGRO_NTRIP_PASSWORD" //) ?? string.Empty; diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/camera_multispectral.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/camera_multispectral.py index a91ddb836..69ab8b3e9 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/camera_multispectral.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/camera_multispectral.py @@ -16,7 +16,7 @@ from camera_worker.oak_fcc3_core.oak_fcc3_client import OakFcc3Client from camera_worker.camera_imu import IMUCamera -CAMERA_MULTISPECTRAL_VERSION = "production_v1_2026_08_24" +CAMERA_MULTISPECTRAL_VERSION = "production_v1_2026_09_12_raw_save_light" class CameraMultispectral: @@ -116,6 +116,7 @@ class CameraMultispectral: self.ultimo_tensor_multispec = None self.ultimo_frame_rgb = None self.ultimo_raw_multi = None + self.ultimo_raw_meta = None self.ultimo_meta = None self.ultimo_decoded = None @@ -584,6 +585,7 @@ class CameraMultispectral: self.ultimo_tensor_multispec = None self.ultimo_frame_rgb = None self.ultimo_raw_multi = None + self.ultimo_raw_meta = None self.ultimo_meta = None self.ultimo_decoded = None @@ -889,6 +891,9 @@ class CameraMultispectral: # Arrays do pacote são imutáveis após a captura. # Mantemos referências e copiamos somente no salvamento. self.ultimo_raw_multi = dict(frame) + # Meta científico pertence EXATAMENTE ao mesmo pacote RAW. + # Mantido no mesmo lock para impedir RAW N + meta N+1. + self.ultimo_raw_meta = dict(meta) self.timestamp_ultimo_raw_multi = ts_raw_perf self._ultimo_resultado_raw = { "erro": None, @@ -1008,7 +1013,7 @@ class CameraMultispectral: } def requisitar_frame_raw_multi(self, force: bool = False, max_age_s: float = None): - """Retorna cópia do último RAW capturado pelo fluxo operacional.""" + """Retorna snapshot por referência do último RAW operacional imutável.""" try: if max_age_s is None or float(max_age_s) <= 0: max_age_s = max(1.0, self._cache_max_age_s) @@ -1023,10 +1028,10 @@ class CameraMultispectral: if idade > float(max_age_s): raise RuntimeError(f"cache RAW antigo: {idade:.2f}s") - raw_frame = { - cam_id: arr.copy() - for cam_id, arr in self.ultimo_raw_multi.items() - } + # Snapshot barato: arrays do pacote operacional são imutáveis + # após a captura. Copiamos somente o dicionário de referências. + # A thread de salvamento mantém os ndarrays vivos enquanto escreve. + raw_frame = dict(self.ultimo_raw_multi) resultado = dict(self._ultimo_resultado_raw) resultado["cache_age_s"] = float(idade) resultado["force_ignorado"] = bool(force) @@ -1785,7 +1790,12 @@ class CameraMultispectral: def _ts_name(self) -> str: return datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3] - def requisitar_bundle_raw_multispec(self, force: bool = True, max_age_s: float = None): + def requisitar_bundle_raw_multispec( + self, + force: bool = True, + max_age_s: float = None, + include_preview: bool = False, + ): """ Monta o bundle a partir do cache do fluxo operacional. @@ -1796,16 +1806,30 @@ class CameraMultispectral: if max_age_s is None or float(max_age_s) <= 0: max_age_s = max(1.0, self._cache_max_age_s) - raw_frame, resultado = self.requisitar_frame_raw_multi( - force=False, - max_age_s=max_age_s, - ) - - if raw_frame is None: - raise RuntimeError(resultado.get("erro") or "RAW cache indisponível") - + agora = time.perf_counter() with self._lock: - raw_meta = dict(self.ultimo_meta or {}) + if self.ultimo_raw_multi is None or self.timestamp_ultimo_raw_multi is None: + raise RuntimeError("cache RAW ainda não disponível") + + idade = agora - self.timestamp_ultimo_raw_multi + if idade > float(max_age_s): + raise RuntimeError(f"cache RAW antigo: {idade:.2f}s") + + # Snapshot científico ATÔMICO. Os ndarrays continuam por + # referência, sem copiar megabytes dentro do lock. + raw_frame = dict(self.ultimo_raw_multi) + raw_meta = dict(self.ultimo_raw_meta or {}) + resultado = dict(self._ultimo_resultado_raw) + resultado["cache_age_s"] = float(idade) + resultado["force_ignorado"] = bool(force) + + frame_id_raw = int(resultado.get("frame_id", 0) or 0) + frame_id_meta = int(raw_meta.get("frame_id", 0) or 0) + if frame_id_raw and frame_id_meta and frame_id_raw != frame_id_meta: + raise RuntimeError( + "snapshot RAW/meta inconsistente: " + f"raw_frame_id={frame_id_raw} meta_frame_id={frame_id_meta}" + ) # Reforça o contrato científico no próprio stream_meta salvo. raw_meta.setdefault("frame_type", "RAW_BRUTO") @@ -1830,10 +1854,13 @@ class CameraMultispectral: f"Presentes: {sorted(presentes)}" ) - preview_bgr, preview_method = self._build_preview_raw_multispec( - raw_frame=raw_frame, - raw_meta=raw_meta, - ) + preview_bgr = None + preview_method = "disabled_runtime_save" + if include_preview: + preview_bgr, preview_method = self._build_preview_raw_multispec( + raw_frame=raw_frame, + raw_meta=raw_meta, + ) resultado = dict(resultado) resultado.update({ @@ -1920,8 +1947,7 @@ class CameraMultispectral: """ Salva pacote RAW_BRUTO multiespectral no mesmo espírito do capture de dataset. - Saída: - .png + Saída operacional: .json _CAM_A.bin _CAM_B.bin @@ -1939,6 +1965,7 @@ class CameraMultispectral: bundle, resultado = self.requisitar_bundle_raw_multispec( force=True, max_age_s=0.0, + include_preview=False, ) if not resultado.get("frame_valido", False): @@ -1977,17 +2004,10 @@ class CameraMultispectral: if not payload_files: raise RuntimeError("Nenhum payload RAW foi salvo.") + # Em operação, PosProcessamento salva APENAS os RAW .bin + JSON. + # Preview bonito é reconstruído offline a partir do próprio bundle. caminho_preview = None - if preview_bgr is not None and hasattr(preview_bgr, "size") and preview_bgr.size > 0: - caminho_preview = os.path.join( - pasta, - f"{nome_base}.png" - ) - - cv2.imwrite(caminho_preview, preview_bgr) - caminhos.append(caminho_preview) - meta_save = { "ts": datetime.now().isoformat(timespec="milliseconds"), "source": "operacao_robo", diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_client.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_client.py index d3c50e6ea..16ad1aee7 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_client.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_client.py @@ -45,6 +45,7 @@ class OakFcc3Client: imu_modo="rotation_vector", imu_freq_hz=200, evaluate_quality=True, + require_product_contract=False, hardware_sync_enabled=None, frame_sync_master=None, @@ -65,6 +66,30 @@ class OakFcc3Client: self.module_calibration_json = module_calibration_json self.module_params = self._load_module_params(module_calibration_json) self.fusion_config = self.module_params.get("fusion_config", {}) or {} + self.require_product_contract = bool(require_product_contract) + + assembly = self.module_params.get("assembly_metadata", {}) or {} + self.product_contract = bool( + self.module_params.get("schema") == "multispec_module_params_v3" + and assembly.get("schema") == "multispec_module_params_assembly_v1" + ) + + if self.require_product_contract and not self.product_contract: + raise RuntimeError( + "OakFcc3Client exige module_params de produção homologado: " + f"schema={self.module_params.get('schema')!r} " + f"assembly={assembly.get('schema')!r}" + ) + + # Em produto, Bayer e raster RGB nativo vêm do MP, não de fallbacks do caller. + mp_bayer = str(self.module_params.get("bayer_pattern", "") or "").upper() + if mp_bayer: + self.bayer = mp_bayer + + rgb_size = (self.module_params.get("sensor_size_by_role", {}) or {}).get("rgb") + if isinstance(rgb_size, (list, tuple)) and len(rgb_size) == 2: + self.width = int(rgb_size[0]) + self.height = int(rgb_size[1]) self.imu_modo = str(imu_modo).strip().lower() self.imu_freq_hz = int(imu_freq_hz) @@ -84,14 +109,15 @@ class OakFcc3Client: self.svc = OakFcc3Service( timeout=10, fps=fps, - width=width, - height=height, + width=self.width, + height=self.height, frame_type=frame_type, output_dtype=output_dtype, capture_mode=capture_mode, raw_policy=raw_policy, mx_id=self.mx_id, module_calibration_json=module_calibration_json, + require_product_contract=self.require_product_contract, imu_modo=self.imu_modo, imu_freq_hz=self.imu_freq_hz, @@ -108,15 +134,15 @@ class OakFcc3Client: self.applied_camera_controls = {} self.radiometric_controller = None self.core = RawProcessorCore( - sensor_width=width, - sensor_height=height, - bayer_pattern=bayer, + sensor_width=self.width, + sensor_height=self.height, + bayer_pattern=self.bayer, calibration_json_path=module_calibration_json, ) self.preview = RawProcessorPreview( - sensor_width=width, - sensor_height=height, - bayer_pattern=bayer, + sensor_width=self.width, + sensor_height=self.height, + bayer_pattern=self.bayer, ) def __enter__(self): @@ -133,6 +159,46 @@ class OakFcc3Client: with open(path, "r", encoding="utf-8") as f: return json.load(f) + def get_contract(self): + """ + Contrato estático/runtime consumido por CameraMultispectral. + + O module_params é autoridade de hardware/calibração. O target final + pode ser null no contrato novo; quando existir é apenas um default + compatível/legado. O Weed CameraManager passa o target ONNX ao caller. + """ + try: + service_contract = self.svc.get_contract() or {} + except Exception: + service_contract = {} + + mp = self.module_params or {} + fusion = mp.get("fusion_config", {}) or {} + target = fusion.get("target_size") + default_target = None + if isinstance(target, (list, tuple)) and len(target) == 2: + tw, th = int(target[0]), int(target[1]) + if tw > 0 and th > 0: + default_target = [tw, th] + + sensor_sizes = mp.get("sensor_size_by_role", {}) or {} + camera_hw = mp.get("camera_hardware", {}) or {} + + out = dict(service_contract) + out.update({ + "product_contract": bool(self.product_contract), + "require_product_contract": bool(self.require_product_contract), + "module_params_schema": mp.get("schema"), + "module_calibration_json": self.module_calibration_json, + "sensor_size_by_role": sensor_sizes, + "camera_hardware": camera_hw, + "bayer_pattern": mp.get("bayer_pattern") or out.get("bayer_pattern"), + "frame_type": self.frame_type, + "raw_policy": self.raw_policy, + "default_target_size": default_target, + }) + return out + def apply_module_camera_settings(self): camera_settings = self.module_params.get("camera_settings", {}) or {} @@ -374,8 +440,8 @@ class OakFcc3Client: evaluate_quality = self.evaluate_quality evaluate_quality = bool(evaluate_quality) - tensor = self.core.fuse_multispec_cameras(decoded, meta, channels_expected) - tensor = self.core.resize_tensor_chw(tensor, target_size=target_size) + tensor = self.core.fuse_multispec_cameras(decoded, meta, channels_expected, target_size=target_size) + #tensor = self.core.resize_tensor_chw(tensor, target_size=target_size) # Mantém paridade com build_infer_tensor_from_stream: se a calibração # habilitar patch normalization, ela também vale no caminho decoded. diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_manager.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_manager.py index 3588af9bc..c827ccdc8 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_manager.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/oak_fcc3_manager.py @@ -53,14 +53,15 @@ class OakFcc3Manager: mx_id=None, module_calibration_json=None, module_params=None, + require_product_contract=False, imu_modo="rotation_vector", imu_freq_hz=200, ): self.fps = fps # Para compatibilidade, mantemos width/height. - # No RAW_BRUTO isso não muda o sensor, pois usamos 800p fixo. - # No MULTISPEC isso representa a saída final alinhada da OAK. + # No RAW_BRUTO a resolução física é definida pelo module_params/hardware. + # No MULTISPEC width/height representa a saída final alinhada da OAK. self.width = int(width) self.height = int(height) self.size = (self.width, self.height) @@ -120,6 +121,52 @@ class OakFcc3Manager: self.fusion_config = (self.module_params or {}).get("fusion_config", {}) or {} self.aligned_geometry = None + # Contrato produto: MP descreve hardware/calibração. target_size pode ser null. + self.require_product_contract = bool(require_product_contract) + self.module_params_schema = (self.module_params or {}).get("schema") + assembly = (self.module_params or {}).get("assembly_metadata", {}) or {} + self.product_contract = bool( + self.module_params_schema == "multispec_module_params_v3" + and assembly.get("schema") == "multispec_module_params_assembly_v1" + ) + self.sensor_size_by_role = copy.deepcopy( + (self.module_params or {}).get("sensor_size_by_role", {}) or {} + ) + self.camera_hardware_expected = copy.deepcopy( + (self.module_params or {}).get("camera_hardware", {}) or {} + ) + self.bayer_pattern = str( + (self.module_params or {}).get("bayer_pattern", "") or "" + ).upper() or None + + if self.require_product_contract and not self.product_contract: + raise RuntimeError( + "OakFcc3Manager exige module_params de produção homologado: " + f"schema={self.module_params_schema!r} assembly={assembly.get('schema')!r}" + ) + + if self.product_contract: + self._validate_static_product_contract() + + roles_from_mp = {} + for role in ("rgb", "re", "nir"): + hw = self.camera_hardware_expected.get(role, {}) or {} + socket_name = str(hw.get("socket", "") or "").upper() + if socket_name: + roles_from_mp[socket_name] = role + if roles_from_mp: + self.roles = roles_from_mp + + rgb_size = self.sensor_size_by_role.get("rgb") + if isinstance(rgb_size, (list, tuple)) and len(rgb_size) == 2: + self.sensor_width = int(rgb_size[0]) + self.sensor_height = int(rgb_size[1]) + + self.camera_controls = { + cam_id: self._default_controls_for_role(role) + for cam_id, role in self.roles.items() + } + sync_cfg = (self.module_params.get("capture_synchronization", {}) if isinstance(self.module_params, dict) else {}) if hardware_sync_enabled is None: hardware_sync_enabled = sync_cfg.get("hardware_sync_enabled", False) if frame_sync_master is None: frame_sync_master = sync_cfg.get("frame_sync_master", "CAM_A") @@ -135,6 +182,17 @@ class OakFcc3Manager: self.async_capture_enabled = True self.async_capture_mode = "latest" # latest | queue self.async_capture_max_queue = 2 + + # Default de produção: materialização antecipada no producer, + # pois apresentou menor latência ponta a ponta. + # Para testar materialização diferida: + # OAK_FCC3_DEFER_RAW_MATERIALIZATION=1 + _defer_env = str( + os.environ.get("OAK_FCC3_DEFER_RAW_MATERIALIZATION", "0") + ).strip().lower() + self.defer_raw_materialization = _defer_env not in ( + "0", "false", "no", "off", "disabled" + ) self._capture_thread = None self._capture_stop_event = threading.Event() self._capture_lock = threading.RLock() @@ -183,6 +241,113 @@ class OakFcc3Manager: with open(path, "r", encoding="utf-8") as f: return json.load(f) + def _validate_static_product_contract(self): + sizes = self.sensor_size_by_role or {} + hardware = self.camera_hardware_expected or {} + + for role in ("rgb", "re", "nir"): + size = sizes.get(role) + hw = hardware.get(role) + if not (isinstance(size, (list, tuple)) and len(size) == 2): + raise RuntimeError(f"module_params sem sensor_size_by_role.{role}") + w, h = int(size[0]), int(size[1]) + if w <= 0 or h <= 0: + raise RuntimeError(f"sensor_size_by_role.{role} inválido: {size}") + if not isinstance(hw, dict): + raise RuntimeError(f"module_params sem camera_hardware.{role}") + socket_name = str(hw.get("socket", "") or "").upper() + sensor_name = str(hw.get("sensor", "") or "").upper() + if not socket_name or not sensor_name: + raise RuntimeError( + f"camera_hardware.{role} precisa conter socket e sensor: {hw}" + ) + hw_size = hw.get("size") + if hw_size is not None: + got = [int(hw_size[0]), int(hw_size[1])] + if got != [w, h]: + raise RuntimeError( + f"camera_hardware.{role}.size={got} diverge de sensor_size_by_role={size}" + ) + + if self.bayer_pattern not in ("RGGB", "BGGR", "GRBG", "GBRG"): + raise RuntimeError( + f"bayer_pattern inválido no module_params: {self.bayer_pattern!r}" + ) + + def _expected_native_size_for_role(self, role): + size = (self.sensor_size_by_role or {}).get(str(role).lower()) + if isinstance(size, (list, tuple)) and len(size) == 2: + return [int(size[0]), int(size[1])] + return None + + def _set_resolution_by_native_size(self, cam, role, is_color): + size = self._expected_native_size_for_role(role) + + if size is None: + if is_color: + names = ("THE_800_P", "THE_1080_P") + enum_cls = dai.ColorCameraProperties.SensorResolution + else: + names = ("THE_800_P", "THE_720_P", "THE_400_P") + enum_cls = dai.MonoCameraProperties.SensorResolution + for name in names: + value = getattr(enum_cls, name, None) + if value is not None: + try: + cam.setResolution(value) + return + except Exception: + continue + return + + key = tuple(size) + if is_color: + names = { + (1280, 800): "THE_800_P", + (1920, 1200): "THE_1200_P", + (1920, 1080): "THE_1080_P", + (1280, 720): "THE_720_P", + } + enum_cls = dai.ColorCameraProperties.SensorResolution + else: + names = { + (1280, 800): "THE_800_P", + (1280, 720): "THE_720_P", + (640, 400): "THE_400_P", + } + enum_cls = dai.MonoCameraProperties.SensorResolution + + enum_name = names.get(key) + enum_value = getattr(enum_cls, enum_name, None) if enum_name else None + if enum_value is None: + raise RuntimeError( + f"Resolução nativa {size} da role={role} não possui enum DepthAI suportado " + f"nesta versão do runtime (esperado={enum_name})." + ) + cam.setResolution(enum_value) + + def _validate_connected_hardware(self, features): + if not self.product_contract: + return True + + by_socket = {str(f.socket.name).upper(): f for f in features} + for role in ("rgb", "re", "nir"): + hw = self.camera_hardware_expected.get(role, {}) or {} + socket_name = str(hw.get("socket", "") or "").upper() + expected_sensor = str(hw.get("sensor", "") or "").upper() + feature = by_socket.get(socket_name) + if feature is None: + raise RuntimeError( + f"Hardware calibrado ausente: role={role} socket={socket_name}" + ) + actual_sensor = str(getattr(feature, "sensorName", "") or "").upper() + if expected_sensor and actual_sensor and actual_sensor != expected_sensor: + raise RuntimeError( + f"Sensor divergente em {socket_name}/{role}: " + f"detectado={actual_sensor} homologado={expected_sensor}" + ) + return True + def _default_controls_for_role(self, role: str): role = str(role).lower() @@ -293,11 +458,11 @@ class OakFcc3Manager: """ Fluxo clássico. - RGB/OV9782: - ColorCamera raw para RAW_BRUTO. + RGB (OV9782/AR0234): + ColorCamera raw na resolução nativa homologada pelo module_params. MONO/OV9282: - MonoCamera raw quando disponível. + MonoCamera raw na resolução nativa homologada pelo module_params. """ sensor_name_u = str(sensor_name or "").upper() role_u = str(role or "").lower() @@ -311,14 +476,7 @@ class OakFcc3Manager: if is_rgb: cam = self.pipeline.createColorCamera() cam.setBoardSocket(socket) - - try: - cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_800_P) - except Exception: - try: - cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P) - except Exception: - pass + self._set_resolution_by_native_size(cam, role_u, is_color=True) cam.setInterleaved(False) cam.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB) @@ -335,17 +493,7 @@ class OakFcc3Manager: mono = self.pipeline.create(dai.node.MonoCamera) mono.setBoardSocket(socket) - - try: - mono.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P) - except Exception: - try: - mono.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P) - except Exception: - try: - mono.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P) - except Exception: - pass + self._set_resolution_by_native_size(mono, role_u, is_color=False) mono.setFps(float(self.fps)) @@ -490,7 +638,7 @@ class OakFcc3Manager: def _create_color_camera_multispec(self, socket): cam = self.pipeline.create(dai.node.ColorCamera) cam.setBoardSocket(socket) - cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_800_P) + self._set_resolution_by_native_size(cam, "rgb", is_color=True) cam.setFps(float(self.fps)) cam.setInterleaved(False) @@ -500,10 +648,10 @@ class OakFcc3Manager: cam.setVideoSize(int(self.sensor_width), int(self.sensor_height)) return cam - def _create_mono_camera_multispec(self, socket): + def _create_mono_camera_multispec(self, socket, role): cam = self.pipeline.create(dai.node.MonoCamera) cam.setBoardSocket(socket) - cam.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P) + self._set_resolution_by_native_size(cam, role, is_color=False) cam.setFps(float(self.fps)) return cam @@ -625,7 +773,7 @@ class OakFcc3Manager: bit_depth = 8 raw_format = "BGR888p" elif role == "re": - cam = self._create_mono_camera_multispec(socket) + cam = self._create_mono_camera_multispec(socket, "re") src_output = cam.out quad = self.aligned_geometry["quad_re"] out_type = dai.ImgFrame.Type.GRAY8 @@ -634,7 +782,7 @@ class OakFcc3Manager: bit_depth = 8 raw_format = "GRAY8" elif role == "nir": - cam = self._create_mono_camera_multispec(socket) + cam = self._create_mono_camera_multispec(socket, "nir") src_output = cam.out quad = self.aligned_geometry["quad_nir"] out_type = dai.ImgFrame.Type.GRAY8 @@ -896,6 +1044,7 @@ class OakFcc3Manager: self.pipeline = dai.Pipeline() features = self.device.getConnectedCameraFeatures() + self._validate_connected_hardware(features) self.queues.clear() self.buffers.clear() @@ -943,11 +1092,14 @@ class OakFcc3Manager: self.buffers[cam_id] = deque(maxlen=self.buffer_size) self.control_queues[cam_id] = None + native_size = self._expected_native_size_for_role(role) self.camera_info[cam_id] = { "id": cam_id, "socket": socket_name, "sensor": f.sensorName, "role": role, + "native_size": native_size, + "bayer_pattern": self.bayer_pattern if str(role).lower() == "rgb" else None, } self._validate_capture_mode() @@ -1143,6 +1295,14 @@ class OakFcc3Manager: return { "mx_id": self.mx_id, "backend": "oak_fcc3", + "manager_version": "production_v2_2026_09_12", + "product_contract": bool(self.product_contract), + "require_product_contract": bool(self.require_product_contract), + "module_params_schema": self.module_params_schema, + "module_calibration_json": self.module_calibration_json, + "sensor_size_by_role": copy.deepcopy(self.sensor_size_by_role), + "camera_hardware_expected": copy.deepcopy(self.camera_hardware_expected), + "bayer_pattern": self.bayer_pattern, "running": bool(self.running), "fps": self.fps, "width": self.width, @@ -1201,9 +1361,10 @@ class OakFcc3Manager: t0 = time.perf_counter() deadline = t0 + float(timeout) + packet = None + with self._capture_cond: while time.perf_counter() < deadline: - packet = None mode = str(getattr(self, "async_capture_mode", "latest")).lower() @@ -1218,22 +1379,7 @@ class OakFcc3Manager: if packet is not None: seq = int(packet.get("seq", 0)) self._last_consumed_packet_seq = seq - - frames = packet["frames"] - meta = dict(packet["meta"]) - - age_ms = (time.perf_counter() - float(packet.get("created_perf_counter", time.perf_counter()))) * 1000.0 - get_wait_ms = (time.perf_counter() - t0) * 1000.0 - - cp = dict(meta.get("capture_perf", {}) or {}) - cp["async_consumer"] = True - cp["async_packet_seq"] = seq - cp["async_packet_age_ms"] = float(age_ms) - cp["async_get_wait_ms"] = float(get_wait_ms) - cp["async_status"] = self.get_async_capture_status() - meta["capture_perf"] = cp - - return frames, meta + break remaining = deadline - time.perf_counter() if remaining <= 0: @@ -1241,6 +1387,76 @@ class OakFcc3Manager: self._capture_cond.wait(timeout=min(0.005, remaining)) + if packet is not None: + # IMPORTANTE: materialização RAW acontece FORA do lock da captura. + # Assim a thread OAK pode continuar drenando/sincronizando enquanto + # o consumidor transforma somente a tripleta que realmente usará. + seq = int(packet.get("seq", 0)) + age_ms = ( + time.perf_counter() + - float(packet.get("created_perf_counter", time.perf_counter())) + ) * 1000.0 + get_wait_ms = (time.perf_counter() - t0) * 1000.0 + + if bool(packet.get("deferred_raw_packet", False)): + cp = dict(packet.get("capture_perf", {}) or {}) + t_mat0 = time.perf_counter() + + selected_items = packet.get("selected_items", {}) or {} + frames = {} + frame_controls = {} + + for cam_id, item in selected_items.items(): + frame, controls = self._materialize_selected_item( + cam_id=cam_id, + item=item, + perf=cp, + ) + if frame is None: + raise RuntimeError( + f"Frame RAW assíncrono selecionado inválido: {cam_id}" + ) + frames[cam_id] = frame + frame_controls[cam_id] = controls or {} + + consumer_materialize_ms = ( + time.perf_counter() - t_mat0 + ) * 1000.0 + + t_meta0 = time.perf_counter() + meta = self._build_meta( + frames, + packet.get("timestamps", {}) or {}, + float(packet.get("sync_dt_ms", 0.0) or 0.0), + bool(packet.get("sync_ok", False)), + frame_controls=frame_controls, + frame_id_override=packet.get("frame_id"), + ) + consumer_meta_ms = ( + time.perf_counter() - t_meta0 + ) * 1000.0 + + cp["async_consumer_materialize_ms"] = float( + consumer_materialize_ms + ) + cp["async_consumer_meta_ms"] = float(consumer_meta_ms) + cp["async_deferred_packet"] = True + + else: + frames = packet["frames"] + meta = dict(packet["meta"]) + cp = dict(meta.get("capture_perf", {}) or {}) + cp["async_deferred_packet"] = False + + cp["async_consumer"] = True + cp["async_packet_seq"] = seq + cp["async_packet_age_ms"] = float(age_ms) + cp["async_get_wait_ms"] = float(get_wait_ms) + cp["async_status"] = self.get_async_capture_status() + meta["capture_perf"] = cp + + return frames, meta + raise TimeoutError( f"Timeout aguardando pacote assíncrono do OAK-FFC-3. " f"status={self.get_async_capture_status()}" @@ -1456,47 +1672,36 @@ class OakFcc3Manager: } else: - t0_data = self._cap_now_ms() - data = msg.getData() - if perf is not None: - perf["drain_get_data_ms"] += self._cap_now_ms() - t0_data - - t0_copy = self._cap_now_ms() - raw = np.frombuffer(data, dtype=np.uint8).copy() - if perf is not None: - perf["drain_frombuffer_copy_ms"] += self._cap_now_ms() - t0_copy - - t0_shape = self._cap_now_ms() - h = int(msg.getHeight()) - w = int(msg.getWidth()) - stride = self._get_imgframe_stride(msg, raw.size, h, w) - - expected = h * stride - - if raw.size < expected: - raise RuntimeError( - f"RAW menor que esperado: raw.size={raw.size}, esperado={expected}, " - f"w={w}, h={h}, stride={stride}" + if bool(getattr(self, "defer_raw_materialization", True)): + # Não toca no payload RAW aqui. O ImgFrame permanece vivo + # no deque e só será materializado se entrar na tripleta + # escolhida pelo sincronizador. Frames descartados custam + # apenas metadados/timestamp. + frame = None + frame_controls = None + deferred_raw = True + if perf is not None: + perf["deferred_raw_msgs"] += 1 + else: + frame, frame_controls = self._materialize_raw_msg( + cam_id=cam_id, + msg=msg, + perf=perf, + metric_prefix="drain", ) + deferred_raw = False - frame = raw[:expected].reshape((h, stride)) + if self._is_preview_mode() or self._is_multispec_mode(): + t0_ctrl = self._cap_now_ms() + frame_controls = self._extract_frame_controls(msg) if perf is not None: - perf["drain_reshape_ms"] += self._cap_now_ms() - t0_shape - - self._last_raw_dims[cam_id] = { - "sensor_width": w, - "sensor_height": h, - "stride": stride, - "packed_width": stride, - } - - t0_ctrl = self._cap_now_ms() - frame_controls = self._extract_frame_controls(msg) - if perf is not None: - perf["drain_controls_ms"] += self._cap_now_ms() - t0_ctrl + perf["drain_controls_ms"] += self._cap_now_ms() - t0_ctrl + deferred_raw = False self.buffers[cam_id].append({ "frame": frame, + "msg": msg if deferred_raw else None, + "deferred_raw": bool(deferred_raw), "timestamp": ts_start, "timestamp_start": ts_start, "timestamp_end": ts_end, @@ -1507,6 +1712,80 @@ class OakFcc3Manager: perf["drained_total"] += 1 perf["drained_by_cam"][cam_id] += 1 + def _materialize_raw_msg(self, cam_id, msg, perf=None, metric_prefix="selected"): + """Materializa um RAW10 packed somente quando ele realmente será usado.""" + if msg is None: + raise RuntimeError(f"ImgFrame ausente para materialização RAW: {cam_id}") + + t_all = self._cap_now_ms() + + t0_data = self._cap_now_ms() + data = msg.getData() + data_ms = self._cap_now_ms() - t0_data + + t0_copy = self._cap_now_ms() + raw = np.frombuffer(data, dtype=np.uint8).copy() + copy_ms = self._cap_now_ms() - t0_copy + + t0_shape = self._cap_now_ms() + h = int(msg.getHeight()) + w = int(msg.getWidth()) + stride = self._get_imgframe_stride(msg, raw.size, h, w) + expected = h * stride + + if raw.size < expected: + raise RuntimeError( + f"RAW menor que esperado: raw.size={raw.size}, esperado={expected}, " + f"w={w}, h={h}, stride={stride}" + ) + + frame = raw[:expected].reshape((h, stride)) + shape_ms = self._cap_now_ms() - t0_shape + + self._last_raw_dims[cam_id] = { + "sensor_width": w, + "sensor_height": h, + "stride": stride, + "packed_width": stride, + } + + t0_ctrl = self._cap_now_ms() + frame_controls = self._extract_frame_controls(msg) + controls_ms = self._cap_now_ms() - t0_ctrl + + if perf is not None: + if metric_prefix == "drain": + perf["drain_get_data_ms"] += data_ms + perf["drain_frombuffer_copy_ms"] += copy_ms + perf["drain_reshape_ms"] += shape_ms + perf["drain_controls_ms"] += controls_ms + else: + perf["selected_materialize_ms"] += self._cap_now_ms() - t_all + perf["selected_get_data_ms"] += data_ms + perf["selected_frombuffer_copy_ms"] += copy_ms + perf["selected_reshape_ms"] += shape_ms + perf["selected_controls_ms"] += controls_ms + perf["selected_materialized_frames"] += 1 + perf["selected_materialized_bytes"] += int(expected) + + return frame, frame_controls + + def _materialize_selected_item(self, cam_id, item, perf=None): + if not bool(item.get("deferred_raw", False)): + return item.get("frame"), item.get("controls", {}) or {} + + frame, controls = self._materialize_raw_msg( + cam_id=cam_id, + msg=item.get("msg"), + perf=perf, + metric_prefix="selected", + ) + item["frame"] = frame + item["controls"] = controls + item["deferred_raw"] = False + item["msg"] = None + return frame, controls + def _get_imgframe_stride(self, msg, raw_size: int, h: int, w: int) -> int: try: return int(msg.getStride()) @@ -1521,7 +1800,7 @@ class OakFcc3Manager: return int(np.ceil(w * 5.0 / 4.0)) - def _try_get_synced_packet(self, perf=None): + def _try_get_synced_packet(self, perf=None, materialize=True): required_cam_ids = self._get_required_cam_ids() if not required_cam_ids: @@ -1549,11 +1828,6 @@ class OakFcc3Manager: for cam_id, item in selected.items() } - frame_controls = { - cam_id: item.get("controls", {}) - for cam_id, item in selected.items() - } - ts_values = list(timestamps.values()) sync_dt_ms = ( @@ -1570,11 +1844,6 @@ class OakFcc3Manager: for cam_id, ts in timestamps.items() } - perf["selected_seq_by_cam"] = { - cam_id: item.get("controls", {}).get("sequence_num") - for cam_id, item in selected.items() - } - perf["sync_dt_ms"] = float(sync_dt_ms) perf["sync_ok"] = bool(sync_ok) @@ -1601,10 +1870,41 @@ class OakFcc3Manager: # Em best/best_effort, entrega a melhor combinação disponível, # mesmo quando estiver fora da tolerância. - frames = { - cam_id: item["frame"] - for cam_id, item in selected.items() - } + # + # No caminho assíncrono/latest, materialize=False mantém apenas as + # referências ImgFrame escolhidas. Se esse pacote for substituído por + # outro antes do consumidor pegá-lo, nenhum payload RAW foi copiado. + # A materialização fica para get_next_frame(), uma única vez, no pacote + # que realmente alimentará o tensor. + if materialize: + payload = {} + frame_controls = {} + for cam_id, item in selected.items(): + frame, controls = self._materialize_selected_item( + cam_id=cam_id, + item=item, + perf=perf, + ) + if frame is None: + raise RuntimeError(f"Frame selecionado inválido: {cam_id}") + payload[cam_id] = frame + frame_controls[cam_id] = controls or {} + + if perf is not None: + perf["selected_seq_by_cam"] = { + cam_id: frame_controls.get(cam_id, {}).get("sequence_num") + for cam_id in selected.keys() + } + else: + # Shallow copy do descritor. O ImgFrame permanece vivo por sua + # referência Python, sem copiar os megabytes do RAW. + payload = { + cam_id: dict(item) + for cam_id, item in selected.items() + } + frame_controls = {} + if perf is not None: + perf["selected_deferred_for_consumer"] = len(payload) # Consome todos os frames anteriores e o próprio frame selecionado. for cam_id, used_item in selected.items(): @@ -1622,7 +1922,7 @@ class OakFcc3Manager: ) return ( - frames, + payload, timestamps, sync_dt_ms, sync_ok, @@ -1669,7 +1969,15 @@ class OakFcc3Manager: # Meta # ============================================================ - def _build_meta(self, frames, timestamps, sync_dt_ms, sync_ok, frame_controls): + def _build_meta( + self, + frames, + timestamps, + sync_dt_ms, + sync_ok, + frame_controls, + frame_id_override=None, + ): payload_sources = list(frames.keys()) shapes = { @@ -1730,7 +2038,11 @@ class OakFcc3Manager: camera_info[cam_id] = item meta = { - "frame_id": self.frame_id, + "frame_id": ( + int(self.frame_id) + if frame_id_override is None + else int(frame_id_override) + ), "backend": "oak_fcc3", "frame_type": self.frame_type, "capture_mode": self.capture_mode, @@ -1974,6 +2286,17 @@ class OakFcc3Manager: "drain_frombuffer_copy_ms": 0.0, "drain_reshape_ms": 0.0, "drain_controls_ms": 0.0, + "deferred_raw_msgs": 0, + "selected_materialize_ms": 0.0, + "selected_get_data_ms": 0.0, + "selected_frombuffer_copy_ms": 0.0, + "selected_reshape_ms": 0.0, + "selected_controls_ms": 0.0, + "selected_materialized_frames": 0, + "selected_materialized_bytes": 0, + "selected_deferred_for_consumer": 0, + "async_consumer_materialize_ms": 0.0, + "async_consumer_meta_ms": 0.0, "sync_select_ms": 0.0, "meta_ms": 0.0, "sleep_ms": 0.0, @@ -2093,32 +2416,64 @@ class OakFcc3Manager: # Importante: esta thread é a única que mexe nas queues/buffers. self._drain_queues_to_buffers(perf=perf) - synced = self._try_get_synced_packet(perf=perf) + + # Em RAW_BRUTO + async, o produtor publica apenas descritores + # ImgFrame da tripleta sincronizada. O consumidor materializa + # somente o pacote latest que realmente pegar. Isso elimina + # getData()/copy() de pacotes completos que seriam sobrescritos. + defer_packet_to_consumer = bool( + self._is_raw_mode() + and getattr(self, "defer_raw_materialization", True) + ) + + synced = self._try_get_synced_packet( + perf=perf, + materialize=not defer_packet_to_consumer, + ) if synced is None: # Dorme curto. Pode testar 0.0005 se quiser reduzir latência. time.sleep(0.001) continue - frames, timestamps, sync_dt_ms, sync_ok, frame_controls = synced + payload, timestamps, sync_dt_ms, sync_ok, frame_controls = synced self.frame_id += 1 - meta = self._build_meta( - frames, - timestamps, - sync_dt_ms, - sync_ok, - frame_controls=frame_controls, - ) + packet_frame_id = int(self.frame_id) + + meta = None + if not defer_packet_to_consumer: + meta = self._build_meta( + payload, + timestamps, + sync_dt_ms, + sync_ok, + frame_controls=frame_controls, + frame_id_override=packet_frame_id, + ) now = time.perf_counter() packet = { "seq": int(self._latest_packet_seq + 1), "created_perf_counter": float(now), - "frames": frames, - "meta": meta, + "frame_id": packet_frame_id, + "deferred_raw_packet": bool(defer_packet_to_consumer), } + if defer_packet_to_consumer: + packet.update({ + "selected_items": payload, + "timestamps": timestamps, + "sync_dt_ms": float(sync_dt_ms), + "sync_ok": bool(sync_ok), + "capture_perf": perf or {}, + }) + else: + packet.update({ + "frames": payload, + "meta": meta, + }) + # Adiciona perf de captura assíncrona no meta. if perf is not None: perf["async_thread"] = True @@ -2126,7 +2481,10 @@ class OakFcc3Manager: perf["sync_dt_ms"] = float(sync_dt_ms) perf["sync_ok"] = bool(sync_ok) perf["wait_reason"] = "async_packet_ready" - meta["capture_perf"] = perf + if meta is not None: + meta["capture_perf"] = perf + else: + packet["capture_perf"] = perf with self._capture_cond: self._latest_packet_seq += 1 @@ -2195,6 +2553,7 @@ class OakFcc3Manager: st.update({ "enabled": bool(getattr(self, "async_capture_enabled", True)), "mode": str(getattr(self, "async_capture_mode", "latest")), + "defer_raw_materialization": bool(getattr(self, "defer_raw_materialization", True)), "thread_alive": bool(self._capture_thread is not None and self._capture_thread.is_alive()), "latest_seq": int(getattr(self, "_latest_packet_seq", 0)), "last_consumed_seq": int(getattr(self, "_last_consumed_packet_seq", 0)), diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/raw_processor_core.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/raw_processor_core.py index 43d923868..3c7dd7144 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/raw_processor_core.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/camera_worker/oak_fcc3_core/raw_processor_core.py @@ -28,7 +28,7 @@ module_params is loaded. The strict fail-closed behavior is enabled only for module_params generated by the production assembler. """ -RAW_PROCESSOR_CORE_VERSION = "production_v2_2026_09_12_radiometric_singlepass" +RAW_PROCESSOR_CORE_VERSION = "production_v5_2026_09_13_flat_finalspace_fast" import json import os @@ -428,20 +428,177 @@ if _HAS_NUMBA: img[y, x, k] = v + @_numba.njit(cache=True, fastmath=True, inline="always") + def _direct_sample_mono_fixedmap_numba(img, x0, y0, frac_code): + """ + Bilinear compatível com os mapas CV_16SC2/CV_16UC1 produzidos por + cv2.convertMaps(..., CV_16SC2). + + OpenCV INTER_LINEAR usa uma tabela de 32 passos por eixo. map1 guarda + a parte inteira e map2 guarda os índices fracionários quantizados. + Usar o mesmo código evita recalcular coordenadas/homografias por pixel. + """ + tab = int(frac_code) + fx_i = tab & 31 + fy_i = (tab >> 5) & 31 + + wx1 = fx_i * (1.0 / 32.0) + wy1 = fy_i * (1.0 / 32.0) + wx0 = 1.0 - wx1 + wy0 = 1.0 - wy1 + + h = img.shape[0] + w = img.shape[1] + x1 = x0 + 1 + y1 = y0 + 1 + + # Fast path: os quatro vizinhos estão dentro da imagem. + if x0 >= 0 and y0 >= 0 and x1 < w and y1 < h: + v00 = float(img[y0, x0]) + v01 = float(img[y0, x1]) + v10 = float(img[y1, x0]) + v11 = float(img[y1, x1]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + # BORDER_CONSTANT=0.0, incluindo pixels parcialmente fora do sensor. + v00 = 0.0 + v01 = 0.0 + v10 = 0.0 + v11 = 0.0 + + if y0 >= 0 and y0 < h: + if x0 >= 0 and x0 < w: + v00 = float(img[y0, x0]) + if x1 >= 0 and x1 < w: + v01 = float(img[y0, x1]) + + if y1 >= 0 and y1 < h: + if x0 >= 0 and x0 < w: + v10 = float(img[y1, x0]) + if x1 >= 0 and x1 < w: + v11 = float(img[y1, x1]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + + @_numba.njit(cache=True, fastmath=True, inline="always") + def _direct_sample_rgb_fixedmap_numba(rgb, x0, y0, frac_code, channel): + tab = int(frac_code) + fx_i = tab & 31 + fy_i = (tab >> 5) & 31 + + wx1 = fx_i * (1.0 / 32.0) + wy1 = fy_i * (1.0 / 32.0) + wx0 = 1.0 - wx1 + wy0 = 1.0 - wy1 + + h = rgb.shape[0] + w = rgb.shape[1] + x1 = x0 + 1 + y1 = y0 + 1 + + if x0 >= 0 and y0 >= 0 and x1 < w and y1 < h: + v00 = float(rgb[y0, x0, channel]) + v01 = float(rgb[y0, x1, channel]) + v10 = float(rgb[y1, x0, channel]) + v11 = float(rgb[y1, x1, channel]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + v00 = 0.0 + v01 = 0.0 + v10 = 0.0 + v11 = 0.0 + + if y0 >= 0 and y0 < h: + if x0 >= 0 and x0 < w: + v00 = float(rgb[y0, x0, channel]) + if x1 >= 0 and x1 < w: + v01 = float(rgb[y0, x1, channel]) + + if y1 >= 0 and y1 < h: + if x0 >= 0 and x0 < w: + v10 = float(rgb[y1, x0, channel]) + if x1 >= 0 and x1 < w: + v11 = float(rgb[y1, x1, channel]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + + @_numba.njit(cache=True, fastmath=True, parallel=True) + def _direct_fused5_fixedmap_numba( + rgb, + re_img, + nir_img, + rgb_map1, + rgb_map2, + re_map1, + re_map2, + nir_map1, + nir_map2, + out, + ): + """ + RGB + RE + NIR -> tensor CHW [R,G,B,RE,NIR] em uma única passagem. + + - usa os mesmos mapas fixos já cacheados para cv2.remap; + - reproduz a quantização 1/32 do INTER_LINEAR de OpenCV; + - BORDER_CONSTANT = 0.0; + - escreve diretamente no tensor final, sem HWC temporário nem packing. + """ + height = out.shape[1] + width = out.shape[2] + + for y in _numba.prange(height): + for x in range(width): + # RGB compartilha um único par de mapas para os 3 canais. + rx = int(rgb_map1[y, x, 0]) + ry = int(rgb_map1[y, x, 1]) + rf = rgb_map2[y, x] + + out[0, y, x] = _direct_sample_rgb_fixedmap_numba(rgb, rx, ry, rf, 0) + out[1, y, x] = _direct_sample_rgb_fixedmap_numba(rgb, rx, ry, rf, 1) + out[2, y, x] = _direct_sample_rgb_fixedmap_numba(rgb, rx, ry, rf, 2) + + ex = int(re_map1[y, x, 0]) + ey = int(re_map1[y, x, 1]) + ef = re_map2[y, x] + out[3, y, x] = _direct_sample_mono_fixedmap_numba(re_img, ex, ey, ef) + + nx = int(nir_map1[y, x, 0]) + ny = int(nir_map1[y, x, 1]) + nf = nir_map2[y, x] + out[4, y, x] = _direct_sample_mono_fixedmap_numba(nir_img, nx, ny, nf) + + @_numba.njit(cache=True, fastmath=True, parallel=True) def _apply_tensor_flat_gain_chw_numba(tensor, gain, channels, height, width, clip_output): - for c in _numba.prange(channels): - for y in range(height): - for x in range(width): - v = float(tensor[c, y, x]) * float(gain[c, y, x]) + # Raw5 tem só 5 canais; paralelizar apenas em C limita o kernel a 5 + # tarefas. Paralelizando por (canal, linha), CPUs com mais threads + # conseguem trabalhar no tensor final inteiro sem mudar a matemática. + total_rows = channels * height + for cy in _numba.prange(total_rows): + c = cy // height + y = cy - c * height + for x in range(width): + v = float(tensor[c, y, x]) * float(gain[c, y, x]) - if clip_output: - if v < 0.0: - v = 0.0 - elif v > 1.0: - v = 1.0 + if clip_output: + if v < 0.0: + v = 0.0 + elif v > 1.0: + v = 1.0 - tensor[c, y, x] = v + tensor[c, y, x] = v @@ -715,7 +872,7 @@ class RawProcessorCore: # OAK_CORE_PERF_LOG_INTERVAL_S=1.0 # OAK_CORE_SHAPES_LOG_INTERVAL_S=5.0 self.core_perf_debug = str( - os.getenv("OAK_CORE_PERF_DEBUG", "1") + os.getenv("OAK_CORE_PERF_DEBUG", "0") ).strip().lower() not in ("0", "false", "no", "off") try: self.core_perf_log_interval_s = max(0.2, float( @@ -2049,8 +2206,15 @@ class RawProcessorCore: "warp_details_ms": warp_details, "crop_resize_ms": t_crop_resize_ms, "concat_ms": t_concat_ms, - "spatial_direct_ms": float(t_rgb_ms + t_warp_total_ms), + "spatial_direct_ms": float(direct_perf.get("spatial_direct_ms", t_rgb_ms + t_warp_total_ms)), + "direct_backend_requested": str(direct_perf.get("direct_backend_requested", "auto")), + "direct_backend": str(direct_perf.get("direct_backend", "opencv_fastpath")), + "fused5_used": bool(direct_perf.get("fused5_used", False)), + "fused5_ms": float(direct_perf.get("fused5_ms", 0.0)), + "fused5_fallback_reason": str(direct_perf.get("fused5_fallback_reason", "")), "final_flat_ms": final_flat_ms, + "final_flat_prepare_ms": float(direct_perf.get("final_flat_prepare_ms", 0.0)), + "final_flat_apply_ms": float(direct_perf.get("final_flat_apply_ms", 0.0)), "final_ms": t_final_ms, "total_ms": t_total_ms, "decode_perf": getattr(self, "last_decode_perf", {}), @@ -2105,13 +2269,21 @@ class RawProcessorCore: f"geom={t_geometry_ms:.2f} " f"alloc={t_tensor_alloc_ms:.2f} " f"remap_cache={float(direct_perf.get('remap_cache_ms', 0.0)):.2f} " + f"backend={str(direct_perf.get('direct_backend', 'opencv_fastpath'))} " + f"fused5={float(direct_perf.get('fused5_ms', 0.0)):.2f} " f"rgb={t_rgb_ms:.2f} " + f"rgb_remap={float(direct_perf.get('rgb_remap_ms', 0.0)):.2f} " + f"rgb_pack={float(direct_perf.get('rgb_pack_ms', 0.0)):.2f} " + f"rgb_backend={str(direct_perf.get('rgb_backend', '-'))} " f"re={float(warp_details.get('re', 0.0)):.2f} " f"nir={float(warp_details.get('nir', 0.0)):.2f} " - f"final_flat={final_flat_ms:.2f} | " + f"final_flat={final_flat_ms:.2f} " + f"flat_prep={float(direct_perf.get('final_flat_prepare_ms', 0.0)):.2f} " + f"flat_apply={float(direct_perf.get('final_flat_apply_ms', 0.0)):.2f} | " f"orient_fused_rgb={1 if direct_perf.get('rgb_orientation_fused_into_remap') else 0} " f"cache geom={'HIT' if direct_perf.get('geometry_cache_hit') else 'MISS'} " f"remap={'HIT' if direct_perf.get('remap_cache_hit') else 'MISS'} " + f"fallback={str(direct_perf.get('fused5_fallback_reason', '') or '-')} " f"shape={tuple(out.shape)}" ) @@ -2157,6 +2329,7 @@ class RawProcessorCore: f"target={tuple(direct_perf.get('target_size', ())) if direct_perf.get('target_size') else target_size} " f"crop={direct_perf.get('crop_box')} " f"remap_shapes={direct_perf.get('remap_map_shapes', {})} " + f"direct_backend={str(direct_perf.get('direct_backend', 'opencv_fastpath'))} " f"flat_space={flat_apply_space}" ) @@ -4088,10 +4261,11 @@ class RawProcessorCore: ) ).lower() - if flat_space != "native_camera_space": + if flat_space not in ("native_camera_space", "final_tensor_space"): raise RuntimeError( - "Produto exige Flat-Field no espaço nativo; " - f"apply_space={flat_space!r}" + "Flat-Field produto com apply_space inválido; " + f"apply_space={flat_space!r}. " + "Use 'native_camera_space' ou 'final_tensor_space'." ) if bool( @@ -7437,6 +7611,128 @@ class RawProcessorCore: tensor[int(channel_index)] = out + def _direct_fusion_can_use_numba_fused5_fast( + self, + decoded, + role_to_cam, + remap_cache, + channels_expected, + use_remap_cache, + use_remap_for_rgb, + use_remap_for_spec, + ): + """ + Valida o contrato mínimo para o backend Numba fused 5ch. + + O backend é intencionalmente conservador: se qualquer premissa não for + satisfeita, o caller volta automaticamente ao caminho OpenCV anterior. + """ + if not _HAS_NUMBA: + return False, "numba_unavailable" + + if int(channels_expected) != 5: + return False, f"channels_expected={channels_expected}" + + if not bool(use_remap_cache): + return False, "remap_cache_disabled" + + if not bool(use_remap_for_rgb): + return False, "rgb_remap_disabled" + + if not bool(use_remap_for_spec): + return False, "spec_remap_disabled" + + maps = (remap_cache or {}).get("maps", {}) or {} + for role in ("rgb", "re", "nir"): + if role not in role_to_cam: + return False, f"missing_role:{role}" + + entry = maps.get(role) + if not isinstance(entry, dict): + return False, f"missing_maps:{role}" + + map1 = entry.get("map1") + map2 = entry.get("map2") + if not isinstance(map1, np.ndarray) or not isinstance(map2, np.ndarray): + return False, f"invalid_maps:{role}" + if map1.ndim != 3 or map1.shape[2] != 2: + return False, f"invalid_map1_shape:{role}:{getattr(map1, 'shape', None)}" + if map2.ndim != 2 or map2.shape[:2] != map1.shape[:2]: + return False, f"invalid_map2_shape:{role}:{getattr(map2, 'shape', None)}" + + rgb = decoded[role_to_cam["rgb"]].get("image") + re_img = decoded[role_to_cam["re"]].get("image") + nir_img = decoded[role_to_cam["nir"]].get("image") + + if not isinstance(rgb, np.ndarray) or rgb.ndim != 3 or rgb.shape[2] != 3: + return False, "invalid_rgb" + if not isinstance(re_img, np.ndarray) or re_img.ndim != 2: + return False, "invalid_re" + if not isinstance(nir_img, np.ndarray) or nir_img.ndim != 2: + return False, "invalid_nir" + + # Produto já trabalha em float32 após decode/radiometria. Evitamos + # conversões/copias escondidas dentro do backend fused. + if rgb.dtype != np.float32: + return False, f"rgb_dtype={rgb.dtype}" + if re_img.dtype != np.float32: + return False, f"re_dtype={re_img.dtype}" + if nir_img.dtype != np.float32: + return False, f"nir_dtype={nir_img.dtype}" + + dst_shape = maps["rgb"]["map2"].shape + if maps["re"]["map2"].shape != dst_shape or maps["nir"]["map2"].shape != dst_shape: + return False, "target_map_shape_mismatch" + + return True, "ok" + + + def _direct_fusion_write_numba_fused5_fast( + self, + tensor, + decoded, + role_to_cam, + remap_cache, + ): + """Executa RGB+RE+NIR -> Raw5 CHW em uma única chamada Numba.""" + maps = remap_cache["maps"] + + rgb = decoded[role_to_cam["rgb"]]["image"] + re_img = decoded[role_to_cam["re"]]["image"] + nir_img = decoded[role_to_cam["nir"]]["image"] + + rgb_maps = maps["rgb"] + re_maps = maps["re"] + nir_maps = maps["nir"] + + # Se o backend OpenCV atual usaria warpAffine para o RGB, usamos o + # fixed-map equivalente ao warpAffine. Caso contrário, usamos o remap + # normal. Assim o fused compara com a ÚLTIMA versão, não com uma etapa + # anterior do pipeline. + rgb_map1 = rgb_maps.get("warp_affine_map1", rgb_maps["map1"]) + rgb_map2 = rgb_maps.get("warp_affine_map2", rgb_maps["map2"]) + + t0 = time.perf_counter() + _direct_fused5_fixedmap_numba( + rgb, + re_img, + nir_img, + rgb_map1, + rgb_map2, + re_maps["map1"], + re_maps["map2"], + nir_maps["map1"], + nir_maps["map2"], + tensor, + ) + elapsed_ms = (time.perf_counter() - t0) * 1000.0 + + return { + "fused5_ms": float(elapsed_ms), + "backend": "numba_fused_5ch_fixedmap", + } + + def _fuse_multispec_direct_to_target_fast( self, decoded, @@ -7580,92 +7876,149 @@ class RawProcessorCore: remap_map_shapes = {} # ------------------------------------------------------------ - # RGB + # Backend espacial: Numba fused 5ch ou OpenCV conservador # ------------------------------------------------------------ - t0_rgb = time.perf_counter() - rgb_maps = remap_cache["maps"].get("rgb") - if use_remap_cache and use_remap_for_rgb and rgb_maps is not None: - self._direct_fusion_write_rgb_remap_fast( - tensor=tensor, - rgb=rgb, - remap_entry=rgb_maps, - ) - else: - self._direct_fusion_write_rgb_fast( - tensor, - rgb, - crop_box, - target_size, - ) - t_rgb_ms = (time.perf_counter() - t0_rgb) * 1000.0 + direct_backend_requested = str( + (self.fusion_config or {}).get("direct_backend", "auto") or "auto" + ).strip().lower() + force_opencv = direct_backend_requested in ( + "opencv", + "cv2", + "legacy", + "opencv_fastpath", + ) + request_numba = direct_backend_requested in ( + "auto", + "numba", + "numba5", + "fused5", + "numba_fused_5ch", + ) + + if not force_opencv and not request_numba: + # Valor desconhecido: fail-safe para OpenCV e registra o motivo. + force_opencv = True + + fused5_used = False + fused5_ms = 0.0 + fused5_fallback_reason = "forced_opencv" if force_opencv else "not_attempted" + + if request_numba and not force_opencv: + can_fused5, fused5_fallback_reason = self._direct_fusion_can_use_numba_fused5_fast( + decoded=decoded, + role_to_cam=role_to_cam, + remap_cache=remap_cache, + channels_expected=channels_expected, + use_remap_cache=use_remap_cache, + use_remap_for_rgb=use_remap_for_rgb, + use_remap_for_spec=use_remap_for_spec, + ) + + if can_fused5: + fused_perf = self._direct_fusion_write_numba_fused5_fast( + tensor=tensor, + decoded=decoded, + role_to_cam=role_to_cam, + remap_cache=remap_cache, + ) + fused5_used = True + fused5_ms = float(fused_perf.get("fused5_ms", 0.0)) + fused5_fallback_reason = "" + + # Perf fields históricos. No fused, o trabalho RGB/RE/NIR acontece + # inseparavelmente dentro de fused5_ms, por isso os subtempos ficam 0. + rgb_remap_perf = { + "remap_ms": 0.0, + "pack_ms": 0.0, + "backend": ( + "numba_fused_5ch_fixedmap" if fused5_used else "legacy_crop_resize" + ), + } warp_details = {} t_warp_total_ms = 0.0 + t_rgb_ms = 0.0 - # ------------------------------------------------------------ - # RE - # ------------------------------------------------------------ - if "re" in role_to_cam: - t0w = time.perf_counter() - re_img = decoded[role_to_cam["re"]]["image"] - re_maps = remap_cache["maps"].get("re") - - if use_remap_cache and use_remap_for_spec and re_maps is not None: - self._direct_fusion_write_spec_remap_fast( + if not fused5_used: + # -------------------------------------------------------- + # OpenCV fast-path anterior, preservado como fallback/A-B + # -------------------------------------------------------- + t0_rgb = time.perf_counter() + rgb_maps = remap_cache["maps"].get("rgb") + if use_remap_cache and use_remap_for_rgb and rgb_maps is not None: + rgb_remap_perf = self._direct_fusion_write_rgb_remap_fast( tensor=tensor, - channel_index=3, - img=re_img, - remap_entry=re_maps, + rgb=rgb, + remap_entry=rgb_maps, ) else: - self._direct_fusion_write_spec_cached_fast( - tensor=tensor, - channel_index=3, - img=re_img, - role="re", - geom=geom, - ref_size=ref_size, - target_size=target_size, + self._direct_fusion_write_rgb_fast( + tensor, + rgb, + crop_box, + target_size, ) + t_rgb_ms = (time.perf_counter() - t0_rgb) * 1000.0 - warp_details["re"] = (time.perf_counter() - t0w) * 1000.0 - t_warp_total_ms += warp_details["re"] + if "re" in role_to_cam: + t0w = time.perf_counter() + re_img = decoded[role_to_cam["re"]]["image"] + re_maps = remap_cache["maps"].get("re") - # ------------------------------------------------------------ - # NIR - # ------------------------------------------------------------ - if "nir" in role_to_cam: - t0w = time.perf_counter() - nir_img = decoded[role_to_cam["nir"]]["image"] - nir_maps = remap_cache["maps"].get("nir") + if use_remap_cache and use_remap_for_spec and re_maps is not None: + self._direct_fusion_write_spec_remap_fast( + tensor=tensor, + channel_index=3, + img=re_img, + remap_entry=re_maps, + ) + else: + self._direct_fusion_write_spec_cached_fast( + tensor=tensor, + channel_index=3, + img=re_img, + role="re", + geom=geom, + ref_size=ref_size, + target_size=target_size, + ) - if use_remap_cache and use_remap_for_spec and nir_maps is not None: - self._direct_fusion_write_spec_remap_fast( - tensor=tensor, - channel_index=4, - img=nir_img, - remap_entry=nir_maps, - ) - else: - self._direct_fusion_write_spec_cached_fast( - tensor=tensor, - channel_index=4, - img=nir_img, - role="nir", - geom=geom, - ref_size=ref_size, - target_size=target_size, - ) + warp_details["re"] = (time.perf_counter() - t0w) * 1000.0 + t_warp_total_ms += warp_details["re"] - warp_details["nir"] = (time.perf_counter() - t0w) * 1000.0 - t_warp_total_ms += warp_details["nir"] + if "nir" in role_to_cam: + t0w = time.perf_counter() + nir_img = decoded[role_to_cam["nir"]]["image"] + nir_maps = remap_cache["maps"].get("nir") + + if use_remap_cache and use_remap_for_spec and nir_maps is not None: + self._direct_fusion_write_spec_remap_fast( + tensor=tensor, + channel_index=4, + img=nir_img, + remap_entry=nir_maps, + ) + else: + self._direct_fusion_write_spec_cached_fast( + tensor=tensor, + channel_index=4, + img=nir_img, + role="nir", + geom=geom, + ref_size=ref_size, + target_size=target_size, + ) + + warp_details["nir"] = (time.perf_counter() - t0w) * 1000.0 + t_warp_total_ms += warp_details["nir"] # ------------------------------------------------------------ # Flat-field no espaço final do tensor # ------------------------------------------------------------ - t0_final_flat = time.perf_counter() final_flat_enabled = False final_flat_cache_available = False + final_flat_prepare_ms = 0.0 + final_flat_apply_ms = 0.0 flat_cfg = self.flatfield_config or {} apply_final_flat = ( @@ -7675,6 +8028,8 @@ class RawProcessorCore: if apply_final_flat: final_flat_enabled = True + + t0_flat_prepare = time.perf_counter() gain_tensor = self._get_final_flat_gain_tensor_cached_fast( decoded=decoded, role_to_cam=role_to_cam, @@ -7683,16 +8038,41 @@ class RawProcessorCore: crop_box=crop_box, target_size=target_size, ) + final_flat_prepare_ms = (time.perf_counter() - t0_flat_prepare) * 1000.0 if gain_tensor is not None: final_flat_cache_available = True + t0_flat_apply = time.perf_counter() tensor = self._apply_final_flat_gain_tensor_inplace( tensor=tensor, gain_tensor=gain_tensor, clip_output=bool(flat_cfg.get("clip_output", True)), ) + final_flat_apply_ms = (time.perf_counter() - t0_flat_apply) * 1000.0 - final_flat_ms = (time.perf_counter() - t0_final_flat) * 1000.0 + self.last_flatfield_result = { + "enabled": True, + "applied": True, + "apply_space": "final_tensor_space", + "backend": "final_tensor_gain_numba" if _HAS_NUMBA else "final_tensor_gain_numpy", + "warnings": ( + ["saturation_guard_not_applied_in_final_tensor_space"] + if bool(flat_cfg.get("saturation_guard_enabled", True)) + else [] + ), + "by_role": { + "rgb": {"applied": True, "channels": ["R", "G", "B"]}, + "re": {"applied": True, "channels": ["RE"]}, + "nir": {"applied": True, "channels": ["NIR"]}, + }, + "applied_roles": ["nir", "re", "rgb"], + "missing_roles": [], + "perf_by_role_ms": {}, + "prepare_ms": float(final_flat_prepare_ms), + "apply_ms": float(final_flat_apply_ms), + } + + final_flat_ms = float(final_flat_prepare_ms + final_flat_apply_ms) if tensor.shape[0] != channels_expected: raise RuntimeError( @@ -7708,11 +8088,21 @@ class RawProcessorCore: "geometry_ms": float(t_geometry_ms), "tensor_alloc_ms": float(t_tensor_alloc_ms), "rgb_crop_resize_ms": float(t_rgb_ms), + "rgb_remap_ms": float(rgb_remap_perf.get("remap_ms", 0.0)), + "rgb_pack_ms": float(rgb_remap_perf.get("pack_ms", 0.0)), + "rgb_backend": str(rgb_remap_perf.get("backend", "unknown")), "warp_total_ms": float(t_warp_total_ms), "warp_details_ms": warp_details, "crop_resize_ms": float(t_rgb_ms), "concat_ms": 0.0, - "spatial_direct_ms": float(t_rgb_ms + t_warp_total_ms), + "spatial_direct_ms": float(fused5_ms if fused5_used else (t_rgb_ms + t_warp_total_ms)), + "direct_backend_requested": str(direct_backend_requested), + "direct_backend": ( + "numba_fused_5ch_fixedmap" if fused5_used else "opencv_fastpath" + ), + "fused5_used": bool(fused5_used), + "fused5_ms": float(fused5_ms), + "fused5_fallback_reason": str(fused5_fallback_reason or ""), "geometry_cache_hit": bool(geom.get("prepare_cache_hit", False)), "geometry_cache_hits": int(geom.get("cache_hits", 0)), "geometry_cache_misses": int(geom.get("cache_misses", 0)), @@ -7728,6 +8118,8 @@ class RawProcessorCore: "final_flat_enabled": bool(final_flat_enabled), "final_flat_cache_available": bool(final_flat_cache_available), "final_flat_ms": float(final_flat_ms), + "final_flat_prepare_ms": float(final_flat_prepare_ms), + "final_flat_apply_ms": float(final_flat_apply_ms), "target_size": [int(target_w), int(target_h)], "crop_box": [int(v) for v in crop_box], "direct_total_ms": float(t_direct_total_ms), @@ -7806,6 +8198,9 @@ class RawProcessorCore: self._direct_fusion_remap_cache_hits = 0 self._direct_fusion_remap_cache_misses = 0 + # Buffers de destino dependem do target final. Recriados sob demanda. + self._direct_fusion_runtime_buffers = {} + def _direct_fusion_get_geometry_cached_fast( self, decoded, @@ -7958,6 +8353,70 @@ class RawProcessorCore: tensor[int(channel_index)] = out + def _direct_fusion_build_warp_affine_fixedmap_fast( + self, + M_src_to_dst: np.ndarray, + dst_size: tuple, + ): + """ + Materializa uma vez o mesmo fixed-point map usado por cv2.warpAffine + com INTER_LINEAR. Isso permite ao kernel fused Numba reproduzir também + o fast-path RGB atual, em vez de voltar à convenção do cv2.remap. + """ + M = np.asarray(M_src_to_dst, dtype=np.float64) + if M.shape == (3, 3): + A = M[:2, :] + elif M.shape == (2, 3): + A = M + else: + raise RuntimeError(f"M affine inválida: shape={M.shape}") + + target_w, target_h = int(dst_size[0]), int(dst_size[1]) + inv = cv2.invertAffineTransform(A).astype(np.float64, copy=False) + + inter_bits = 5 + inter_tab = 1 << inter_bits + ab_bits = max(10, inter_bits) + ab_scale = 1 << ab_bits + round_delta = ab_scale // inter_tab // 2 + shift = ab_bits - inter_bits + + xs = np.arange(target_w, dtype=np.float64) + ys = np.arange(target_h, dtype=np.float64) + + # np.rint reproduz o arredondamento para o inteiro mais próximo usado + # pelo cvRound/saturate_cast neste domínio de coordenadas. + adelta = np.rint(inv[0, 0] * xs * ab_scale).astype(np.int64) + bdelta = np.rint(inv[1, 0] * xs * ab_scale).astype(np.int64) + + x0 = ( + np.rint((inv[0, 1] * ys + inv[0, 2]) * ab_scale).astype(np.int64) + + int(round_delta) + ) + y0 = ( + np.rint((inv[1, 1] * ys + inv[1, 2]) * ab_scale).astype(np.int64) + + int(round_delta) + ) + + X = (x0[:, None] + adelta[None, :]) >> shift + Y = (y0[:, None] + bdelta[None, :]) >> shift + + ix = X >> inter_bits + iy = Y >> inter_bits + fx = X & (inter_tab - 1) + fy = Y & (inter_tab - 1) + + # O produto opera em coordenadas pequenas (< 2k), mas clamp explícito + # preserva o contrato de armazenamento CV_16SC2. + i16 = np.iinfo(np.int16) + map1 = np.empty((target_h, target_w, 2), dtype=np.int16) + map1[:, :, 0] = np.clip(ix, i16.min, i16.max).astype(np.int16) + map1[:, :, 1] = np.clip(iy, i16.min, i16.max).astype(np.int16) + map2 = (fy * inter_tab + fx).astype(np.uint16) + + return map1, map2 + + def _direct_fusion_build_remap_from_src_to_dst_fast( self, M_src_to_dst: np.ndarray, @@ -8021,15 +8480,41 @@ class RawProcessorCore: # convertMaps deixa o remap mais barato em muitos casos. map1, map2 = cv2.convertMaps(map_x, map_y, cv2.CV_16SC2) - return { + entry = { "map_x": map_x, "map_y": map_y, "map1": map1, "map2": map2, + # Mantemos também a matriz forward original. Para RGB, quando ela + # for affine e axis-aligned (crop/scale + 0/180/flips), podemos usar + # warpAffine no runtime e evitar ler os mapas densos por frame. + "M_src_to_dst": M.astype(np.float32, copy=False), "src_shape": [src_h, src_w], "dst_size": [target_w, target_h], } + eps = 1e-7 + axis_affine = bool( + abs(float(M[0, 1])) <= eps + and abs(float(M[1, 0])) <= eps + and abs(float(M[2, 0])) <= eps + and abs(float(M[2, 1])) <= eps + and abs(float(M[2, 2]) - 1.0) <= eps + ) + + if axis_affine: + warp_map1, warp_map2 = self._direct_fusion_build_warp_affine_fixedmap_fast( + M_src_to_dst=M, + dst_size=(target_w, target_h), + ) + entry["warp_affine_map1"] = warp_map1 + entry["warp_affine_map2"] = warp_map2 + entry["axis_affine"] = True + else: + entry["axis_affine"] = False + + return entry + def _direct_fusion_get_remap_cache_key_fast( self, geom: dict, @@ -8178,6 +8663,43 @@ class RawProcessorCore: cache[key] = remap return remap + def _direct_fusion_get_rgb_scratch_fast(self, target_h: int, target_w: int): + """ + Buffer HWC float32 reutilizável para o remap RGB. + + O tensor final é CHW, enquanto cv2.remap é mais eficiente processando o + RGB intercalado em uma única chamada. Reutilizamos este scratch para + eliminar a alocação HWC por frame e depois usamos cv2.mixChannels para + copiar os 3 canais diretamente para os planos CHW do tensor. + """ + target_h = int(target_h) + target_w = int(target_w) + key = (target_h, target_w) + + cache = getattr(self, "_direct_fusion_runtime_buffers", None) + if not isinstance(cache, dict): + cache = {} + self._direct_fusion_runtime_buffers = cache + + entry = cache.get(key) + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("rgb_hwc"), np.ndarray) + or entry["rgb_hwc"].shape != (target_h, target_w, 3) + or entry["rgb_hwc"].dtype != np.float32 + ): + entry = { + "rgb_hwc": np.empty( + (target_h, target_w, 3), + dtype=np.float32, + ) + } + if len(cache) >= 4: + cache.clear() + cache[key] = entry + + return entry["rgb_hwc"] + def _direct_fusion_write_rgb_remap_fast( self, tensor: np.ndarray, @@ -8185,23 +8707,81 @@ class RawProcessorCore: remap_entry: dict, ): """ - RGB HWC -> tensor CHW usando cv2.remap direto para target final. + RGB HWC -> tensor CHW usando o mesmo cv2.remap do caminho anterior, + porém sem alocar um HWC novo em todo frame. + + Matemática/interpolação permanecem idênticas: mesmos map1/map2, + INTER_LINEAR e BORDER_CONSTANT. Somente o destino e o empacotamento + HWC->CHW foram otimizados. """ map1 = remap_entry["map1"] map2 = remap_entry["map2"] - rgb_out = cv2.remap( - rgb.astype(np.float32, copy=False), - map1, - map2, - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=0.0, + target_h, target_w = int(map1.shape[0]), int(map1.shape[1]) + rgb_out = self._direct_fusion_get_rgb_scratch_fast( + target_h, + target_w, ) - tensor[0] = rgb_out[:, :, 0] - tensor[1] = rgb_out[:, :, 1] - tensor[2] = rgb_out[:, :, 2] + rgb_f = rgb.astype(np.float32, copy=False) + M = remap_entry.get("M_src_to_dst") + + # Fast-path exato para a geometria RGB de produto atual: + # crop/scale + orientação 0/180/flips => matriz affine axis-aligned. + # Para essa família, OpenCV warpAffine e o remap fixo gerado por + # convertMaps produzem os mesmos pixels, mas warpAffine evita ler + # map1/map2 densos a cada frame. + use_axis_affine = False + if isinstance(M, np.ndarray) and M.shape == (3, 3): + eps = 1e-7 + use_axis_affine = bool( + abs(float(M[0, 1])) <= eps + and abs(float(M[1, 0])) <= eps + and abs(float(M[2, 0])) <= eps + and abs(float(M[2, 1])) <= eps + and abs(float(M[2, 2]) - 1.0) <= eps + ) + + t0 = time.perf_counter() + if use_axis_affine: + cv2.warpAffine( + rgb_f, + M[:2], + (target_w, target_h), + dst=rgb_out, + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=0.0, + ) + backend = "opencv_warpAffine_axis_aligned_cached_hwc" + else: + cv2.remap( + rgb_f, + map1, + map2, + interpolation=cv2.INTER_LINEAR, + dst=rgb_out, + borderMode=cv2.BORDER_CONSTANT, + borderValue=0.0, + ) + backend = "opencv_remap_cached_hwc" + remap_ms = (time.perf_counter() - t0) * 1000.0 + + t0 = time.perf_counter() + # mixChannels faz o HWC -> 3 planos CHW em uma única rotina OpenCV, + # evitando três atribuições NumPy independentes. + cv2.mixChannels( + [rgb_out], + [tensor[0], tensor[1], tensor[2]], + [0, 0, 1, 1, 2, 2], + ) + pack_ms = (time.perf_counter() - t0) * 1000.0 + + return { + "remap_ms": float(remap_ms), + "pack_ms": float(pack_ms), + "backend": backend + "_mixchannels", + } def _direct_fusion_write_spec_remap_fast( self, @@ -8211,21 +8791,26 @@ class RawProcessorCore: remap_entry: dict, ): """ - RE/NIR -> tensor usando cv2.remap direto para target final. + RE/NIR -> tensor usando cv2.remap diretamente no plano CHW final. + + Evita o ndarray temporário 960x600 e a cópia subsequente para + tensor[channel_index], sem alterar a interpolação. """ map1 = remap_entry["map1"] map2 = remap_entry["map2"] + dst = tensor[int(channel_index)] - out = cv2.remap( + cv2.remap( img.astype(np.float32, copy=False), map1, map2, interpolation=cv2.INTER_LINEAR, + dst=dst, borderMode=cv2.BORDER_CONSTANT, borderValue=0.0, ) - tensor[int(channel_index)] = out + return dst def _resolve_homography_profile_name_for_role(self, role: str) -> str: """ @@ -8725,7 +9310,6 @@ class RawProcessorCore: strength = float(cfg.get("strength", 1.0)) strength_by_channel = cfg.get("strength_by_channel", {}) or {} - gain_min_runtime = float(cfg.get("gain_min_runtime", 0.0)) gain_max_runtime = float(cfg.get("gain_max_runtime", 999.0)) @@ -8734,7 +9318,7 @@ class RawProcessorCore: runtime_smooth_ksize += 1 key = ( - "final_flat_gain_tensor", + "final_flat_gain_tensor_v2_same_direct_maps", geom.get("key"), tuple(int(v) for v in crop_box), int(target_w), @@ -8751,25 +9335,56 @@ class RawProcessorCore: return cached gain_tensor = np.ones((5, target_h, target_w), dtype=np.float32) + maps = (remap_cache or {}).get("maps", {}) or {} + x0, y0, x1, y1 = [int(v) for v in crop_box] - # ------------------------------------------------------------ - # RGB: usa crop + resize, igual ao RGB real. - # ------------------------------------------------------------ - rgb_cam = role_to_cam.get("rgb") - if rgb_cam is not None: - rgb_img = decoded[rgb_cam]["image"] - rgb_shape = rgb_img.shape[:2] + # O gain de cada banda atravessa exatamente a mesma transformação + # espacial da banda real. Isso é especialmente importante no RGB, + # cuja orientação 180° pode estar fundida no mapa do direct. + channel_specs = ( + ("rgb", "R", 0), + ("rgb", "G", 1), + ("rgb", "B", 2), + ("re", "RE", 3), + ("nir", "NIR", 4), + ) - x0, y0, x1, y1 = [int(v) for v in crop_box] + for role, ch, ci in channel_specs: + cam_id = role_to_cam.get(role) + if cam_id is None: + continue - for ci, ch in enumerate(("R", "G", "B")): - gain_eff = self._get_runtime_gain_eff_map(ch, rgb_shape, cfg) + img = decoded[cam_id]["image"] + gain_eff = self._get_runtime_gain_eff_map(ch, img.shape[:2], cfg) + if gain_eff is None: + continue - if gain_eff is None: + gain_eff = gain_eff.astype(np.float32, copy=False) + remap_entry = maps.get(role) + + if isinstance(remap_entry, dict): + if role == "rgb": + map1 = remap_entry.get("warp_affine_map1", remap_entry.get("map1")) + map2 = remap_entry.get("warp_affine_map2", remap_entry.get("map2")) + else: + map1 = remap_entry.get("map1") + map2 = remap_entry.get("map2") + + if isinstance(map1, np.ndarray) and isinstance(map2, np.ndarray): + gain_out = cv2.remap( + gain_eff, + map1, + map2, + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=1.0, + ) + gain_tensor[ci] = gain_out.astype(np.float32, copy=False) continue - gain_crop = gain_eff[y0:y1, x0:x1].astype(np.float32, copy=False) - + # Fallback conservador se algum perfil rodar sem remap cache. + if role == "rgb": + gain_crop = gain_eff[y0:y1, x0:x1] if gain_crop.shape[1] != target_w or gain_crop.shape[0] != target_h: gain_out = cv2.resize( gain_crop, @@ -8778,49 +9393,12 @@ class RawProcessorCore: ) else: gain_out = gain_crop - - gain_tensor[ci] = gain_out.astype(np.float32, copy=False) - - # ------------------------------------------------------------ - # RE/NIR: usa o mesmo remap cacheado da imagem real. - # ------------------------------------------------------------ - maps = (remap_cache or {}).get("maps", {}) or {} - - spec_map = { - "re": ("RE", 3), - "nir": ("NIR", 4), - } - - for role, (ch, ci) in spec_map.items(): - cam_id = role_to_cam.get(role) - if cam_id is None: - continue - - img = decoded[cam_id]["image"] - gain_eff = self._get_runtime_gain_eff_map(ch, img.shape[:2], cfg) - - if gain_eff is None: - continue - - remap_entry = maps.get(role) - - if remap_entry is not None: - gain_out = cv2.remap( - gain_eff.astype(np.float32, copy=False), - remap_entry["map1"], - remap_entry["map2"], - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=1.0, - ) else: M_role_to_target = geom["M_role_to_target"].get(role) - if M_role_to_target is None: continue - gain_out = cv2.warpPerspective( - gain_eff.astype(np.float32, copy=False), + gain_eff, M_role_to_target, (target_w, target_h), flags=cv2.INTER_LINEAR, @@ -8830,6 +9408,9 @@ class RawProcessorCore: gain_tensor[ci] = gain_out.astype(np.float32, copy=False) + if not gain_tensor.flags.c_contiguous: + gain_tensor = np.ascontiguousarray(gain_tensor) + self._flatfield_runtime_cache[key] = gain_tensor return gain_tensor diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/camera_manager.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/camera_manager.py index b94d63500..83de8c489 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/camera_manager.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/camera_manager.py @@ -1,6 +1,8 @@ +import base64 import datetime import json import os +import queue import threading import time from pathlib import Path @@ -26,7 +28,7 @@ from visual_worker.utils import converter_valores_numpy from shared.perf_monitor import VisualPerfMonitor -CAMERA_MANAGER_VERSION = "production_v1_2026_08_24" +CAMERA_MANAGER_VERSION = "production_v1_2026_09_12_runtime_priority_v3" PRODUCT_ASSEMBLY_SCHEMA = "multispec_module_params_assembly_v1" PRODUCT_MODULE_SCHEMA = "multispec_module_params_v3" @@ -87,6 +89,8 @@ class CameraManager: self._loop_deteccao_iniciado = False self._loop_analise_iniciado = False self._loop_stream_iniciado = False + self._loop_preview_iniciado = False + self._loop_replay_writer_iniciado = False self._loop_publicacao_iniciado = False self._vida_lock = threading.RLock() @@ -101,6 +105,25 @@ class CameraManager: self._pub_lock = threading.RLock() self._preview_lock = threading.RLock() + # Preview/replay são explicitamente secundários ao runtime. + # Produção visual ocorre em loop próprio, em baixa frequência, + # e consumidores só leem o cache pronto. + self.preview_fps = 1.0 + self.preview_max_width = 480 + self.preview_jpeg_quality = 45 + self._preview_encoded_cache = {} + + # Snapshot local do estado Redis usado pelo hot path. + # Operação/controle/contexto são atualizados em até 100 ms; equipamento + # é quase estático e pode ser atualizado bem mais devagar. + self.runtime_snapshot_interval_s = 0.10 + self.runtime_equipment_interval_s = 2.0 + self.runtime_snapshot_max_age_s = 0.50 + + # Escrita de replay é assíncrona e descartável sob pressão. + # Nunca bloqueia tensor/inferência/detecção por causa de disco. + self._replay_save_queue = queue.Queue(maxsize=32) + self.perf = VisualPerfMonitor(janela=180) self._ultimo_posproc_save_ts = 0.0 @@ -164,6 +187,8 @@ class CameraManager: self._ultimo_preview_seg_ts = 0.0 self._ultimo_preview_overlay_ts = 0.0 self._ultimo_preview_debug_ts = 0.0 + with self._preview_lock: + self._preview_encoded_cache = {} self._fps_infer_last_ts = None self._fps_infer_ema = 0.0 @@ -172,6 +197,16 @@ class CameraManager: self._ultimo_loop_analise_fps = 0.0 self._ultimo_config_update_ts = 0.0 + self._ultimo_runtime_snapshot_ts = 0.0 + self._ultimo_equipment_snapshot_ts = 0.0 + self._runtime_equipment_cache = {} + self._runtime_ctx_snapshot = { + "ts": 0.0, + "operacao": {}, + "controle": {}, + "contexto": {}, + "equipamento": {}, + } # Diagnostico fino do pipeline. Estes campos sao somente observabilidade: # nao alteram frequencias, caches, gates ou comandos dos bicos. @@ -352,9 +387,19 @@ class CameraManager: ) fusion = mp.get("fusion_config", {}) or {} - mp_target = self._normalizar_size_wh( - fusion.get("target_size"), - "module_params.fusion_config.target_size", + mp_target_raw = fusion.get("target_size") + + # Contrato atual: + # - fusion_config.target_size pode ser None no module_params; + # - ONNX estático é a autoridade do tamanho final; + # - se um MP legado trouxer target_size, ele continua validado. + mp_target = ( + None + if mp_target_raw is None + else self._normalizar_size_wh( + mp_target_raw, + "module_params.fusion_config.target_size", + ) ) sizes_raw = mp.get("sensor_size_by_role", {}) or {} @@ -408,11 +453,9 @@ class CameraManager: """ Fecha a autoridade da resolução antes de abrir a OAK: - ONNX input [W,H] - == - module_params.fusion_config.target_size - == - seg_config.ia_resolution + ONNX input [W,H] = autoridade final do runtime + module_params.fusion_config.target_size = opcional/legado + seg_config.ia_resolution = opcional, mas deve casar com ONNX camera_width/camera_height deixam de participar da decisão física. """ @@ -422,9 +465,16 @@ class CameraManager: mx_id, ) - mp_target = list(mp_info["target_size"]) + mp_target_raw = mp_info.get("target_size") + mp_target = ( + None + if mp_target_raw is None + else list(mp_target_raw) + ) - if model_target != mp_target: + # MP novo pode omitir o target final. Nesse caso o ONNX manda. + # MP legado com target explícito continua fail-closed se divergir. + if mp_target is not None and model_target != mp_target: raise RuntimeError( "Contrato de resolução incompatível entre ONNX e module_params: " f"ONNX={model_target} module_params={mp_target}" @@ -439,7 +489,7 @@ class CameraManager: if cfg_target != model_target: raise RuntimeError( - "Contrato de resolução incompatível entre config, ONNX e MP: " + "Contrato de resolução incompatível entre config e ONNX: " f"config={cfg_target} ONNX={model_target} MP={mp_target}" ) else: @@ -763,7 +813,8 @@ class CameraManager: self.mx_id = str(mx_id) from weed_worker.config import load_seg_config - self.seg_config = load_seg_config() + runtime_snapshot = self._atualizar_snapshot_runtime(force=True) + self.seg_config = load_seg_config(runtime_snapshot=runtime_snapshot) self.posproc_intervalo_min_s = float( self.seg_config.get("posproc_intervalo_min_s", 5.0) ) @@ -1067,7 +1118,7 @@ class CameraManager: if self.weed_detector is not None: return - self.weed_detector = WeedDetector() + self.weed_detector = WeedDetector(config=self.seg_config) def _iniciar_loops_se_necessario(self, seg_config): freq_analise = float(seg_config.get("analise_fps", 15.0)) @@ -1076,6 +1127,20 @@ class CameraManager: freq_inferencia = float(seg_config.get("inferencia_fps", freq_tensor)) freq_deteccao = float(seg_config.get("deteccao_fps", freq_inferencia)) + # Parâmetros visuais são de baixa prioridade e podem ser ajustados + # sem tocar no contrato científico/runtime. + self.preview_fps = max(0.2, float(seg_config.get("preview_fps", 1.0) or 1.0)) + self.preview_max_width = max(160, int(seg_config.get("preview_max_width", 480) or 480)) + self.preview_jpeg_quality = max(20, min(85, int(seg_config.get("preview_jpeg_quality", 45) or 45))) + + if not self._loop_preview_iniciado: + self._iniciar_loop_preview_cache(freq=self.preview_fps) + self._loop_preview_iniciado = True + + if not self._loop_replay_writer_iniciado: + self._iniciar_loop_replay_writer() + self._loop_replay_writer_iniciado = True + if not self._loop_stream_iniciado: stream = getattr(self.camera, "stream", None) stream_fps = float(getattr(stream, "_op_fps", 2.0) or 2.0) @@ -1261,7 +1326,7 @@ class CameraManager: if detector is None or reset_nome is None: # Fallback seguro: recria apenas o detector, nunca câmera/modelo. - self.weed_detector = WeedDetector() + self.weed_detector = WeedDetector(config=self.seg_config) if self.seg_config and hasattr(self.weed_detector, "atualizar_config"): self.weed_detector.atualizar_config(self.seg_config) reset_nome = "recriado" @@ -1540,6 +1605,63 @@ class CameraManager: except Exception as e: self.mostrar_log(f"[saude] erro: {e}") + def _atualizar_snapshot_runtime(self, intervalo_s=None, force=False): + """ + Atualiza um snapshot local de Redis fora do caminho frame-a-frame. + + Atribuição do dict final é atômica em CPython: leitores do detector + enxergam o snapshot antigo ou o novo, nunca um objeto parcialmente + montado. Em caso de falha, preserva o último snapshot válido; o gate + fail-closed verifica a idade antes de liberar pulverização. + """ + agora = time.time() + if intervalo_s is None: + intervalo_s = float(self.runtime_snapshot_interval_s) + + atual = self._runtime_ctx_snapshot or {} + if ( + not force + and float(atual.get("ts", 0.0) or 0.0) > 0.0 + and (agora - float(self._ultimo_runtime_snapshot_ts or 0.0)) < float(intervalo_s) + ): + return atual + + try: + operacao = ContextoGlobalRedis.get_operacao() or {} + controle = ContextoGlobalRedis.get_controle() or {} + contexto = ContextoGlobalRedis.get_contexto() or {} + + equipamento = self._runtime_equipment_cache + precisa_equip = ( + force + or not equipamento + or (agora - float(self._ultimo_equipment_snapshot_ts or 0.0)) + >= float(self.runtime_equipment_interval_s) + ) + if precisa_equip: + equipamento_novo = ContextoGlobalRedis.get_equipamento() or {} + if equipamento_novo: + equipamento = equipamento_novo + self._runtime_equipment_cache = equipamento_novo + self._ultimo_equipment_snapshot_ts = agora + + novo = { + "ts": float(agora), + "operacao": operacao, + "controle": controle, + "contexto": contexto, + "equipamento": equipamento or {}, + } + self._runtime_ctx_snapshot = novo + self._ultimo_runtime_snapshot_ts = agora + return novo + + except Exception as e: + # Não transforma uma falha transitória de Redis em exceção dentro + # do detector. Snapshot antigo será rejeitado pelo gate se envelhecer. + self.perf.inc("runtime_snapshot_erros") + return atual + def _atualizar_config_dinamica_detector(self, intervalo_s=0.20): """ Atualiza config do detector e a cabeça ONNX em baixa frequência. @@ -1560,7 +1682,10 @@ class CameraManager: try: from weed_worker.config import load_seg_config - cfg = load_seg_config() + snapshot = self._atualizar_snapshot_runtime( + intervalo_s=self.runtime_snapshot_interval_s + ) + cfg = load_seg_config(runtime_snapshot=snapshot) cfg = self._validar_config_dinamica_contrato(cfg) self.seg_config = cfg self.qtd_bicos = int(cfg.get("qtd_bicos", self.qtd_bicos or 7) or 7) @@ -1773,73 +1898,151 @@ class CameraManager: return None, None, None def get_selected_frame(self, _frame_type: TipoFrameCamera): - agora = time.time() + """ + Compatibilidade externa: retorna SOMENTE o último preview cacheado. - try: - if _frame_type not in self.FRAME_TYPES_PREVIEW: - return None + Regra de produto: + - nunca chama preview_infer_cached(); + - nunca monta overlay/debug sob demanda; + - nunca força inferência; + - nunca toca no cursor da câmera. - # Se ainda não tem runtime/tensor, tenta devolver o último preview válido. - if self.model_svc is None or self._ultimo_raw_input is None: - return self._get_cached_preview_frame(_frame_type) + O produtor de preview é _iniciar_loop_preview_cache(). + """ + if _frame_type not in self.FRAME_TYPES_PREVIEW: + return None + return self._get_cached_preview_frame(_frame_type) - # Janela curta: usa cache recente. - if (agora - self._ultimo_preview_ts) < 0.20: - cached = self._get_cached_preview_frame(_frame_type) - if cached is not None: - return cached - - rgb_frame, seg_frame, overlay_frame, _, _ = self.model_svc.preview_infer_cached( - self._ultimo_raw_input, - self._ultimo_predictions, - alpha=0.5, - ) - - debug_frame = None - if _frame_type == TipoFrameCamera.Debug: - debug_frame = self.get_debug_frame( - mostrar=False, - overlay_bgr=overlay_frame, - ) - - # Atualiza o timestamp da tentativa de preview. - self._ultimo_preview_ts = agora - - # Importante: - # só atualiza cache se o frame novo for válido. - # Nunca apaga último válido com None. - self._set_cached_preview_frame(TipoFrameCamera.Rgb, rgb_frame, agora) - self._set_cached_preview_frame(TipoFrameCamera.Segmentacao, seg_frame, agora) - self._set_cached_preview_frame(TipoFrameCamera.Overlay, overlay_frame, agora) - self._set_cached_preview_frame(TipoFrameCamera.Debug, debug_frame, agora) - - # Retorna o frame pedido. - # Se o novo veio None, devolve o último válido cacheado. - return self._get_cached_preview_frame(_frame_type) - - except Exception as e: - self.mostrar_log(f"[weed] erro em get_selected_frame: {e}") - - # Mesmo em erro, tenta devolver último válido. - try: - return self._get_cached_preview_frame(_frame_type) - except Exception: - return None - - def _get_cached_preview_frame(self, frame_type): + def _get_cached_preview_frame_ref(self, frame_type): + """Retorna referência imutável do cache visual, sem cópia.""" with self._preview_lock: if frame_type == TipoFrameCamera.Rgb: - frame = self._ultimo_preview_rgb - elif frame_type == TipoFrameCamera.Segmentacao: - frame = self._ultimo_preview_seg - elif frame_type == TipoFrameCamera.Overlay: - frame = self._ultimo_preview_overlay - elif frame_type == TipoFrameCamera.Debug: - frame = self._ultimo_preview_debug - else: - frame = None + return self._ultimo_preview_rgb + if frame_type == TipoFrameCamera.Segmentacao: + return self._ultimo_preview_seg + if frame_type == TipoFrameCamera.Overlay: + return self._ultimo_preview_overlay + if frame_type == TipoFrameCamera.Debug: + return self._ultimo_preview_debug + return None - return self._copiar_frame(frame) + def _get_cached_preview_frame(self, frame_type): + # API legada pode receber uma cópia pequena (preview já reduzido). + # O hot path interno usa _get_cached_preview_frame_ref(). + return self._copiar_frame(self._get_cached_preview_frame_ref(frame_type)) + + def get_cached_preview_payload(self, frame_type): + """ + Retorna payload já comprimido/convertido para Base64. + É o caminho do GetCameraFrame via Redis e não executa OpenCV. + """ + if frame_type not in self.FRAME_TYPES_PREVIEW: + return None + + with self._preview_lock: + item = self._preview_encoded_cache.get(frame_type) + if not item: + return None + return { + "base64": item.get("base64"), + "jpeg": item.get("jpeg"), + "source_ts": float(item.get("source_ts", 0.0) or 0.0), + "width": int(item.get("width", 0) or 0), + "height": int(item.get("height", 0) or 0), + "quality": int(item.get("quality", self.preview_jpeg_quality) or self.preview_jpeg_quality), + } + + def get_cached_preview_jpeg(self, frame_type): + """Retorna bytes JPEG imutáveis do cache para replay em disco.""" + with self._preview_lock: + item = self._preview_encoded_cache.get(frame_type) + if not item: + return None, 0.0 + return item.get("jpeg"), float(item.get("source_ts", 0.0) or 0.0) + + def _preparar_preview_leve(self, frame): + if not self._frame_valido(frame): + return None + + arr = frame + if arr.dtype != np.uint8: + arr = self._normalizar_frame_para_uint8(arr) + + h, w = arr.shape[:2] + max_w = int(self.preview_max_width) + if w > max_w: + escala = max_w / float(w) + novo_h = max(1, int(round(h * escala))) + arr = cv2.resize(arr, (max_w, novo_h), interpolation=cv2.INTER_AREA) + else: + # Isola o cache de buffers reaproveitados pelo gerador de debug. + arr = arr.copy() + + return np.ascontiguousarray(arr) + + def _cache_preview_bundle(self, frames_por_tipo: dict, source_ts: float): + """ + Reduz, comprime e publica atomicamente o bundle visual. + Este método é chamado SOMENTE pelo preview producer. + """ + novos = {} + quality = int(self.preview_jpeg_quality) + + for frame_type, frame in frames_por_tipo.items(): + leve = self._preparar_preview_leve(frame) + if not self._frame_valido(leve): + continue + + ok, enc = cv2.imencode( + ".jpg", + leve, + [int(cv2.IMWRITE_JPEG_QUALITY), quality], + ) + if not ok: + continue + + jpeg_bytes = enc.tobytes() + b64 = base64.b64encode(jpeg_bytes).decode("ascii") + h, w = leve.shape[:2] + novos[frame_type] = { + "frame": leve, + "jpeg": jpeg_bytes, + "base64": b64, + "source_ts": float(source_ts), + "width": int(w), + "height": int(h), + "quality": quality, + } + + if not novos: + return False + + with self._preview_lock: + self._preview_encoded_cache.update(novos) + + item = novos.get(TipoFrameCamera.Rgb) + if item: + self._ultimo_preview_rgb = item["frame"] + self._ultimo_preview_rgb_ts = source_ts + + item = novos.get(TipoFrameCamera.Segmentacao) + if item: + self._ultimo_preview_seg = item["frame"] + self._ultimo_preview_seg_ts = source_ts + + item = novos.get(TipoFrameCamera.Overlay) + if item: + self._ultimo_preview_overlay = item["frame"] + self._ultimo_preview_overlay_ts = source_ts + + item = novos.get(TipoFrameCamera.Debug) + if item: + self._ultimo_preview_debug = item["frame"] + self._ultimo_preview_debug_ts = source_ts + + self._ultimo_preview_ts = float(source_ts) + + return True def get_debug_frame(self, mostrar=False, overlay_bgr=None): overlay = overlay_bgr if overlay_bgr is not None else self._ultimo_preview_overlay @@ -1854,7 +2057,9 @@ class CameraManager: "fps_loop": self._ultimo_loop_analise_fps, } - self._ultimo_preview_debug = self._montar_debug_overlay( + # Não publica diretamente no cache: o preview producer faz a + # publicação atômica depois de reduzir/comprimir o bundle inteiro. + return self._montar_debug_overlay( overlay_bgr=overlay, atuacao_bicos=self._ultimo_controle or {}, config=self.seg_config or {}, @@ -1862,8 +2067,6 @@ class CameraManager: mostrar=mostrar, ) - return self._ultimo_preview_debug - def _montar_debug_overlay( self, overlay_bgr, @@ -2012,42 +2215,151 @@ class CameraManager: return frame def _set_cached_preview_frame(self, frame_type, frame, ts=None): - if not self._frame_valido(frame): - return False - - ts = time.time() if ts is None else ts - frame_copy = self._copiar_frame(frame) - - if not self._frame_valido(frame_copy): - return False - - with self._preview_lock: - if frame_type == TipoFrameCamera.Rgb: - self._ultimo_preview_rgb = frame_copy - self._ultimo_preview_rgb_ts = ts - return True - - if frame_type == TipoFrameCamera.Segmentacao: - self._ultimo_preview_seg = frame_copy - self._ultimo_preview_seg_ts = ts - return True - - if frame_type == TipoFrameCamera.Overlay: - self._ultimo_preview_overlay = frame_copy - self._ultimo_preview_overlay_ts = ts - return True - - if frame_type == TipoFrameCamera.Debug: - self._ultimo_preview_debug = frame_copy - self._ultimo_preview_debug_ts = ts - return True - - return False + """Compatibilidade interna; publica um único frame no cache leve.""" + ts = time.time() if ts is None else float(ts) + return self._cache_preview_bundle({frame_type: frame}, ts) # ============================================================ # Loops # ============================================================ + def _iniciar_loop_preview_cache(self, freq=1.0): + """ + Produtor visual de baixa prioridade. + + Ele reaproveita o ÚLTIMO tensor + ÚLTIMA predição já calculados pelo + runtime e monta os previews em frequência baixa. Consumidores nunca + provocam esta construção. + """ + def loop(): + ultimo_pred_marker = None + + while True: + t0 = time.time() + t_perf0 = time.perf_counter() + did_work = False + + try: + if self._em_warmup or not self.operante: + time.sleep(0.20) + continue + + model_svc = self.model_svc + with self._pred_lock: + pred_cache = self._pred_cache + predictions = pred_cache.get("predictions") + raw_input = pred_cache.get("raw_input") + pred_ts = float(pred_cache.get("ts", 0.0) or 0.0) + tensor_ts = float(pred_cache.get("tensor_ts", 0.0) or 0.0) + generation = int(pred_cache.get("generation", -1)) + + if raw_input is None: + raw_input = self._ultimo_raw_input + + if model_svc is None or raw_input is None or predictions is None: + time.sleep(0.10) + continue + + # Marcador lógico do frame, independente do endereço Python + # do ndarray. Evita falso "mesmo frame" por reutilização de id(). + pred_marker = (generation, pred_ts, tensor_ts) + if pred_marker == ultimo_pred_marker: + time.sleep(min(0.10, max(0.02, 1.0 / max(freq, 0.2)))) + continue + + did_work = True + rgb_frame, seg_frame, overlay_frame, _, _ = model_svc.preview_infer_cached( + raw_input, + predictions, + alpha=0.5, + ) + + debug_frame = self.get_debug_frame( + mostrar=False, + overlay_bgr=overlay_frame, + ) + + source_ts = pred_ts if pred_ts > 0.0 else time.time() + ok = self._cache_preview_bundle( + { + TipoFrameCamera.Rgb: rgb_frame, + TipoFrameCamera.Segmentacao: seg_frame, + TipoFrameCamera.Overlay: overlay_frame, + TipoFrameCamera.Debug: debug_frame, + }, + source_ts=source_ts, + ) + + if ok: + ultimo_pred_marker = pred_marker + + except Exception as e: + self.perf.inc("preview_cache_erros") + self.mostrar_log(f"[weed] erro no preview producer: {e}") + + finally: + t_perf1 = time.perf_counter() + if did_work: + self.perf.tick( + "preview_cache", + latencia_ms=(t_perf1 - t_perf0) * 1000.0, + ) + + dt = time.time() - t0 + time.sleep(max(0.0, (1.0 / max(freq, 0.2)) - dt)) + + threading.Thread( + target=loop, + daemon=True, + name="weed-preview-cache", + ).start() + + def _iniciar_loop_replay_writer(self): + """Único escritor assíncrono dos JPEGs de replay.""" + def loop(): + while True: + try: + caminho, jpeg_bytes = self._replay_save_queue.get() + try: + os.makedirs(os.path.dirname(caminho), exist_ok=True) + with open(caminho, "wb") as f: + f.write(jpeg_bytes) + finally: + self._replay_save_queue.task_done() + except Exception as e: + self.mostrar_log(f"[weed][REPLAY_IO] erro: {e}") + time.sleep(0.05) + + threading.Thread( + target=loop, + daemon=True, + name="weed-replay-writer", + ).start() + + def _enfileirar_replay_jpeg(self, caminho: str, jpeg_bytes: bytes) -> bool: + if not jpeg_bytes: + return False + + try: + self._replay_save_queue.put_nowait((caminho, jpeg_bytes)) + return True + except queue.Full: + # Replay é best-effort. Sob pressão, descartamos o mais antigo + # para nunca sacrificar o runtime científico. + try: + self._replay_save_queue.get_nowait() + self._replay_save_queue.task_done() + except Exception: + pass + + try: + self._replay_save_queue.put_nowait((caminho, jpeg_bytes)) + self.perf.inc("replay_queue_drop_oldest") + return True + except queue.Full: + self.perf.inc("replay_queue_drop") + return False + def _iniciar_loop_captura_tensor(self, freq=25.0): def loop(): periodo = 1.0 / max(float(freq), 0.1) @@ -2273,6 +2585,7 @@ class CameraManager: "ts": pred_ts, "tensor_ts": tensor_ts, "predictions": predictions, + "raw_input": tensor5, "res": res, "infer_ms": float(infer_ms), "infer_gpu_ms": float(infer_forward_ms or 0.0), @@ -2368,6 +2681,9 @@ class CameraManager: t_cfg0 = time.perf_counter() cfg_cpu0 = time.thread_time() + self._atualizar_snapshot_runtime( + intervalo_s=self.runtime_snapshot_interval_s + ) self._atualizar_config_dinamica_detector(intervalo_s=0.20) t_cfg1 = time.perf_counter() config_ms = (t_cfg1 - t_cfg0) * 1000.0 @@ -2419,10 +2735,6 @@ class CameraManager: analise = analise_completa.get("dados_visuais", {}) detector_ms = (t_det1 - t_det0) * 1000.0 - t_conv0 = time.perf_counter() - analise_convertida = converter_valores_numpy(analise) - t_conv1 = time.perf_counter() - # ==================================================== # Gate de pulverização # ==================================================== @@ -2435,6 +2747,12 @@ class CameraManager: analise["pulverizacao"] = debug_pulverizacao t_ctrl1 = time.perf_counter() + # Converte SOMENTE depois do gate, para que telemetria e + # comando publiquem exatamente o mesmo estado final. + t_conv0 = time.perf_counter() + analise_convertida = converter_valores_numpy(analise) + t_conv1 = time.perf_counter() + # Publicação atômica em relação a fechar/substituir câmera. # Se a geração mudou, nenhum dado antigo chega aos caches/bicos. with self._vida_lock: @@ -2668,11 +2986,15 @@ class CameraManager: frame_type = TipoFrameCamera( camera_ctx.get("frame_type", TipoFrameCamera.Rgb.value) ) - frame = self.get_selected_frame(frame_type) + # Stream também é consumidor de cache. Não monta preview. + frame = self._get_cached_preview_frame_ref(frame_type) self.camera.enviar_frame_tcp(frame) if self.debug_visual: - self.get_debug_frame(mostrar=True) + dbg = self._get_cached_preview_frame_ref(TipoFrameCamera.Debug) + if dbg is not None: + cv2.imshow("Debug Weed Worker", dbg) + cv2.waitKey(1) except Exception as e: self.mostrar_log(f"Erro no loop de stream: {e}") @@ -2923,9 +3245,23 @@ class CameraManager: } try: - operacao = ContextoGlobalRedis.get_operacao() - controle = ContextoGlobalRedis.get_controle() - contexto = ContextoGlobalRedis.get_contexto() + snapshot = self._runtime_ctx_snapshot or {} + snapshot_ts = float(snapshot.get("ts", 0.0) or 0.0) + snapshot_age_s = (time.time() - snapshot_ts) if snapshot_ts > 0.0 else float("inf") + debug["snapshot_age_ms"] = ( + snapshot_age_s * 1000.0 + if np.isfinite(snapshot_age_s) + else None + ) + + # Segurança: nunca libera pulverização a partir de estado velho. + if snapshot_age_s > float(self.runtime_snapshot_max_age_s): + debug["motivo"] = f"runtime snapshot antigo: {snapshot_age_s:.3f}s" + return False, debug + + operacao = snapshot.get("operacao") or {} + controle = snapshot.get("controle") or {} + contexto = snapshot.get("contexto") or {} traj = (contexto.get("Trajetoria", {}) or {}) gerais = (contexto.get("Gerais", {}) or {}) @@ -3232,6 +3568,12 @@ class CameraManager: } resumo["pub_debug"] = getattr(self, "_ultimo_pub_debug", {}) + resumo["preview_cache"] = { + "fps_config": float(self.preview_fps), + "max_width": int(self.preview_max_width), + "jpeg_quality": int(self.preview_jpeg_quality), + "replay_queue_size": int(self._replay_save_queue.qsize()), + } self._set_pub_cache("performance_weed", resumo) @@ -3248,6 +3590,7 @@ class CameraManager: det = loops.get("deteccao", {}) pub = loops.get("publicacao", {}) stream = loops.get("stream", {}) + preview = loops.get("preview_cache", {}) pub_dbg = getattr(self, "_ultimo_pub_debug", {}) @@ -3257,7 +3600,8 @@ class CameraManager: f"inf={self._fps(inf):.1f} " f"det={self._fps(det):.1f} " f"pub={self._fps(pub):.1f} " - f"stream={self._fps(stream):.1f} | " + f"stream={self._fps(stream):.1f} " + f"preview={self._fps(preview):.1f} | " f"period inf={self._fmt(self._per(inf))}ms " f"det={self._fmt(self._per(det))}ms " f"tensor={self._fmt(self._per(tensor))}ms" @@ -3285,7 +3629,9 @@ class CameraManager: f"redis={self._fmt(self._m(pub, 'redis_ms'))} " f"debug_pub={pub_dbg.get('publicou', 0)} " f"campos={pub_dbg.get('campos', 0)} " - f"redis_dbg={pub_dbg.get('redis_ms', 0):.1f}ms" + f"redis_dbg={pub_dbg.get('redis_ms', 0):.1f}ms | " + f"PREVIEW total={self._fmt(self._lat(preview))} " + f"q={self._replay_save_queue.qsize()}" ) self.mostrar_log( @@ -3468,38 +3814,26 @@ class CameraManager: continue # ==================================================== - # 2) FLUXO ATUAL: imagens para replay + # 2) REPLAY VISUAL: grava exatamente o JPEG já cacheado. + # Nenhum resize/overlay/imencode é executado sob demanda. # ==================================================== - frame = self.get_selected_frame(tipo_enum) + jpeg_bytes, source_ts = self.get_cached_preview_jpeg(tipo_enum) - if not self._frame_valido(frame): + if not jpeg_bytes: self.mostrar_log( f"⚠️ Frame não salvo | " f"tipo={tipo_nome} " f"nome={nome_frame} " - f"motivo=sem_frame_valido_em_cache" + f"motivo=sem_jpeg_cacheado" ) continue - frame = self._copiar_frame(frame) - caminho = os.path.join(pasta, f"{nome_frame}.jpg") - - # Replay leve: JPEG comprimido. - # Mantém baixo uso de disco durante operação longa. - if frame.dtype != "uint8": - frame_salvar = self._normalizar_frame_para_uint8(frame) - else: - frame_salvar = frame - - ok = cv2.imwrite(caminho, frame_salvar, [int(cv2.IMWRITE_JPEG_QUALITY), 70]) - - if not ok or not os.path.exists(caminho): + ok = self._enfileirar_replay_jpeg(caminho, jpeg_bytes) + if not ok: self.mostrar_log( - f"❌ Falha ao salvar frame | " - f"tipo={tipo_nome} " - f"nome={nome_frame} " - f"caminho={caminho}" + f"⚠️ Replay descartado para proteger runtime | " + f"tipo={tipo_nome} nome={nome_frame}" ) continue diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/config.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/config.py index 1f7339c1c..3cbb17f1e 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/config.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/config.py @@ -28,9 +28,9 @@ Hardware/calibração: module_params.json homologado Resolução final do tensor: - ONNX input H/W - == - module_params.fusion_config.target_size + ONNX input H/W é a autoridade operacional. + module_params.fusion_config.target_size pode ser null no contrato novo; + quando preenchido, é apenas compatibilidade/default e deve casar com o ONNX. Ordem nominal dos canais: input_channels deste config @@ -171,7 +171,7 @@ WEED_DEFAULT_CONFIG = { # ======================================================== "debug_visual": False, "debug_perf": False, - "detector_debug_perf": True, + "detector_debug_perf": False, # RAW científico para pós-processamento. "posproc_intervalo_min_s": 5.0, @@ -577,10 +577,12 @@ def _positive( def aplicar_overrides_redis( cfg: dict, + runtime_snapshot: dict | None = None, ) -> dict: + snapshot = runtime_snapshot if isinstance(runtime_snapshot, dict) else None + operacao = ( - ContextoGlobalRedis - .get_operacao() + (snapshot.get("operacao") if snapshot is not None else ContextoGlobalRedis.get_operacao()) or {} ) @@ -593,14 +595,12 @@ def aplicar_overrides_redis( ) contexto = ( - ContextoGlobalRedis - .get_contexto() + (snapshot.get("contexto") if snapshot is not None else ContextoGlobalRedis.get_contexto()) or {} ) equipamento = ( - ContextoGlobalRedis - .get_equipamento() + (snapshot.get("equipamento") if snapshot is not None else ContextoGlobalRedis.get_equipamento()) or {} ) @@ -1011,18 +1011,16 @@ def normalizar_config_runtime( # API # ============================================================ -def load_seg_config(): +def load_seg_config(runtime_snapshot: dict | None = None): """ - Retorna snapshot NOVO a cada chamada. + Retorna um snapshot NOVO de configuração. - Não existe cache de valores dinâmicos: - - velocidade; - - cabeça; - - classe semântica; - - qtd_bicos; - - zona de atuação. + Se ``runtime_snapshot`` for informado, usa os valores já lidos pelo + CameraManager (operacao/contexto/equipamento) e NÃO toca no Redis. + Isso permite atualizar configuração em baixa frequência sem colocar + leituras externas no hot path de detecção. - O lock só impede leituras concorrentes inconsistentes durante a montagem. + Sem snapshot, preserva o comportamento legado e lê diretamente do Redis. """ with _CONFIG_LOCK: cfg = deepcopy( @@ -1030,7 +1028,8 @@ def load_seg_config(): ) cfg = aplicar_overrides_redis( - cfg + cfg, + runtime_snapshot=runtime_snapshot, ) cfg = normalizar_config_runtime( diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/main.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/main.py index 12bb59441..66ec85bd5 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/main.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/main.py @@ -11,7 +11,7 @@ os.environ.setdefault("OMP_WAIT_POLICY", "PASSIVE") os.environ.setdefault("KMP_BLOCKTIME", "0") # Limita kernels Numba, caso o Visual Worker utilize Numba # direta ou indiretamente. -os.environ.setdefault("NUMBA_NUM_THREADS", "4") +os.environ.setdefault("NUMBA_NUM_THREADS", "16") def main(): @@ -21,12 +21,16 @@ def main(): # Importar e configurar OpenCV antes dos módulos do Visual Worker, # pois eles podem carregar OpenCV internamente. import cv2 - cv2.setNumThreads(2) + opencv_threads = max(1, int(os.environ.get("WEED_OPENCV_THREADS", "8"))) + cv2.setNumThreads(opencv_threads) cv2.ocl.setUseOpenCL(False) + print( + f"[weed][PERF] OpenCV threads requested={opencv_threads} " + f"effective={cv2.getNumThreads()} preview_cache=True" + ) from weed_worker.config import mostrar_log, get_camera_manager, iniciar_camera_manager from shared.enums import WeedWorkerCommandType, TipoFrameCamera - from shared.utils import encode_image_base64 from shared.contexto_global_redis import ContextoGlobalRedis, CmdKey, CtxKey def loop_ativo(): @@ -65,16 +69,28 @@ def main(): get_camera_manager().atualizar_saude_camera() elif acao == WeedWorkerCommandType.GetCameraFrame: tipo = TipoFrameCamera(dados.get("params", TipoFrameCamera.Rgb.value)) - frame = get_camera_manager().get_selected_frame(tipo) - if frame is not None: - base64_img = encode_image_base64(frame) - if base64_img is not None: - resposta = { - "frame": base64_img, - "timestamp": time.time(), - "tipo": tipo.value - } - ContextoGlobalRedis.publicar_comando(CmdKey.WeedWorkerTx, { "cmd": WeedWorkerCommandType.GetCameraFrame.value, "params": resposta }) + + # Leitura barata: JPEG/Base64 já foi produzido pelo loop de preview. + # Este comando nunca monta imagem, nunca roda OpenCV e nunca infere. + payload = get_camera_manager().get_cached_preview_payload(tipo) + if payload is not None and payload.get("base64"): + resposta = { + "frame": payload["base64"], + # Mantém compatibilidade com o C#: timestamp da RESPOSTA + # precisa ser posterior ao instante em que ele enviou o request. + "timestamp": time.time(), + "frame_timestamp": payload.get("source_ts", 0.0), + "tipo": tipo.value, + "width": payload.get("width", 0), + "height": payload.get("height", 0), + } + ContextoGlobalRedis.publicar_comando( + CmdKey.WeedWorkerTx, + { + "cmd": WeedWorkerCommandType.GetCameraFrame.value, + "params": resposta, + }, + ) elif acao == WeedWorkerCommandType.SaveCameraFrames: nome = dados.get("params", {}).get("nome", "") pasta = dados.get("params", {}).get("caminho", "frames_salvos") diff --git a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/weed_detector.py b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/weed_detector.py index 394d09fb6..166718d68 100644 --- a/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/weed_detector.py +++ b/AgroBase/AgroBase/bin/x64/Debug/Python/Scripts/workers/weed_worker/weed_detector.py @@ -42,10 +42,15 @@ class WeedDetector: CONTRATO_OFICIAL = "target_binary" TARGET_ID = 1 - def __init__(self): - from weed_worker.config import load_seg_config + def __init__(self, config: Optional[dict] = None): + # O CameraManager já mantém um snapshot de configuração. Quando ele é + # fornecido, não existe motivo para o detector reler Redis durante a + # construção. Mantemos o fallback legado para usos isolados/testes. + if config is None: + from weed_worker.config import load_seg_config + config = load_seg_config() - self.config = load_seg_config() + self.config = dict(config or {}) self.prediction_contract = str( self.config.get("prediction_contract", self.CONTRATO_OFICIAL) @@ -120,6 +125,12 @@ class WeedDetector: self._morf_cache_k: Optional[int] = None self._morf_kernel = None + # Geometria discreta da grade depende somente de H/W, quantidade de + # bicos e quantidade de células. Evita reconstruir linspace/áreas em + # todo frame. O cache é invalidado automaticamente quando a geometria + # muda, pois _inicializar_estado() é chamado nesses casos. + self._grid_geometry_cache = {} + # Tempo/movimento. self._last_update_ts: Optional[float] = None self._distancia_deslocada_total_cm = 0.0 @@ -305,11 +316,19 @@ class WeedDetector: # mas aqui tratamos como velocidade em m/s. vel_norm = vel_mps + # O contrato operacional é SEMPRE binário, inclusive quando a + # cabeça ONNX selecionada é semantic: o SegFormerService converte + # a classe semantic_target_class escolhida para 0/1 antes daqui. + # Construímos a máscara uma única vez e a reutilizamos no radar e + # na memória espacial. + mask_target = (predictions == self.TARGET_ID) + t_rad0 = time.perf_counter() target_no_radar_frame, estat_target = self._decidir_target_no_radar( predictions, cfg, vel_norm=vel_norm, + mask_target=mask_target, ) t_rad1 = time.perf_counter() @@ -319,6 +338,7 @@ class WeedDetector: cfg=cfg, vel_mps=vel_mps, agora=agora, + mask_target=mask_target, ) t_bic1 = time.perf_counter() @@ -377,6 +397,7 @@ class WeedDetector: predictions: np.ndarray, cfg: dict, vel_norm: float = 0.0, + mask_target: Optional[np.ndarray] = None, ): on_global = float(cfg.get("min_frac_erva_global_on", 0.0020)) off_global = float(cfg.get("min_frac_erva_global_off", 0.0015)) @@ -393,7 +414,12 @@ class WeedDetector: h, w = predictions.shape[:2] inv_total = 1.0 / max(1, h * w) - target_sum = int(self._is_target[predictions].sum()) + if mask_target is None: + mask_target = (predictions == self.TARGET_ID) + + # count_nonzero evita indexação por LUT + array temporário adicional + # quando a máscara já foi montada pelo ciclo principal. + target_sum = int(np.count_nonzero(mask_target)) frac_global = target_sum * inv_total alpha = float(cfg.get("erva_frac_ema", 0.30)) @@ -430,6 +456,7 @@ class WeedDetector: cfg: dict, vel_mps: float, agora: float, + mask_target: Optional[np.ndarray] = None, ): qtd_bicos = self.qtd_bicos @@ -460,6 +487,7 @@ class WeedDetector: frac_grid, grid_info = self._calcular_frac_grid_por_bico_cell( predictions=predictions, cfg=cfg, + mask_target=mask_target, ) self._frac_grid_last = frac_grid @@ -657,69 +685,122 @@ class WeedDetector: return shifted + def _get_grid_geometry(self, h: int, w: int): + key = (int(h), int(w), int(self.qtd_bicos), int(self.num_cells)) + cached = self._grid_geometry_cache.get(key) + if cached is not None: + return cached + + x_edges = np.linspace(0, w, self.qtd_bicos + 1, dtype=np.int32) + y_edges = np.linspace(0, h, self.num_cells + 1, dtype=np.int32) + x_widths = np.diff(x_edges).astype(np.int32, copy=False) + y_heights = np.diff(y_edges).astype(np.int32, copy=False) + areas = ( + y_heights[:, None].astype(np.int64) + * x_widths[None, :].astype(np.int64) + ) + + cached = { + "x_edges": x_edges, + "y_edges": y_edges, + "x_widths": x_widths, + "y_heights": y_heights, + "areas": areas, + "valid_cell_count": int(np.count_nonzero(y_heights > 0)), + "reduceat_safe": bool( + np.all(x_widths > 0) + and np.all(y_heights > 0) + ), + } + + # Uma única geometria é usada no runtime normal. Limita crescimento + # acidental caso algum caller varie resolução em teste. + if len(self._grid_geometry_cache) >= 4: + self._grid_geometry_cache.clear() + self._grid_geometry_cache[key] = cached + return cached + def _calcular_frac_grid_por_bico_cell( self, predictions: np.ndarray, cfg: dict, + mask_target: Optional[np.ndarray] = None, ): h, w = predictions.shape[:2] - mask_target = self._is_target[predictions] + if mask_target is None: + mask_target = (predictions == self.TARGET_ID) + mask_target = self._aplicar_filtros_opcionais( mask_target=mask_target, cfg=cfg, ) - # Integral image em inteiro com sinal. - # Evita overflow nos cálculos A - B - C + D. - ii = np.pad( - mask_target.astype(np.int64, copy=False) - .cumsum(axis=0, dtype=np.int64) - .cumsum(axis=1, dtype=np.int64), - ((1, 0), (1, 0)), - mode="constant", - constant_values=0, - ) + geom = self._get_grid_geometry(h, w) + x_edges = geom["x_edges"] + y_edges = geom["y_edges"] + areas = geom["areas"] - x_edges = np.linspace(0, w, self.qtd_bicos + 1, dtype=np.int32) - y_edges = np.linspace(0, h, self.num_cells + 1, dtype=np.int32) + if geom["reduceat_safe"]: + # Soma retangular em duas reduções segmentadas. Mantém exatamente + # os mesmos limites discretos de np.linspace usados pela versão + # anterior, mas evita integral int64 HxW + 700 loops Python. + # bool -> reduceat produz contagem inteira sem alterar a máscara. + sums_y = np.add.reduceat( + mask_target, + y_edges[:-1], + axis=0, + ) + counts_yx = np.add.reduceat( + sums_y, + x_edges[:-1], + axis=1, + ) - frac_grid = np.zeros( - (self.qtd_bicos, self.num_cells), - dtype=np.float32, - ) + frac_cells_bicos = np.divide( + counts_yx, + areas, + out=np.zeros_like(areas, dtype=np.float32), + where=areas > 0, + ).astype(np.float32, copy=False) - valid_cell_count = 0 - - for c in range(self.num_cells): - y_top = int(y_edges[c]) - y_bot = int(y_edges[c + 1]) - - if y_bot <= y_top: - continue - - valid_cell_count += 1 - cell_h = y_bot - y_top - - for b in range(self.qtd_bicos): - x0 = int(x_edges[b]) - x1 = int(x_edges[b + 1]) - - if x1 <= x0: + frac_grid = np.ascontiguousarray( + frac_cells_bicos.T, + dtype=np.float32, + ) + else: + # Fallback para geometrias patológicas em que há bins vazios + # (ex.: mais células que pixels). Preserva a semântica antiga. + ii = np.pad( + mask_target.astype(np.int64, copy=False) + .cumsum(axis=0, dtype=np.int64) + .cumsum(axis=1, dtype=np.int64), + ((1, 0), (1, 0)), + mode="constant", + constant_values=0, + ) + frac_grid = np.zeros( + (self.qtd_bicos, self.num_cells), + dtype=np.float32, + ) + for c in range(self.num_cells): + y_top = int(y_edges[c]) + y_bot = int(y_edges[c + 1]) + if y_bot <= y_top: continue - - area = float(cell_h * (x1 - x0)) - if area <= 0: - continue - - total = ( - ii[y_bot, x1] - - ii[y_top, x1] - - ii[y_bot, x0] - + ii[y_top, x0] - ) - - frac_grid[b, c] = float(total) / area + for b in range(self.qtd_bicos): + x0 = int(x_edges[b]) + x1 = int(x_edges[b + 1]) + area = int((y_bot - y_top) * (x1 - x0)) + if area <= 0: + continue + total = ( + ii[y_bot, x1] + - ii[y_top, x1] + - ii[y_bot, x0] + + ii[y_top, x0] + ) + frac_grid[b, c] = np.float32(float(total) / float(area)) # A segmentação é naturalmente da esquerda para a direita. # Quando habilitado, inverte associação região imagem -> bico físico. @@ -727,12 +808,17 @@ class WeedDetector: frac_grid = frac_grid[::-1, :].copy() return frac_grid, { - "valid_cell_count": int(valid_cell_count), + "valid_cell_count": int(geom["valid_cell_count"]), "height": int(h), "width": int(w), "x_edges": x_edges, "y_edges": y_edges, "zona_coord": "frac_top_to_bottom", + "grid_backend": ( + "reduceat_segmented" + if geom["reduceat_safe"] + else "integral_fallback" + ), } def _atualizar_score_memoria( diff --git a/AgroBase/OperationControl/Services/GpsService.cs b/AgroBase/OperationControl/Services/GpsService.cs index 58802628d..1e67c0765 100644 --- a/AgroBase/OperationControl/Services/GpsService.cs +++ b/AgroBase/OperationControl/Services/GpsService.cs @@ -129,7 +129,7 @@ namespace OperationControl.Services public int NtripPort { get; set; } = 2101; public string NtripMountpoint { get; set; } = "EESC0"; public string NtripUsername { get; set; } = "Zendion"; // Environment.GetEnvironmentVariable("AGRO_NTRIP_USERNAME") ?? string.Empty; - public string NtripPassword { get; set; } = "c3pc7*9N"; //Environment.GetEnvironmentVariable("AGRO_NTRIP_PASSWORD") ?? string.Empty; + public string NtripPassword { get; set; } = "kY3zd$*5"; //Environment.GetEnvironmentVariable("AGRO_NTRIP_PASSWORD") ?? string.Empty; private GnssExpectedRole _expectedRole = GnssExpectedRole.PreserveCurrentConfiguration; private BaseFixedConfiguration _lastBaseConfiguration; diff --git a/Python/OAK/datasets/oak-fcc-3/calibration/mp_ar0234/module_params.json b/Python/OAK/datasets/oak-fcc-3/calibration/mp_ar0234/module_params.json index 1e0b60a6c..46c1dc6c7 100644 --- a/Python/OAK/datasets/oak-fcc-3/calibration/mp_ar0234/module_params.json +++ b/Python/OAK/datasets/oak-fcc-3/calibration/mp_ar0234/module_params.json @@ -722,10 +722,11 @@ }, "flatfield_config": { "enabled": true, - "npz_file": "calibration/flatfield_maps_v1.npz", + "npz_file": "flatfield_maps_v1.npz", "apply_before_fusion": true, "apply_after_decode": true, - "apply_space": "native_camera_space", + "saturation_guard_enabled": false, + "apply_space": "final_tensor_space", "map_type": "gain", "channels": [ "R", @@ -758,7 +759,7 @@ }, "subtract_dark": false, "clip_output": true, - "json_file": "calibration/flatfield_maps_v1.json", + "json_file": "flatfield_maps_v1.json", "schema": "multispec_flatfield_production_v2", "created_at": "2026-09-08 16:31:07" }, diff --git a/Python/OAK/datasets/oak-fcc-3/calibration/mp_ov9782/module_params.json b/Python/OAK/datasets/oak-fcc-3/calibration/mp_ov9782/module_params.json index 9c22d7c4e..04f834a17 100644 --- a/Python/OAK/datasets/oak-fcc-3/calibration/mp_ov9782/module_params.json +++ b/Python/OAK/datasets/oak-fcc-3/calibration/mp_ov9782/module_params.json @@ -848,8 +848,8 @@ "created_at": "2026-05-08 15:26:18", "json_file": "flatfield_maps_v1.json", "npz_file": "flatfield_maps_v1.npz", - "apply_before_fusion": false, - "apply_after_decode": false, + "apply_before_fusion": true, + "apply_after_decode": true, "apply_space": "final_tensor_space", "map_type": "gain", "formula": "channel_corrected = max(channel_linear - dark, 0) * gain_map", diff --git a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py index d3c50e6ea..16ad1aee7 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py +++ b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_client.py @@ -45,6 +45,7 @@ class OakFcc3Client: imu_modo="rotation_vector", imu_freq_hz=200, evaluate_quality=True, + require_product_contract=False, hardware_sync_enabled=None, frame_sync_master=None, @@ -65,6 +66,30 @@ class OakFcc3Client: self.module_calibration_json = module_calibration_json self.module_params = self._load_module_params(module_calibration_json) self.fusion_config = self.module_params.get("fusion_config", {}) or {} + self.require_product_contract = bool(require_product_contract) + + assembly = self.module_params.get("assembly_metadata", {}) or {} + self.product_contract = bool( + self.module_params.get("schema") == "multispec_module_params_v3" + and assembly.get("schema") == "multispec_module_params_assembly_v1" + ) + + if self.require_product_contract and not self.product_contract: + raise RuntimeError( + "OakFcc3Client exige module_params de produção homologado: " + f"schema={self.module_params.get('schema')!r} " + f"assembly={assembly.get('schema')!r}" + ) + + # Em produto, Bayer e raster RGB nativo vêm do MP, não de fallbacks do caller. + mp_bayer = str(self.module_params.get("bayer_pattern", "") or "").upper() + if mp_bayer: + self.bayer = mp_bayer + + rgb_size = (self.module_params.get("sensor_size_by_role", {}) or {}).get("rgb") + if isinstance(rgb_size, (list, tuple)) and len(rgb_size) == 2: + self.width = int(rgb_size[0]) + self.height = int(rgb_size[1]) self.imu_modo = str(imu_modo).strip().lower() self.imu_freq_hz = int(imu_freq_hz) @@ -84,14 +109,15 @@ class OakFcc3Client: self.svc = OakFcc3Service( timeout=10, fps=fps, - width=width, - height=height, + width=self.width, + height=self.height, frame_type=frame_type, output_dtype=output_dtype, capture_mode=capture_mode, raw_policy=raw_policy, mx_id=self.mx_id, module_calibration_json=module_calibration_json, + require_product_contract=self.require_product_contract, imu_modo=self.imu_modo, imu_freq_hz=self.imu_freq_hz, @@ -108,15 +134,15 @@ class OakFcc3Client: self.applied_camera_controls = {} self.radiometric_controller = None self.core = RawProcessorCore( - sensor_width=width, - sensor_height=height, - bayer_pattern=bayer, + sensor_width=self.width, + sensor_height=self.height, + bayer_pattern=self.bayer, calibration_json_path=module_calibration_json, ) self.preview = RawProcessorPreview( - sensor_width=width, - sensor_height=height, - bayer_pattern=bayer, + sensor_width=self.width, + sensor_height=self.height, + bayer_pattern=self.bayer, ) def __enter__(self): @@ -133,6 +159,46 @@ class OakFcc3Client: with open(path, "r", encoding="utf-8") as f: return json.load(f) + def get_contract(self): + """ + Contrato estático/runtime consumido por CameraMultispectral. + + O module_params é autoridade de hardware/calibração. O target final + pode ser null no contrato novo; quando existir é apenas um default + compatível/legado. O Weed CameraManager passa o target ONNX ao caller. + """ + try: + service_contract = self.svc.get_contract() or {} + except Exception: + service_contract = {} + + mp = self.module_params or {} + fusion = mp.get("fusion_config", {}) or {} + target = fusion.get("target_size") + default_target = None + if isinstance(target, (list, tuple)) and len(target) == 2: + tw, th = int(target[0]), int(target[1]) + if tw > 0 and th > 0: + default_target = [tw, th] + + sensor_sizes = mp.get("sensor_size_by_role", {}) or {} + camera_hw = mp.get("camera_hardware", {}) or {} + + out = dict(service_contract) + out.update({ + "product_contract": bool(self.product_contract), + "require_product_contract": bool(self.require_product_contract), + "module_params_schema": mp.get("schema"), + "module_calibration_json": self.module_calibration_json, + "sensor_size_by_role": sensor_sizes, + "camera_hardware": camera_hw, + "bayer_pattern": mp.get("bayer_pattern") or out.get("bayer_pattern"), + "frame_type": self.frame_type, + "raw_policy": self.raw_policy, + "default_target_size": default_target, + }) + return out + def apply_module_camera_settings(self): camera_settings = self.module_params.get("camera_settings", {}) or {} @@ -374,8 +440,8 @@ class OakFcc3Client: evaluate_quality = self.evaluate_quality evaluate_quality = bool(evaluate_quality) - tensor = self.core.fuse_multispec_cameras(decoded, meta, channels_expected) - tensor = self.core.resize_tensor_chw(tensor, target_size=target_size) + tensor = self.core.fuse_multispec_cameras(decoded, meta, channels_expected, target_size=target_size) + #tensor = self.core.resize_tensor_chw(tensor, target_size=target_size) # Mantém paridade com build_infer_tensor_from_stream: se a calibração # habilitar patch normalization, ela também vale no caminho decoded. diff --git a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py index 3588af9bc..16ad1aee7 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py +++ b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_manager.py @@ -1,156 +1,149 @@ +import numpy as np +import cv2 import json import os -import time -from collections import deque -import threading -import copy -import cv2 -import depthai as dai -import numpy as np - -from itertools import product +from .oak_fcc3_service import OakFcc3Service +from .raw_processor_core import RawProcessorCore +from .raw_processor_preview import RawProcessorPreview +from .radiometric_controller import RadiometricController -class OakFcc3Manager: - """ - Manager OAK-FFC-3 com dois fluxos principais: +PHYSICAL_CHANNEL_NAMES = ("R", "G", "B", "RE", "NIR") +PHYSICAL_CHANNEL_COUNT = len(PHYSICAL_CHANNEL_NAMES) - 1) RAW_BRUTO - - Mantém o comportamento antigo. - - CAM_A/CAM_B/CAM_C enviam RAW10 packed direto para o PC. - - O PC faz decode, flat/radiometric, homografia/fusão/crop/resize. - 2) MULTISPEC - - Câmeras sempre em 800p nativo. - - OAK aplica homografia/crop/resize via ImageManip. - - PC recebe frames já alinhados: - CAM_A/rgb -> BGR uint8 - CAM_B/re -> GRAY uint8 - CAM_C/nir -> GRAY uint8 - - O Client deve montar o tensor sem reaplicar homografia. - """ - - SENSOR_W = 1280 - SENSOR_H = 800 +class OakFcc3Client: + @staticmethod + def _validate_physical_channel_count(channels_expected): + try: + count = int(channels_expected) + except (TypeError, ValueError) as exc: + raise RuntimeError( + f"channels_expected inválido: {channels_expected!r}" + ) from exc + if count != PHYSICAL_CHANNEL_COUNT: + raise RuntimeError( + "OakFcc3Client transporta somente Raw5 físico " + f"{list(PHYSICAL_CHANNEL_NAMES)}; recebido channels_expected={count}. " + "Canais derivados pertencem ao serviço de inferência." + ) + return count def __init__( self, - fps=30, width=640, height=400, + bayer="BGGR", + fps=30, frame_type="RAW_BRUTO", output_dtype="uint8", capture_mode="AUTO", raw_policy="allow_single", - roles=None, - sync_mode=None, - hardware_sync_enabled=None, - frame_sync_master=None, - sync_tolerance_ms=None, - buffer_size=None, - only_camera=None, - mx_id=None, module_calibration_json=None, - module_params=None, + mx_id=None, imu_modo="rotation_vector", imu_freq_hz=200, + evaluate_quality=True, + require_product_contract=False, + + hardware_sync_enabled=None, + frame_sync_master=None, + sync_mode=None, + sync_tolerance_ms=None, + buffer_size=None, + + **kwargs, ): + self.width = width + self.height = height + self.bayer = bayer self.fps = fps - - # Para compatibilidade, mantemos width/height. - # No RAW_BRUTO isso não muda o sensor, pois usamos 800p fixo. - # No MULTISPEC isso representa a saída final alinhada da OAK. - self.width = int(width) - self.height = int(height) - self.size = (self.width, self.height) - - self.sensor_width = self.SENSOR_W - self.sensor_height = self.SENSOR_H - - self.frame_type = str(frame_type).upper() + self.frame_type = frame_type self.output_dtype = output_dtype self.capture_mode = capture_mode self.raw_policy = raw_policy - self.only_camera = only_camera + self.module_calibration_json = module_calibration_json + self.module_params = self._load_module_params(module_calibration_json) + self.fusion_config = self.module_params.get("fusion_config", {}) or {} + self.require_product_contract = bool(require_product_contract) - self.roles = roles or { - "CAM_A": "rgb", - "CAM_B": "re", - "CAM_C": "nir", - } + assembly = self.module_params.get("assembly_metadata", {}) or {} + self.product_contract = bool( + self.module_params.get("schema") == "multispec_module_params_v3" + and assembly.get("schema") == "multispec_module_params_assembly_v1" + ) - self.mx_id = str(mx_id) if mx_id else None - self.dev_info = None - self.device = None - self.pipeline = None - self.queues = {} - self.buffers = {} - self.camera_info = {} - - self.imu_modo = self._validate_imu_modo(imu_modo) - self.imu_freq_hz = int(imu_freq_hz) - - if self.imu_freq_hz <= 0: - raise ValueError( - f"imu_freq_hz deve ser maior que zero: {self.imu_freq_hz}" + if self.require_product_contract and not self.product_contract: + raise RuntimeError( + "OakFcc3Client exige module_params de produção homologado: " + f"schema={self.module_params.get('schema')!r} " + f"assembly={assembly.get('schema')!r}" ) - self.imu_sensor_type = None + # Em produto, Bayer e raster RGB nativo vêm do MP, não de fallbacks do caller. + mp_bayer = str(self.module_params.get("bayer_pattern", "") or "").upper() + if mp_bayer: + self.bayer = mp_bayer - self.has_imu_pipeline = False - self.tem_imu = False - self.q_imu = None + rgb_size = (self.module_params.get("sensor_size_by_role", {}) or {}).get("rgb") + if isinstance(rgb_size, (list, tuple)) and len(rgb_size) == 2: + self.width = int(rgb_size[0]) + self.height = int(rgb_size[1]) - self.running = False - self.frame_id = 0 + self.imu_modo = str(imu_modo).strip().lower() + self.imu_freq_hz = int(imu_freq_hz) - # Serializa start/stop e leituras nativas de telemetria. - # Evita getChipTemperature/getUsbSpeed concorrendo com device.close(). - self._device_lock = threading.RLock() - self.control_queues = {} - self.camera_controls = { - cam_id: self._default_controls_for_role(role) - for cam_id, role in self.roles.items() - } - self._last_raw_dims = {} + # Auditoria radiométrica completa do Raw5. + # + # True mantém o comportamento histórico e é útil para captura científica, + # normalize/auditoria e ferramentas offline. + # + # No runtime em tempo real deve ficar False: evaluate_frame_quality() + # calcula estatísticas/percentis pesados e não faz parte da montagem + # necessária para a inferência. + self.evaluate_quality = bool(evaluate_quality) - self.module_calibration_json = module_calibration_json - self.module_params = module_params if isinstance(module_params, dict) else self._load_module_params(module_calibration_json) - self.fusion_config = (self.module_params or {}).get("fusion_config", {}) or {} - self.aligned_geometry = None + self.mx_id = str(mx_id) if mx_id else None - sync_cfg = (self.module_params.get("capture_synchronization", {}) if isinstance(self.module_params, dict) else {}) - if hardware_sync_enabled is None: hardware_sync_enabled = sync_cfg.get("hardware_sync_enabled", False) - if frame_sync_master is None: frame_sync_master = sync_cfg.get("frame_sync_master", "CAM_A") - if sync_mode is None: sync_mode = sync_cfg.get("software_sync_mode", "best") - if sync_tolerance_ms is None: sync_tolerance_ms = sync_cfg.get("sync_tolerance_ms", 12.0) - if buffer_size is None: buffer_size = sync_cfg.get("buffer_size", 8) - self.hardware_sync_enabled = bool(hardware_sync_enabled) - self.frame_sync_master = str(frame_sync_master) - self.sync_mode = str(sync_mode) - self.sync_tolerance_ms = float(sync_tolerance_ms) - self.buffer_size = int(buffer_size) + self.svc = OakFcc3Service( + timeout=10, + fps=fps, + width=self.width, + height=self.height, + frame_type=frame_type, + output_dtype=output_dtype, + capture_mode=capture_mode, + raw_policy=raw_policy, + mx_id=self.mx_id, + module_calibration_json=module_calibration_json, + require_product_contract=self.require_product_contract, - self.async_capture_enabled = True - self.async_capture_mode = "latest" # latest | queue - self.async_capture_max_queue = 2 - self._capture_thread = None - self._capture_stop_event = threading.Event() - self._capture_lock = threading.RLock() - self._capture_cond = threading.Condition(self._capture_lock) - self._latest_packet = None - self._latest_packet_seq = 0 - self._last_consumed_packet_seq = 0 - self._packet_queue = deque(maxlen=self.async_capture_max_queue) - self._capture_thread_stats = { - "started": False, - "packets": 0, - "dropped_latest": 0, - "dropped_queue": 0, - "last_error": None, - "last_loop_ms": 0.0, - } + imu_modo=self.imu_modo, + imu_freq_hz=self.imu_freq_hz, + + hardware_sync_enabled=hardware_sync_enabled, + frame_sync_master=frame_sync_master, + sync_mode=sync_mode, + sync_tolerance_ms=sync_tolerance_ms, + buffer_size=buffer_size, + + **kwargs, + ) + + self.applied_camera_controls = {} + self.radiometric_controller = None + self.core = RawProcessorCore( + sensor_width=self.width, + sensor_height=self.height, + bayer_pattern=self.bayer, + calibration_json_path=module_calibration_json, + ) + self.preview = RawProcessorPreview( + sensor_width=self.width, + sensor_height=self.height, + bayer_pattern=self.bayer, + ) def __enter__(self): self.start() @@ -159,23 +152,6 @@ class OakFcc3Manager: def __exit__(self, exc_type, exc, tb): self.stop() - # ============================================================ - # Modes - # ============================================================ - - def _is_preview_mode(self): - return str(self.frame_type).upper() == "PREVIEW" - - def _is_multispec_mode(self): - return str(self.frame_type).upper() == "MULTISPEC" - - def _is_raw_mode(self): - return str(self.frame_type).upper() == "RAW_BRUTO" - - # ============================================================ - # Config helpers - # ============================================================ - def _load_module_params(self, path): if not path or not os.path.isfile(path): return {} @@ -183,2022 +159,679 @@ class OakFcc3Manager: with open(path, "r", encoding="utf-8") as f: return json.load(f) - def _default_controls_for_role(self, role: str): - role = str(role).lower() - - if role == "rgb": - return { - "ae_enable": False, - "awb_enable": False, - "exposure_time_us": 2000, - "analogue_gain": 1.0, - "colour_gains": [1.0, 1.0], - } - - return { - "ae_enable": False, - "awb_enable": False, - "exposure_time_us": 5000, - "analogue_gain": 1.0, - "colour_gains": None, - } - - # ============================================================ - # Device discovery - # ============================================================ - - def list_cameras(self): - dev_info = self._resolve_device_info() - - with dai.Device(dev_info) as dev: - result = [] - for f in dev.getConnectedCameraFeatures(): - result.append({ - "socket": f.socket.name, - "sensor": f.sensorName, - "role": self.roles.get(f.socket.name, "unknown"), - }) - return result - - def _device_id_from_info(self, dev_info): - for name in ("getMxId", "getDeviceId"): - try: - fn = getattr(dev_info, name, None) - if callable(fn): - value = fn() - if value: - return str(value) - except Exception: - pass + def get_contract(self): + """ + Contrato estático/runtime consumido por CameraMultispectral. + O module_params é autoridade de hardware/calibração. O target final + pode ser null no contrato novo; quando existir é apenas um default + compatível/legado. O Weed CameraManager passa o target ONNX ao caller. + """ try: - value = getattr(dev_info, "mxid", None) - if value: - return str(value) + service_contract = self.svc.get_contract() or {} except Exception: - pass - - try: - value = getattr(dev_info, "deviceId", None) - if value: - return str(value) - except Exception: - pass - - return None - - def _resolve_device_info(self): - devices = dai.Device.getAllAvailableDevices() - - if not devices: - raise RuntimeError("Nenhum dispositivo DepthAI/OAK encontrado.") - - if self.mx_id is None: - return devices[0] - - target = str(self.mx_id).strip() - - for dev_info in devices: - dev_id = self._device_id_from_info(dev_info) - if dev_id == target: - return dev_info - - disponiveis = [ - self._device_id_from_info(d) or str(getattr(d, "name", "unknown")) - for d in devices - ] - - raise RuntimeError( - f"Dispositivo DepthAI com ID '{target}' não encontrado. " - f"Disponíveis: {disponiveis}" - ) - - def _socket_from_name(self, socket_name): - socket_name = str(socket_name).upper() - if socket_name == "CAM_A": - return dai.CameraBoardSocket.CAM_A - if socket_name == "CAM_B": - return dai.CameraBoardSocket.CAM_B - if socket_name == "CAM_C": - return dai.CameraBoardSocket.CAM_C - if socket_name == "CAM_D": - return dai.CameraBoardSocket.CAM_D - raise ValueError(f"Socket não suportado: {socket_name}") - - # ============================================================ - # Pipeline creation, classic RAW/PREVIEW - # ============================================================ - - def _create_camera_node_classic(self, socket, sensor_name: str, role: str): - """ - Fluxo clássico. - - RGB/OV9782: - ColorCamera raw para RAW_BRUTO. - - MONO/OV9282: - MonoCamera raw quando disponível. - """ - sensor_name_u = str(sensor_name or "").upper() - role_u = str(role or "").lower() - - is_rgb = ( - role_u == "rgb" - or "OV9782" in sensor_name_u - or socket == dai.CameraBoardSocket.CAM_A - ) - - if is_rgb: - cam = self.pipeline.createColorCamera() - cam.setBoardSocket(socket) - - try: - cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_800_P) - except Exception: - try: - cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P) - except Exception: - pass - - cam.setInterleaved(False) - cam.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB) - cam.setFps(float(self.fps)) - - if self._is_preview_mode(): - try: - cam.setVideoSize(int(self.width), int(self.height)) - return cam, cam.video - except Exception: - return cam, cam.preview - - return cam, cam.raw - - mono = self.pipeline.create(dai.node.MonoCamera) - mono.setBoardSocket(socket) - - try: - mono.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P) - except Exception: - try: - mono.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P) - except Exception: - try: - mono.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P) - except Exception: - pass - - mono.setFps(float(self.fps)) - - if self._is_preview_mode(): - return mono, mono.out - - if hasattr(mono, "raw"): - return mono, mono.raw - - return mono, mono.out - - def _create_imu_node(self, pipeline): - self.has_imu_pipeline = False - self.imu_sensor_type = None - - try: - imu = pipeline.create(dai.node.IMU) - - if self.imu_modo == "raw_6axis": - imu.enableIMUSensor( - dai.IMUSensor.ACCELEROMETER_RAW, - self.imu_freq_hz, - ) - imu.enableIMUSensor( - dai.IMUSensor.GYROSCOPE_RAW, - self.imu_freq_hz, - ) - - self.imu_sensor_type = ( - "ACCELEROMETER_RAW_GYROSCOPE_RAW" - ) - - elif self.imu_modo == "rotation_vector": - imu.enableIMUSensor( - dai.IMUSensor.GAME_ROTATION_VECTOR, - self.imu_freq_hz, - ) - - self.imu_sensor_type = "GAME_ROTATION_VECTOR" - - else: - # Proteção adicional; normalmente o __init__ já impede isso. - raise RuntimeError( - f"imu_modo não suportado: {self.imu_modo!r}" - ) - - imu.setBatchReportThreshold(1) - imu.setMaxBatchReports(5) - - xout_imu = pipeline.create(dai.node.XLinkOut) - xout_imu.setStreamName("imu") - imu.out.link(xout_imu.input) - - self.has_imu_pipeline = True - - print( - "[OAK] Pipeline IMU criado" - f" | modo={self.imu_modo}" - f" | sensor={self.imu_sensor_type}" - f" | freq={self.imu_freq_hz}Hz" - ) - - except Exception as e: - self.has_imu_pipeline = False - self.imu_sensor_type = None - print( - "[WARN] IMU indisponível no pipeline" - f" | modo={self.imu_modo}" - f" | freq={self.imu_freq_hz}Hz" - f" | erro={e}" - ) - - def _validate_imu_modo(self, modo): - modo = str(modo).strip().lower() - - aliases = { - "raw": "raw_6axis", - "raw_6_axis": "raw_6axis", - "raw_6axis": "raw_6axis", - "rotation": "rotation_vector", - "game_rotation_vector": "rotation_vector", - "rotation_vector": "rotation_vector", - } - - modo_normalizado = aliases.get(modo) - - if modo_normalizado is None: - raise ValueError( - f"imu_modo inválido: {modo!r}. " - "Use 'raw_6axis' ou 'rotation_vector'." - ) - - return modo_normalizado - - # ============================================================ - # Pipeline creation, MULTISPEC aligned on OAK - # ============================================================ - - def _aplicar_frame_sync(self, cam, cam_id): - cam_id_normalizado = str(cam_id).strip().upper() - master = str(self.frame_sync_master).strip().upper() - - if not self.hardware_sync_enabled: - print( - f"[OAK FSYNC] cam={cam_id_normalizado} " - f"habilitado=False modo=DISABLED" - ) - return - - cameras_validas = {"CAM_A", "CAM_B", "CAM_C"} - - if master not in cameras_validas: - raise ValueError( - f"frame_sync_master inválido: {self.frame_sync_master}. " - f"Esperado: CAM_A, CAM_B ou CAM_C." - ) - - if cam_id_normalizado not in cameras_validas: - print( - f"[OAK FSYNC] Câmera ignorada: " - f"cam_id={cam_id_normalizado}" - ) - return - - if cam_id_normalizado == master: - modo = dai.CameraControl.FrameSyncMode.OUTPUT - nome_modo = "OUTPUT" - else: - modo = dai.CameraControl.FrameSyncMode.INPUT - nome_modo = "INPUT" - - cam.initialControl.setFrameSyncMode(modo) - - print( - f"[OAK FSYNC] " - f"cam={cam_id_normalizado} " - f"master={master} " - f"habilitado=True " - f"modo={nome_modo}" - ) - - def _create_color_camera_multispec(self, socket): - cam = self.pipeline.create(dai.node.ColorCamera) - cam.setBoardSocket(socket) - cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_800_P) - cam.setFps(float(self.fps)) - cam.setInterleaved(False) - - # Usamos BGR porque getCvFrame/OpenCV lida direto com BGR. - # O Client converte para RGB float no _frame_to_float01. - cam.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR) - cam.setVideoSize(int(self.sensor_width), int(self.sensor_height)) - return cam - - def _create_mono_camera_multispec(self, socket): - cam = self.pipeline.create(dai.node.MonoCamera) - cam.setBoardSocket(socket) - cam.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P) - cam.setFps(float(self.fps)) - return cam - - def _make_xout(self, name): - if hasattr(dai.node, "XLinkOut"): - xout = self.pipeline.create(dai.node.XLinkOut) - xout.setStreamName(name) - return xout - - if hasattr(self.pipeline, "createXLinkOut"): - xout = self.pipeline.createXLinkOut() - xout.setStreamName(name) - return xout - - raise RuntimeError("Não encontrei XLinkOut nesta versão do DepthAI.") - - def _apply_four_point_transform(self, config, src_quad_px, dst_quad_px): - src_pts = [dai.Point2f(float(x), float(y)) for x, y in src_quad_px] - dst_pts = [dai.Point2f(float(x), float(y)) for x, y in dst_quad_px] - - if hasattr(config, "setWarpTransformFourPoints"): - try: - config.setWarpTransformFourPoints(src_pts, dst_pts, False) - return - except TypeError: - config.setWarpTransformFourPoints(src_pts, False) - return - - if hasattr(config, "addTransformFourPoints"): - config.addTransformFourPoints(src_pts, dst_pts, False) - return - - raise RuntimeError( - "ImageManipConfig não tem setWarpTransformFourPoints nem addTransformFourPoints" - ) - - def _create_warp_manip( - self, - name, - out_w, - out_h, - src_quad_px, - frame_type=None, - max_output_frame_size=None, - ): - manip = self.pipeline.create(dai.node.ImageManip) - - dst_quad_px = [ - (0.0, 0.0), - (float(out_w - 1), 0.0), - (float(out_w - 1), float(out_h - 1)), - (0.0, float(out_h - 1)), - ] - - self._apply_four_point_transform( - manip.initialConfig, - src_quad_px, - dst_quad_px, - ) - - if hasattr(manip.initialConfig, "setResize"): - try: - manip.initialConfig.setResize(int(out_w), int(out_h)) - except Exception: - pass - - if frame_type is not None: - try: - manip.initialConfig.setFrameType(frame_type) - except Exception: - pass - - if max_output_frame_size is None: - max_output_frame_size = int(out_w * out_h * 3) - - manip.setMaxOutputFrameSize(int(max_output_frame_size)) - - xout = self._make_xout(name) - manip.out.link(xout.input) - - return manip, xout - - def _start_multispec_pipeline(self, features): - """ - Cria pipeline onde a OAK entrega frames já alinhados. - Stream names continuam CAM_A/CAM_B/CAM_C para preservar o contrato. - """ - self.aligned_geometry = self._prepare_multispec_geometry() - - feature_by_socket = {f.socket.name: f for f in features} - - required = ["CAM_A", "CAM_B", "CAM_C"] - missing = [cam_id for cam_id in required if cam_id not in feature_by_socket] - if missing: - raise RuntimeError( - f"MULTISPEC exige CAM_A/CAM_B/CAM_C ativos. Ausentes: {missing}" - ) - - for cam_id in required: - if self.only_camera is not None and cam_id != self.only_camera: + service_contract = {} + + mp = self.module_params or {} + fusion = mp.get("fusion_config", {}) or {} + target = fusion.get("target_size") + default_target = None + if isinstance(target, (list, tuple)) and len(target) == 2: + tw, th = int(target[0]), int(target[1]) + if tw > 0 and th > 0: + default_target = [tw, th] + + sensor_sizes = mp.get("sensor_size_by_role", {}) or {} + camera_hw = mp.get("camera_hardware", {}) or {} + + out = dict(service_contract) + out.update({ + "product_contract": bool(self.product_contract), + "require_product_contract": bool(self.require_product_contract), + "module_params_schema": mp.get("schema"), + "module_calibration_json": self.module_calibration_json, + "sensor_size_by_role": sensor_sizes, + "camera_hardware": camera_hw, + "bayer_pattern": mp.get("bayer_pattern") or out.get("bayer_pattern"), + "frame_type": self.frame_type, + "raw_policy": self.raw_policy, + "default_target_size": default_target, + }) + return out + + def apply_module_camera_settings(self): + camera_settings = self.module_params.get("camera_settings", {}) or {} + + applied = {} + + for role, settings in camera_settings.items(): + if not isinstance(settings, dict): continue - f = feature_by_socket[cam_id] - role = str(self.roles.get(cam_id, "unknown")).lower() - socket = f.socket - - print( - f"[OAK] Criando câmera MULTISPEC {cam_id} " - f"sensor={f.sensorName} role={role}" - ) - - if role == "rgb": - cam = self._create_color_camera_multispec(socket) - src_output = cam.video - quad = self.aligned_geometry["quad_rgb"] - out_type = dai.ImgFrame.Type.BGR888p - max_size = self.width * self.height * 3 - channels = 3 - bit_depth = 8 - raw_format = "BGR888p" - elif role == "re": - cam = self._create_mono_camera_multispec(socket) - src_output = cam.out - quad = self.aligned_geometry["quad_re"] - out_type = dai.ImgFrame.Type.GRAY8 - max_size = self.width * self.height - channels = 1 - bit_depth = 8 - raw_format = "GRAY8" - elif role == "nir": - cam = self._create_mono_camera_multispec(socket) - src_output = cam.out - quad = self.aligned_geometry["quad_nir"] - out_type = dai.ImgFrame.Type.GRAY8 - max_size = self.width * self.height - channels = 1 - bit_depth = 8 - raw_format = "GRAY8" - else: - raise RuntimeError(f"Role não suportada no MULTISPEC: cam_id={cam_id}, role={role}") - - self.apply_initial_camera_controls_to_node(cam, cam_id) - self._aplicar_frame_sync(cam, cam_id) - - xin_ctrl = self.pipeline.create(dai.node.XLinkIn) - xin_ctrl.setStreamName(f"{cam_id}_ctrl") - xin_ctrl.out.link(cam.inputControl) - - manip, _ = self._create_warp_manip( - name=cam_id, - out_w=self.width, - out_h=self.height, - src_quad_px=quad, - frame_type=out_type, - max_output_frame_size=max_size, - ) - src_output.link(manip.inputImage) - - self.queues[cam_id] = None - self.buffers[cam_id] = deque(maxlen=self.buffer_size) - self.control_queues[cam_id] = None - - self.camera_info[cam_id] = { - "id": cam_id, - "socket": cam_id, - "sensor": f.sensorName, - "role": role, - "interface": "OAK_ALIGNED", - "raw_format": raw_format, - "channels": channels, - "bit_depth": bit_depth, - "width": int(self.width), - "height": int(self.height), - "aligned_by_oak": True, - "homography_applied": role in ("re", "nir"), - "crop_resize_applied": True, - } - - # ============================================================ - # Homography helpers for MULTISPEC - # ============================================================ - - def _prepare_multispec_geometry(self): - fusion = self.fusion_config or {} - - if str(fusion.get("alignment_mode", "homography")).lower() != "homography": - raise RuntimeError( - "frame_type=MULTISPEC na OAK exige fusion_config.alignment_mode='homography'." - ) - - homographies = fusion.get("homographies", {}) or {} - H_re_raw = homographies.get("re_to_rgb") - H_nir_raw = homographies.get("nir_to_rgb") - - if H_re_raw is None or H_nir_raw is None: - raise RuntimeError( - "module_params precisa conter fusion_config.homographies.re_to_rgb e nir_to_rgb " - "para frame_type=MULTISPEC." - ) - - calib_size = fusion.get("homography_calibration_size", None) - runtime_size = (self.sensor_width, self.sensor_height) - - H_re = self._scale_homography_to_runtime(H_re_raw, calib_size, runtime_size) - H_nir = self._scale_homography_to_runtime(H_nir_raw, calib_size, runtime_size) - - crop_box = self._compute_common_crop_box( - self.sensor_width, - self.sensor_height, - H_re, - H_nir, - ) - - quad_rgb = self._identity_quad_for_crop(crop_box) - quad_re = self._quad_for_output_crop_to_input(H_re, crop_box) - quad_nir = self._quad_for_output_crop_to_input(H_nir, crop_box) - - quad_rgb = self._clamp_quad(quad_rgb, self.sensor_width, self.sensor_height) - quad_re = self._clamp_quad(quad_re, self.sensor_width, self.sensor_height) - quad_nir = self._clamp_quad(quad_nir, self.sensor_width, self.sensor_height) - - print("============================================") - print("[OAK MULTISPEC] Geometria alinhada na OAK") - print(f"sensor : {self.sensor_width}x{self.sensor_height}") - print(f"output : {self.width}x{self.height}") - print(f"crop_box RGB : {crop_box}") - print(f"quad_rgb : {quad_rgb}") - print(f"quad_re : {quad_re}") - print(f"quad_nir : {quad_nir}") - print("============================================") - - return { - "reference": "rgb", - "mode": "homography", - "sensor_size": [int(self.sensor_width), int(self.sensor_height)], - "output_size": [int(self.width), int(self.height)], - "crop_box": [int(v) for v in crop_box], - "H_re": H_re, - "H_nir": H_nir, - "quad_rgb": quad_rgb, - "quad_re": quad_re, - "quad_nir": quad_nir, - } - - def _scale_homography_to_runtime(self, H, calib_size, runtime_size): - H = np.asarray(H, dtype=np.float32) - - if calib_size is None: - if abs(H[2, 2]) > 1e-9: - H = H / H[2, 2] - return H.astype(np.float32) - - calib_w, calib_h = calib_size - runtime_w, runtime_h = runtime_size - - calib_w = float(calib_w) - calib_h = float(calib_h) - runtime_w = float(runtime_w) - runtime_h = float(runtime_h) - - if calib_w <= 0 or calib_h <= 0: - return H.astype(np.float32) - - sx = runtime_w / calib_w - sy = runtime_h / calib_h - - S = np.array( - [ - [sx, 0.0, 0.0], - [0.0, sy, 0.0], - [0.0, 0.0, 1.0], - ], - dtype=np.float32, - ) - - S_inv = np.array( - [ - [1.0 / sx, 0.0, 0.0], - [0.0, 1.0 / sy, 0.0], - [0.0, 0.0, 1.0], - ], - dtype=np.float32, - ) - - H_runtime = S @ H @ S_inv - - if abs(H_runtime[2, 2]) > 1e-9: - H_runtime = H_runtime / H_runtime[2, 2] - - return H_runtime.astype(np.float32) - - def _warp_mask(self, mask, H, out_w, out_h): - return cv2.warpPerspective( - mask, - H, - (out_w, out_h), - flags=cv2.INTER_NEAREST, - borderMode=cv2.BORDER_CONSTANT, - borderValue=0, - ) - - def _compute_common_crop_box(self, runtime_w, runtime_h, H_re, H_nir): - base = np.ones((runtime_h, runtime_w), dtype=np.uint8) * 255 - - rgb_mask = base - re_mask = self._warp_mask(base, H_re, runtime_w, runtime_h) - nir_mask = self._warp_mask(base, H_nir, runtime_w, runtime_h) - - common = (rgb_mask > 0) & (re_mask > 0) & (nir_mask > 0) - - ys, xs = np.where(common) - if xs.size == 0 or ys.size == 0: - raise RuntimeError("Área comum vazia. Verifique as homografias.") - - x0 = int(xs.min()) - x1 = int(xs.max()) + 1 - y0 = int(ys.min()) - y1 = int(ys.max()) + 1 - - return x0, y0, x1, y1 - - def _apply_H_to_point(self, H, x, y): - p = np.array([float(x), float(y), 1.0], dtype=np.float32) - q = H @ p - if abs(q[2]) < 1e-9: - return float(q[0]), float(q[1]) - return float(q[0] / q[2]), float(q[1] / q[2]) - - def _quad_for_output_crop_to_input(self, H_src_to_rgb, crop_box): - x0, y0, x1, y1 = crop_box - - dst_corners_rgb = [ - (x0, y0), - (x1, y0), - (x1, y1), - (x0, y1), - ] - - H_inv = np.linalg.inv(H_src_to_rgb).astype(np.float32) - - src_quad = [] - for x, y in dst_corners_rgb: - sx, sy = self._apply_H_to_point(H_inv, x, y) - src_quad.append((sx, sy)) - - return src_quad - - def _identity_quad_for_crop(self, crop_box): - x0, y0, x1, y1 = crop_box - return [ - (float(x0), float(y0)), - (float(x1), float(y0)), - (float(x1), float(y1)), - (float(x0), float(y1)), - ] - - def _clamp_quad(self, quad, w, h): - out = [] - for x, y in quad: - x = max(0.0, min(float(w - 1), float(x))) - y = max(0.0, min(float(h - 1), float(y))) - out.append((x, y)) - return out - - # ============================================================ - # Start / stop - # ============================================================ - - def start(self): - with self._device_lock: - return self._start_impl() - - def _start_impl(self): - if self.running: - return - - thread_anterior = self._capture_thread - - if thread_anterior is not None and thread_anterior.is_alive(): - raise RuntimeError( - "Não é seguro iniciar novo pipeline: " - "thread OakFcc3AsyncCapture anterior ainda está viva." - ) - - try: - self.dev_info = self._resolve_device_info() - self.mx_id = self._device_id_from_info(self.dev_info) or self.mx_id - - self.device = dai.Device(self.dev_info) - self.pipeline = dai.Pipeline() - - features = self.device.getConnectedCameraFeatures() - - self.queues.clear() - self.buffers.clear() - self.camera_info.clear() - self.control_queues.clear() - self._last_raw_dims.clear() - self.aligned_geometry = None - - if self._is_multispec_mode(): - self._start_multispec_pipeline(features) - else: - for f in features: - socket = f.socket - socket_name = socket.name - if self.only_camera is not None and socket_name != self.only_camera: - continue - - role = self.roles.get(socket_name, "unknown") - - print( - f"[OAK] Criando câmera {socket_name} " - f"sensor={f.sensorName} role={role}" - ) - - cam, output = self._create_camera_node_classic( - socket=socket, - sensor_name=f.sensorName, - role=role, - ) - - self.apply_initial_camera_controls_to_node(cam, socket_name) - self._aplicar_frame_sync(cam, socket_name) - - xin_ctrl = self.pipeline.create(dai.node.XLinkIn) - xin_ctrl.setStreamName(f"{socket_name}_ctrl") - xin_ctrl.out.link(cam.inputControl) - - xout = self.pipeline.create(dai.node.XLinkOut) - xout.setStreamName(socket_name) - output.link(xout.input) - - cam_id = socket_name - - self.queues[cam_id] = None - self.buffers[cam_id] = deque(maxlen=self.buffer_size) - self.control_queues[cam_id] = None - - self.camera_info[cam_id] = { - "id": cam_id, - "socket": socket_name, - "sensor": f.sensorName, - "role": role, - } - - self._validate_capture_mode() - self._create_imu_node(self.pipeline) - self.device.startPipeline(self.pipeline) - - self.q_imu = None - self.tem_imu = False - - if self.has_imu_pipeline: - try: - self.q_imu = self.device.getOutputQueue( - name="imu", - maxSize=8, - blocking=False, - ) - self.tem_imu = True - print("[OAK] Fila IMU criada | maxSize=8") - except Exception as e: - self.q_imu = None - self.tem_imu = False - print(f"[WARN] Fila IMU indisponível: {e}") - - for cam_id in self.camera_info.keys(): - self.queues[cam_id] = self.device.getOutputQueue( - name=cam_id, - maxSize=self.buffer_size, - blocking=False, - ) - - self.control_queues[cam_id] = self.device.getInputQueue( - name=f"{cam_id}_ctrl", - maxSize=4, - blocking=False, - ) - - self.running = True - - if not hasattr(self, "async_capture_enabled"): - self.async_capture_enabled = True - if not hasattr(self, "async_capture_mode"): - self.async_capture_mode = "latest" - if not hasattr(self, "async_capture_max_queue"): - self.async_capture_max_queue = 2 - - self._reset_async_capture_state() - self._start_async_capture_thread() - time.sleep(0.05) - - if not self.running: - erro = self._capture_thread_stats.get("last_error") - raise RuntimeError( - "Pipeline DepthAI caiu imediatamente após iniciar. " - f"Último erro: {erro}" - ) - - except Exception: - # _device_lock é RLock, então a limpeza pode reutilizar stop(). try: - self.stop() - except Exception: - pass - raise + resp = self.svc.apply_camera_controls( + role=role, + controls=settings, + ) + applied[role] = resp - def stop(self): - with self._device_lock: - return self._stop_impl() - - def _stop_impl(self): - # Impede novas capturas imediatamente. - self.running = False - - thread_ok = self._stop_async_capture_thread() - - try: - if self.pipeline is not None and hasattr(self.pipeline, "stop"): - self.pipeline.stop() - except Exception: - pass - - try: - if self.device is not None: - self.device.close() - except Exception: - pass - - self.pipeline = None - self.device = None - self.dev_info = None - - self.queues.clear() - self.buffers.clear() - self.camera_info.clear() - self.control_queues.clear() - self._last_raw_dims.clear() - - self.aligned_geometry = None - self.q_imu = None - self.tem_imu = False - self.has_imu_pipeline = False - - try: - with self._capture_cond: - self._latest_packet = None - self._packet_queue.clear() - self._capture_cond.notify_all() - except Exception: - pass - - return bool(thread_ok is not False) - - def _is_fatal_depthai_error(self, erro): - txt = str(erro).lower() - - sinais = [ - "x_link_error", - "communication exception", - "couldn't read data from stream", - "couldn't open stream", - "device already closed", - "device has been closed", - "x_link_device_not_found", - "failed to find device", - "no available devices", - "nenhum dispositivo depthai", - "device crashed", - ] - - return any(s in txt for s in sinais) - - # ============================================================ - # Status - # ============================================================ - - def get_device_metrics(self): - """Lê métricas nativas sem concorrer com start/stop/device.close().""" - with self._device_lock: - dev = self.device - - if dev is None or not self.running: - return { + except Exception as e: + applied[role] = { "ok": False, - "running": bool(self.running), - "error": "device indisponível", + "error": str(e), + "requested": settings, } - out = { - "ok": True, - "running": True, - "timestamp": time.time(), - } + self.applied_camera_controls = applied + return applied - try: - speed = dev.getUsbSpeed() - out["usb_speed"] = getattr(speed, "name", str(speed)) - except Exception as e: - out["usb_error"] = str(e) + def enable_radiometric_controller(self): + self.radiometric_controller = RadiometricController( + client=self, + config_json_path=self.module_calibration_json, + ) + + self.radiometric_controller.sync_from_camera_controls(self.applied_camera_controls) + self.radiometric_controller.sync_from_actual_camera_controls() + + return self.radiometric_controller - try: - temp = dev.getChipTemperature() - for attr in ("average", "css", "mss", "upa", "dss"): - if hasattr(temp, attr): - out[f"temperature_{attr}_c"] = float(getattr(temp, attr)) - except Exception as e: - out["temperature_error"] = str(e) - - for nome, metodo in ( - ("ddr", "getDdrMemoryUsage"), - ("cmx", "getCmxMemoryUsage"), - ): - try: - uso = getattr(dev, metodo)() - out[f"{nome}_used_bytes"] = int(getattr(uso, "used", 0)) - out[f"{nome}_total_bytes"] = int(getattr(uso, "total", 0)) - except Exception as e: - out[f"{nome}_error"] = str(e) - - for nome, metodo in ( - ("leon_css", "getLeonCssCpuUsage"), - ("leon_mss", "getLeonMssCpuUsage"), - ): - try: - uso = getattr(dev, metodo)() - out[f"{nome}_average"] = float(getattr(uso, "average", uso)) - except Exception as e: - out[f"{nome}_error"] = str(e) - - out["async_capture"] = self.get_async_capture_status() - return out - - def get_status(self): - with self._device_lock: - return { - "mx_id": self.mx_id, - "backend": "oak_fcc3", - "running": bool(self.running), - "fps": self.fps, - "width": self.width, - "height": self.height, - "sensor_width": self.sensor_width, - "sensor_height": self.sensor_height, - "frame_type": self.frame_type, - "output_dtype": self.output_dtype, - "capture_mode": self.capture_mode, - "raw_policy": self.raw_policy, - "sync_tolerance_ms": self.sync_tolerance_ms, - "buffer_size": self.buffer_size, - "geometry_stage": "oak" if self._is_multispec_mode() else "pc", - "aligned_geometry": self._serializable_aligned_geometry(), - "cameras": list(self.camera_info.values()), - "async_capture": self.get_async_capture_status(), - "tem_imu": bool(getattr(self, "tem_imu", False)), - "has_imu_pipeline": bool(getattr(self, "has_imu_pipeline", False)), - "imu_modo": self.imu_modo, - "imu_freq_hz": self.imu_freq_hz, - "imu_sensor_type": self.imu_sensor_type, - } - - def _serializable_aligned_geometry(self): - if not isinstance(self.aligned_geometry, dict): + def update_radiometry(self, decoded, meta=None): + if self.radiometric_controller is None: return None - out = {} - for k, v in self.aligned_geometry.items(): - if isinstance(v, np.ndarray): - out[k] = v.tolist() - else: - out[k] = v - return out + return self.radiometric_controller.update(decoded, meta) - # ============================================================ - # Frame capture - # ============================================================ + def get_current_camera_controls(self): + controls = {} - def get_next_frame(self, timeout=1.0): - if not self.running: - last_error = None + for role in ("rgb", "re", "nir"): try: - last_error = self._capture_thread_stats.get("last_error") - except Exception: - pass - - raise RuntimeError( - f"OakFcc3Manager não está rodando. Último erro: {last_error}" - ) - - # Fallback síncrono se desligar async. - if not bool(getattr(self, "async_capture_enabled", True)): - return self._get_next_frame_sync_instrumented(timeout=timeout) - - t0 = time.perf_counter() - deadline = t0 + float(timeout) - - with self._capture_cond: - while time.perf_counter() < deadline: - packet = None - - mode = str(getattr(self, "async_capture_mode", "latest")).lower() - - if mode == "queue": - if len(self._packet_queue) > 0: - packet = self._packet_queue.popleft() - else: - latest = self._latest_packet - if latest is not None and int(latest.get("seq", 0)) > int(self._last_consumed_packet_seq): - packet = latest - - if packet is not None: - seq = int(packet.get("seq", 0)) - self._last_consumed_packet_seq = seq - - frames = packet["frames"] - meta = dict(packet["meta"]) - - age_ms = (time.perf_counter() - float(packet.get("created_perf_counter", time.perf_counter()))) * 1000.0 - get_wait_ms = (time.perf_counter() - t0) * 1000.0 - - cp = dict(meta.get("capture_perf", {}) or {}) - cp["async_consumer"] = True - cp["async_packet_seq"] = seq - cp["async_packet_age_ms"] = float(age_ms) - cp["async_get_wait_ms"] = float(get_wait_ms) - cp["async_status"] = self.get_async_capture_status() - meta["capture_perf"] = cp - - return frames, meta - - remaining = deadline - time.perf_counter() - if remaining <= 0: - break - - self._capture_cond.wait(timeout=min(0.005, remaining)) - - raise TimeoutError( - f"Timeout aguardando pacote assíncrono do OAK-FFC-3. " - f"status={self.get_async_capture_status()}" - ) - - def _get_next_frame_sync_instrumented(self, timeout=1.0): - if not self.running: - raise RuntimeError("OakFcc3Manager não está rodando. Chame start() primeiro.") - - perf = self._new_capture_perf() if hasattr(self, "_new_capture_perf") else None - t_start_wall = time.time() - t_start = time.perf_counter() - - while time.time() - t_start_wall < timeout: - if perf is not None: - perf["loop_count"] += 1 - perf["buffer_lengths_before_sync"] = self._buffer_lengths_snapshot() - - t0 = time.perf_counter() - self._drain_queues_to_buffers(perf=perf) - if perf is not None: - perf["drain_total_ms"] += (time.perf_counter() - t0) * 1000.0 - perf["buffer_lengths_after_drain"] = self._buffer_lengths_snapshot() - - t0 = time.perf_counter() - synced = self._try_get_synced_packet(perf=perf) - if perf is not None: - perf["sync_select_ms"] += (time.perf_counter() - t0) * 1000.0 - - if synced is not None: - frames, timestamps, sync_dt_ms, sync_ok, frame_controls = synced - self.frame_id += 1 - - t0 = time.perf_counter() - meta = self._build_meta(frames, timestamps, sync_dt_ms, sync_ok, frame_controls=frame_controls) - if perf is not None: - perf["meta_ms"] += (time.perf_counter() - t0) * 1000.0 - perf["wait_total_ms"] = (time.perf_counter() - t_start) * 1000.0 - perf["sync_dt_ms"] = float(sync_dt_ms) - perf["sync_ok"] = bool(sync_ok) - perf["buffer_lengths_after_sync"] = self._buffer_lengths_snapshot() - perf["wait_reason"] = "synced_packet_ready_sync" - meta["capture_perf"] = perf - - return frames, meta - - t0 = time.perf_counter() - time.sleep(0.001) - if perf is not None: - perf["sleep_ms"] += (time.perf_counter() - t0) * 1000.0 - perf["sleep_count"] += 1 - - raise TimeoutError( - f"Timeout aguardando pacote sincronizado do OAK-FFC-3. " - f"Tolerância atual={self.sync_tolerance_ms} ms." - ) - - def _encontrar_melhor_tripleta(self, required_cam_ids): - listas = [ - list(self.buffers[cam_id]) - for cam_id in required_cam_ids - ] - - melhor_selecao = None - melhor_score = None - - for combinacao in product(*listas): - timestamps = [ - item["timestamp"] - for item in combinacao - ] - - menor_ts = min(timestamps) - maior_ts = max(timestamps) - spread_ms = (maior_ts - menor_ts) * 1000.0 - - # Primeiro prioriza menor dispersão. - # Em empate, prefere o pacote mais recente. - score = ( - spread_ms, - -menor_ts, - ) - - if melhor_score is None or score < melhor_score: - melhor_score = score - melhor_selecao = { - cam_id: item - for cam_id, item in zip(required_cam_ids, combinacao) + controls[role] = self.svc.get_camera_controls(role=role) + except Exception as e: + controls[role] = { + "ok": False, + "role": role, + "error": str(e), } - return melhor_selecao - - - def _extract_frame_controls(self, msg): - controls = { - "exposure_time_us": None, - "sensitivity_iso": None, - "analogue_gain_est": None, - "color_temperature_k": None, - "lens_position": None, - "sequence_num": None, - "errors": [], - } - - try: - if hasattr(msg, "getSequenceNum"): - controls["sequence_num"] = int(msg.getSequenceNum()) - except Exception as e: - controls["errors"].append(f"sequence_num:{type(e).__name__}:{e}") - - try: - if hasattr(msg, "getExposureTime"): - exp = msg.getExposureTime() - - if hasattr(exp, "total_seconds"): - controls["exposure_time_us"] = int(exp.total_seconds() * 1_000_000) - else: - controls["exposure_time_us"] = int(exp) - else: - controls["errors"].append("missing:getExposureTime") - except Exception as e: - controls["errors"].append(f"exposure:{type(e).__name__}:{e}") - - try: - if hasattr(msg, "getSensitivity"): - iso = msg.getSensitivity() - controls["sensitivity_iso"] = int(iso) - controls["analogue_gain_est"] = float(iso) / 100.0 - else: - controls["errors"].append("missing:getSensitivity") - except Exception as e: - controls["errors"].append(f"sensitivity:{type(e).__name__}:{e}") - - try: - if hasattr(msg, "getColorTemperature"): - ct = int(msg.getColorTemperature()) - controls["color_temperature_k"] = ct if ct > 0 else None - else: - controls["errors"].append("missing:getColorTemperature") - except Exception as e: - controls["errors"].append(f"color_temperature:{type(e).__name__}:{e}") - - try: - if hasattr(msg, "getLensPosition"): - lp = int(msg.getLensPosition()) - controls["lens_position"] = lp - else: - controls["errors"].append("missing:getLensPosition") - except Exception as e: - controls["errors"].append(f"lens_position:{type(e).__name__}:{e}") - - if not controls["errors"]: - controls.pop("errors", None) - return controls - def _drain_queues_to_buffers(self, perf=None): - for cam_id, q in self.queues.items(): - if perf is not None: - perf["drained_by_cam"].setdefault(cam_id, 0) - perf["queue_has_true_by_cam"].setdefault(cam_id, 0) + def get_radiometric_last_result(self): + if self.radiometric_controller is None: + return None - while True: - t0_has = self._cap_now_ms() - has_msg = q.has() - if perf is not None: - perf["drain_has_ms"] += self._cap_now_ms() - t0_has + return self.radiometric_controller.last_result - if not has_msg: - break + def start(self, print_debug=False): + self.svc.connect() - if perf is not None: - perf["queue_has_true_by_cam"][cam_id] += 1 + resp = self.svc.begin( + frame_type=self.frame_type, + output_dtype=self.output_dtype, + capture_mode=self.capture_mode, + ) - t0_get = self._cap_now_ms() - msg = q.get() - if perf is not None: - perf["drain_get_msg_ms"] += self._cap_now_ms() - t0_get - - t0_ts = self._cap_now_ms() - try: - ts_start = msg.getTimestampDevice( - dai.CameraExposureOffset.START - ).total_seconds() - - ts_end = msg.getTimestampDevice( - dai.CameraExposureOffset.END - ).total_seconds() - - except Exception: - ts_start = msg.getTimestamp().total_seconds() - ts_end = ts_start - - ts = ts_start - if perf is not None: - perf["drain_get_timestamp_ms"] += self._cap_now_ms() - t0_ts - - if self._is_preview_mode() or self._is_multispec_mode(): - t0_data = self._cap_now_ms() - frame = msg.getCvFrame() - if perf is not None: - perf["drain_get_data_ms"] += self._cap_now_ms() - t0_data - - if frame is None: - continue - - if self._is_multispec_mode(): - self._last_raw_dims[cam_id] = { - "sensor_width": int(frame.shape[1]), - "sensor_height": int(frame.shape[0]), - "stride": int(frame.strides[0]) if hasattr(frame, "strides") else int(frame.shape[1]), - "packed_width": int(frame.shape[1]), - } - - else: - t0_data = self._cap_now_ms() - data = msg.getData() - if perf is not None: - perf["drain_get_data_ms"] += self._cap_now_ms() - t0_data - - t0_copy = self._cap_now_ms() - raw = np.frombuffer(data, dtype=np.uint8).copy() - if perf is not None: - perf["drain_frombuffer_copy_ms"] += self._cap_now_ms() - t0_copy - - t0_shape = self._cap_now_ms() - h = int(msg.getHeight()) - w = int(msg.getWidth()) - stride = self._get_imgframe_stride(msg, raw.size, h, w) - - expected = h * stride - - if raw.size < expected: - raise RuntimeError( - f"RAW menor que esperado: raw.size={raw.size}, esperado={expected}, " - f"w={w}, h={h}, stride={stride}" - ) - - frame = raw[:expected].reshape((h, stride)) - if perf is not None: - perf["drain_reshape_ms"] += self._cap_now_ms() - t0_shape - - self._last_raw_dims[cam_id] = { - "sensor_width": w, - "sensor_height": h, - "stride": stride, - "packed_width": stride, - } - - t0_ctrl = self._cap_now_ms() - frame_controls = self._extract_frame_controls(msg) - if perf is not None: - perf["drain_controls_ms"] += self._cap_now_ms() - t0_ctrl - - self.buffers[cam_id].append({ - "frame": frame, - "timestamp": ts_start, - "timestamp_start": ts_start, - "timestamp_end": ts_end, - "controls": frame_controls, - }) - - if perf is not None: - perf["drained_total"] += 1 - perf["drained_by_cam"][cam_id] += 1 - - def _get_imgframe_stride(self, msg, raw_size: int, h: int, w: int) -> int: try: - return int(msg.getStride()) + self.mx_id = self.svc.manager.mx_id except Exception: pass + applied = self.apply_module_camera_settings() + + if print_debug: + print("[OAK CLIENT] START:", resp) + print("[OAK CLIENT] APPLIED CAMERA SETTINGS:", applied) + + self.enable_radiometric_controller() + + return resp + + def stop(self): + # disconnect() já para o manager. Evita manager.stop() duplicado. + return self.svc.disconnect() + + def get_device_metrics(self): + return self.svc.get_device_metrics() + + def get_status(self): + return self.svc.get_status() + + def get_next_raw_frame(self, timeout=1.0): + return self.svc.capture_frame(timeout=timeout) + + def get_next_frame(self, timeout=2.0): + frame, meta, _ = self.get_next_decoded(timeout=timeout) + return frame, meta + + def get_next_decoded(self, timeout=2.0): + raw_frame, raw_meta = self.get_next_raw_frame(timeout=timeout) + + frame_type = str(raw_meta.get("frame_type", self.frame_type)).upper() + meta = dict(raw_meta) + + if frame_type == "RAW_BRUTO": + decoded = self.decode_stream_cameras(raw_frame, raw_meta) + + self.update_radiometry(decoded, raw_meta) + + frame = raw_frame + return frame, meta, decoded + + elif frame_type == "RGB": + decoded = self.decode_stream_cameras(raw_frame, raw_meta) + + self.update_radiometry(decoded, raw_meta) + + frame = self.build_rgb_tensor(decoded) + meta["output_layout"] = "CHW" + meta["channels"] = ["R", "G", "B"] + meta["shape"] = list(frame.shape) + meta["dtype"] = str(frame.dtype) + + return frame, meta, decoded + + elif frame_type == "MULTISPEC": + decoded = self.decode_oak_aligned_multispec(raw_frame, raw_meta) + + self.update_radiometry(decoded, raw_meta) + + # Versão inicial segura: + # não chama core.fuse_multispec_cameras(), porque ali teria homografia de novo. + frame = self.build_multispec_tensor_from_oak_aligned(decoded, meta=raw_meta) + + meta["output_layout"] = "CHW" + meta["channels"] = ["R", "G", "B", "RE", "NIR"] + meta["shape"] = list(frame.shape) + meta["dtype"] = str(frame.dtype) + meta["aligned_by_oak"] = True + meta["geometry_stage"] = "oak" + + return frame, meta, decoded + + elif frame_type == "PREVIEW": + decoded = self.decode_stream_cameras(raw_frame, raw_meta) + frame = raw_frame + return frame, meta, decoded + + else: + raise RuntimeError(f"frame_type não suportado: {frame_type}") + + def get_next_tensor_preview(self, timeout=2.0): + frame, meta, decoded = self.get_next_decoded(timeout=timeout) + + frame_type = str(meta.get("frame_type", self.frame_type)).upper() + + if frame_type == "RGB": + rgb_hwc = np.transpose(frame[:3], (1, 2, 0)) + preview = self._rgb01_to_bgr(rgb_hwc) + return {"rgb_tensor": preview}, meta + + if frame_type == "MULTISPEC": + rgb_hwc = np.transpose(frame[:3], (1, 2, 0)) + re01 = frame[3] + nir01 = frame[4] + + return { + "rgb_tensor": self._rgb01_to_bgr(rgb_hwc), + "re_tensor": self._gray01_to_bgr(re01), + "nir_tensor": self._gray01_to_bgr(nir01), + }, meta + + else: + return self.build_visual_preview_from_raw(frame, meta), meta + + raise RuntimeError(f"frame_type não suportado para preview: {frame_type}") + + def get_next_preview(self, timeout=2.0): + raw_frame, raw_meta = self.get_next_raw_frame(timeout=timeout) + + meta = dict(raw_meta) + + previews = self.build_visual_preview_from_raw(raw_frame, meta) + + return previews, meta + + def get_last_patch_normalization_result(self): try: - if h > 0 and raw_size % h == 0: - return int(raw_size // h) + return self.core.last_patch_normalization_result except Exception: - pass - - return int(np.ceil(w * 5.0 / 4.0)) - - def _try_get_synced_packet(self, perf=None): - required_cam_ids = self._get_required_cam_ids() - - if not required_cam_ids: - if perf is not None: - perf["wait_reason"] = "no_required_cameras" return None - # Todas as câmeras precisam ter pelo menos um frame disponível. - for cam_id in required_cam_ids: - if cam_id not in self.buffers or len(self.buffers[cam_id]) == 0: - if perf is not None: - perf["wait_reason"] = f"empty_buffer:{cam_id}" - return None - - # Procura a melhor combinação entre todos os frames disponíveis. - selected = self._encontrar_melhor_tripleta(required_cam_ids) - - if not selected or len(selected) != len(required_cam_ids): - if perf is not None: - perf["wait_reason"] = "no_valid_selection" - return None - - timestamps = { - cam_id: item["timestamp"] - for cam_id, item in selected.items() - } - - frame_controls = { - cam_id: item.get("controls", {}) - for cam_id, item in selected.items() - } - - ts_values = list(timestamps.values()) - - sync_dt_ms = ( - (max(ts_values) - min(ts_values)) * 1000.0 - if len(ts_values) >= 2 - else 0.0 - ) - - sync_ok = sync_dt_ms <= self.sync_tolerance_ms - - if perf is not None: - perf["selected_ts_by_cam"] = { - cam_id: float(ts) - for cam_id, ts in timestamps.items() - } - - perf["selected_seq_by_cam"] = { - cam_id: item.get("controls", {}).get("sequence_num") - for cam_id, item in selected.items() - } - - perf["sync_dt_ms"] = float(sync_dt_ms) - perf["sync_ok"] = bool(sync_ok) - - modo_sync = str(self.sync_mode).strip().lower() - - # No modo estrito, nunca entrega uma tripleta fora da tolerância. - if not sync_ok and modo_sync == "strict": - #Remove o frame globalmente mais antigo entre as cabeças - #dos buffers. Frames futuros somente estarão mais distantes - #desse frame, então ele não conseguirá formar uma combinação - #melhor posteriormente. - oldest_cam_id = min( - required_cam_ids, - key=lambda cam_id: self.buffers[cam_id][0]["timestamp"] - ) - - dropped_item = self.buffers[oldest_cam_id].popleft() - - if perf is not None: - perf["wait_reason"] = f"strict_drop_oldest:{oldest_cam_id}" - perf["dropped_timestamp"] = float(dropped_item["timestamp"]) - - return None - - # Em best/best_effort, entrega a melhor combinação disponível, - # mesmo quando estiver fora da tolerância. - frames = { - cam_id: item["frame"] - for cam_id, item in selected.items() - } - - # Consome todos os frames anteriores e o próprio frame selecionado. - for cam_id, used_item in selected.items(): - while self.buffers[cam_id]: - item = self.buffers[cam_id].popleft() - - if item is used_item: - break - - if perf is not None: - perf["wait_reason"] = ( - "synced_selected" - if sync_ok - else "best_effort_selected_outside_tolerance" - ) - - return ( - frames, - timestamps, - sync_dt_ms, - sync_ok, - frame_controls, - ) - - def _get_available_cam_ids_ordered(self): - role_order = ["rgb", "re", "nir"] - available = list(self.queues.keys()) - - def sort_key(cam_id): - role = str(self.roles.get(cam_id, "unknown")).lower() - try: - return role_order.index(role) - except ValueError: - return 99 - - return sorted(available, key=sort_key) - - def _get_required_cam_ids(self): - available = self._get_available_cam_ids_ordered() - - if self._is_multispec_mode(): - # MULTISPEC precisa sempre do trio completo para montar [R,G,B,RE,NIR]. - return available[:3] - - if self.capture_mode == "SINGLE": - return available[:1] - - if self.capture_mode == "DOUBLE": - return available[:2] - - if self.capture_mode == "TRIPLE": - return available[:3] - - if self.capture_mode == "AUTO": - if self.raw_policy == "require_triple": - return available[:3] - return available - - return available - - # ============================================================ - # Meta - # ============================================================ - - def _build_meta(self, frames, timestamps, sync_dt_ms, sync_ok, frame_controls): - payload_sources = list(frames.keys()) - - shapes = { - cam_id: list(arr.shape) - for cam_id, arr in frames.items() - } - - dtypes = { - cam_id: str(arr.dtype) - for cam_id, arr in frames.items() - } - - camera_info = {} - for cam_id, info in self.camera_info.items(): - item = dict(info) - arr = frames.get(cam_id) - - if arr is not None: - item["shape"] = list(arr.shape) - item["dtype"] = str(arr.dtype) - - if self._is_multispec_mode(): - role = str(item.get("role", "")).lower() - item["interface"] = "OAK_ALIGNED" - item["raw_format"] = "BGR888p" if role == "rgb" else "GRAY8" - item["channels"] = 3 if arr.ndim == 3 else 1 - item["bit_depth"] = 8 if arr.dtype == np.uint8 else 16 - item["height"] = int(arr.shape[0]) - item["width"] = int(arr.shape[1]) - item["packed"] = False - item["aligned_by_oak"] = True - item["homography_applied"] = role in ("re", "nir") - item["crop_applied"] = True - item["resize_applied"] = True - item["color_order"] = "BGR" if role == "rgb" else None - - elif self._is_preview_mode(): - item["interface"] = "OAK" - item["channels"] = 3 if arr.ndim == 3 else 1 - item["bit_depth"] = 8 if arr.dtype == np.uint8 else 16 - item["height"] = int(arr.shape[0]) - item["width"] = int(arr.shape[1]) - - else: - raw_dims = self._last_raw_dims.get(cam_id, {}) - item["interface"] = "OAK_RAW" - item["raw_format"] = "RAW10_PACKED" - item["channels"] = 1 - item["bit_depth"] = 10 - item["height"] = int(raw_dims.get("sensor_height", arr.shape[0])) - item["width"] = int(raw_dims.get("sensor_width", arr.shape[1])) - item["stride"] = int(raw_dims.get("stride", arr.shape[1])) - item["packed_width"] = int(raw_dims.get("packed_width", arr.shape[1])) - item["shape"] = list(arr.shape) - item["dtype"] = str(arr.dtype) - item["packed"] = True - - camera_info[cam_id] = item - - meta = { - "frame_id": self.frame_id, - "backend": "oak_fcc3", - "frame_type": self.frame_type, - "capture_mode": self.capture_mode, - "output_dtype": self.output_dtype, - "dtype": self.output_dtype, - "payload_sources": payload_sources, - "camera_info": camera_info, - "timestamps": timestamps, - "frame_controls": frame_controls or {}, - "sync_dt_ms": sync_dt_ms, - "sync_ok": sync_ok, - "sync_tolerance_ms": self.sync_tolerance_ms, - "shapes": shapes, - "dtypes": dtypes, - "codec_name": "none", - "codec_family": "none", - "dt_comp": 0.0, - "dt_send_payload_prev": 0.0, - } - - if self._is_multispec_mode(): - meta.update({ - "output_layout": "dict_by_camera_aligned", - "geometry_stage": "oak", - "aligned_by_oak": True, - "fusion_alignment": self._serializable_aligned_geometry(), - }) - else: - meta.update({ - "output_layout": "dict_by_camera", - "geometry_stage": "pc", - "aligned_by_oak": False, - }) - - return meta - - # ============================================================ - # Validation - # ============================================================ - - def _validate_capture_mode(self): - n = len(self.queues) - - if self._is_multispec_mode() and n < 3: - raise RuntimeError(f"frame_type=MULTISPEC exige 3 câmeras, mas detectou {n}.") - - if self.capture_mode == "TRIPLE" and n < 3: - raise RuntimeError(f"CaptureMode TRIPLE exige 3 câmeras, mas detectou {n}.") - - if self.capture_mode == "DOUBLE" and n < 2: - raise RuntimeError(f"CaptureMode DOUBLE exige 2 câmeras, mas detectou {n}.") - - if self.raw_policy == "require_triple" and n < 3: - raise RuntimeError(f"raw_policy=require_triple exige 3 câmeras, mas detectou {n}.") - - # ============================================================ - # Camera controls - # ============================================================ - - def get_camera_controls(self, cam_id): - self._validate_cam_id_known(cam_id) - return dict(self.camera_controls.get(cam_id, {})) - - def set_ae_enable(self, cam_id, enable: bool): - self._validate_cam_id_running(cam_id) - - enable = bool(enable) - ctrl_state = self.camera_controls[cam_id] - ctrl_state["ae_enable"] = enable - - ctrl = dai.CameraControl() - - if enable: - if hasattr(ctrl, "setAutoExposureEnable"): - ctrl.setAutoExposureEnable() - else: - exp_us = int(ctrl_state.get("exposure_time_us") or 15000) - gain = float(ctrl_state.get("analogue_gain") or 1.0) - ctrl.setManualExposure(exp_us, self._gain_to_iso(gain)) - - self._send_control(cam_id, ctrl) - - return dict(ctrl_state) - - def set_awb_enable(self, cam_id, enable: bool): - self._validate_cam_id_running(cam_id) - - enable = bool(enable) - ctrl_state = self.camera_controls[cam_id] - ctrl_state["awb_enable"] = enable - - ctrl = dai.CameraControl() - - if hasattr(dai.CameraControl, "AutoWhiteBalanceMode"): - if enable: - ctrl.setAutoWhiteBalanceMode(dai.CameraControl.AutoWhiteBalanceMode.AUTO) - else: - ctrl.setAutoWhiteBalanceMode(dai.CameraControl.AutoWhiteBalanceMode.OFF) - - self._send_control(cam_id, ctrl) - - return dict(ctrl_state) - - def set_exposure_time(self, cam_id, exposure_time_us: int): - self._validate_cam_id_running(cam_id) - - ctrl_state = self.camera_controls[cam_id] - exposure_time_us = int(exposure_time_us) - exposure_time_us = max(1, exposure_time_us) - - ctrl_state["exposure_time_us"] = exposure_time_us - ctrl_state["ae_enable"] = False - - gain = float(ctrl_state.get("analogue_gain") or 1.0) - - ctrl = dai.CameraControl() - ctrl.setManualExposure(exposure_time_us, self._gain_to_iso(gain)) - - self._send_control(cam_id, ctrl) - - return dict(ctrl_state) - - def set_analogue_gain(self, cam_id, analogue_gain: float): - self._validate_cam_id_running(cam_id) - - ctrl_state = self.camera_controls[cam_id] - analogue_gain = float(analogue_gain) - analogue_gain = max(1.0, analogue_gain) - - ctrl_state["analogue_gain"] = analogue_gain - ctrl_state["ae_enable"] = False - - exposure_time_us = int(ctrl_state.get("exposure_time_us") or 15000) - - ctrl = dai.CameraControl() - ctrl.setManualExposure(exposure_time_us, self._gain_to_iso(analogue_gain)) - - self._send_control(cam_id, ctrl) - - return dict(ctrl_state) - - def set_colour_gains(self, cam_id, red_gain: float, blue_gain: float): - self._validate_cam_id_running(cam_id) - - ctrl_state = self.camera_controls[cam_id] - ctrl_state["colour_gains"] = [float(red_gain), float(blue_gain)] - ctrl_state["awb_enable"] = False - - ctrl = dai.CameraControl() - - if hasattr(ctrl, "setManualWhiteBalance"): - # Placeholder seguro. Algumas versões não expõem red/blue diretamente. - pass - - self._send_control(cam_id, ctrl) - - return dict(ctrl_state) - - def apply_camera_controls(self, cam_id, controls: dict): - self._validate_cam_id_running(cam_id) - - result = dict(self.camera_controls.get(cam_id, {})) - - if "ae_enable" in controls: - result = self.set_ae_enable(cam_id, bool(controls["ae_enable"])) - - if "awb_enable" in controls: - result = self.set_awb_enable(cam_id, bool(controls["awb_enable"])) - - ae_is_on = bool(self.camera_controls[cam_id].get("ae_enable", False)) - - if not ae_is_on: - if "exposure_time_us" in controls and controls["exposure_time_us"] is not None: - result = self.set_exposure_time(cam_id, int(controls["exposure_time_us"])) - - if "analogue_gain" in controls and controls["analogue_gain"] is not None: - result = self.set_analogue_gain(cam_id, float(controls["analogue_gain"])) - - else: - if "exposure_time_us" in controls and controls["exposure_time_us"] is not None: - self.camera_controls[cam_id]["exposure_time_us"] = int(controls["exposure_time_us"]) - - if "analogue_gain" in controls and controls["analogue_gain"] is not None: - self.camera_controls[cam_id]["analogue_gain"] = float(controls["analogue_gain"]) - - result = dict(self.camera_controls[cam_id]) - - return result - - def apply_initial_camera_controls_to_node(self, cam, cam_id): - ctrl_state = self.camera_controls.get(cam_id, {}) - - ae = bool(ctrl_state.get("ae_enable", False)) - exp_us = int(ctrl_state.get("exposure_time_us") or 15000) - gain = float(ctrl_state.get("analogue_gain") or 1.0) - - if not ae: - cam.initialControl.setManualExposure( - exp_us, - self._gain_to_iso(gain), - ) - - def _send_control(self, cam_id, ctrl): - if cam_id not in self.control_queues: - raise RuntimeError(f"Fila de controle não existe para {cam_id}") - - self.control_queues[cam_id].send(ctrl) - - def _validate_cam_id_known(self, cam_id): - if cam_id not in self.camera_controls: - raise ValueError(f"cam_id inválido: {cam_id}") - - def _validate_cam_id_running(self, cam_id): - self._validate_cam_id_known(cam_id) - - if not self.running: - raise RuntimeError("Manager não está rodando.") - - if cam_id not in self.control_queues: - raise RuntimeError(f"Câmera {cam_id} não está ativa no pipeline.") - - @staticmethod - def _gain_to_iso(gain: float) -> int: - gain = max(1.0, float(gain)) - iso = int(round(gain * 100)) - return max(100, min(1600, iso)) - - - - def _cap_now_ms(self): - return time.perf_counter() * 1000.0 - - def _new_capture_perf(self): - return { - "wait_total_ms": 0.0, - "drain_total_ms": 0.0, - "drain_has_ms": 0.0, - "drain_get_msg_ms": 0.0, - "drain_get_timestamp_ms": 0.0, - "drain_get_data_ms": 0.0, - "drain_frombuffer_copy_ms": 0.0, - "drain_reshape_ms": 0.0, - "drain_controls_ms": 0.0, - "sync_select_ms": 0.0, - "meta_ms": 0.0, - "sleep_ms": 0.0, - "sleep_count": 0, - "loop_count": 0, - "drained_total": 0, - "drained_by_cam": {}, - "queue_has_true_by_cam": {}, - "buffer_lengths_before_sync": {}, - "buffer_lengths_after_drain": {}, - "buffer_lengths_after_sync": {}, - "selected_seq_by_cam": {}, - "selected_ts_by_cam": {}, - "sync_dt_ms": None, - "sync_ok": None, - "wait_reason": None, - } - - def _add_ms(self, perf, key, t0_ms): - if perf is not None: - perf[key] = float(perf.get(key, 0.0) or 0.0) + float(self._cap_now_ms() - t0_ms) - - def _buffer_lengths_snapshot(self): - return { - cam_id: int(len(buf)) - for cam_id, buf in self.buffers.items() - } - - - - def _reset_async_capture_state(self): - self._capture_stop_event = threading.Event() - self._capture_lock = threading.RLock() - self._capture_cond = threading.Condition(self._capture_lock) - self._latest_packet = None - self._latest_packet_seq = 0 - self._last_consumed_packet_seq = 0 - self._packet_queue = deque(maxlen=int(getattr(self, "async_capture_max_queue", 2))) - self._capture_thread_stats = { - "started": False, - "packets": 0, - "dropped_latest": 0, - "dropped_queue": 0, - "last_error": None, - "last_loop_ms": 0.0, - "last_packet_age_ms": None, - } - - def _start_async_capture_thread(self): - if not bool(getattr(self, "async_capture_enabled", True)): - return - - if self._capture_thread is not None and self._capture_thread.is_alive(): - return - - self._capture_stop_event.clear() - - self._capture_thread = threading.Thread( - target=self._async_capture_loop, - name="OakFcc3AsyncCapture", - daemon=True, - ) - self._capture_thread.start() - - def _stop_async_capture_thread(self): + def get_last_radiometric_normalization_result(self): try: - if hasattr(self, "_capture_stop_event") and self._capture_stop_event is not None: - self._capture_stop_event.set() - - if hasattr(self, "_capture_cond") and self._capture_cond is not None: - with self._capture_cond: - self._capture_cond.notify_all() - - th = getattr(self, "_capture_thread", None) - if ( - th is not None - and th.is_alive() - and th is not threading.current_thread() - ): - th.join(timeout=1.0) - - if th.is_alive(): - try: - if self.device is not None: - self.device.close() - except Exception: - pass - - th.join(timeout=2.0) - - if th.is_alive(): - self._capture_thread_stats["last_error"] = ("thread de captura não encerrou após fechamento do device") - return False - - self._capture_thread = None - return True + return self.core.get_last_radiometric_normalization_result() except Exception: - pass + return None - self._capture_thread = None + def build_infer_tensor(self, frame, meta, channels_expected, target_size=None): + channels_expected = self._validate_physical_channel_count(channels_expected) + return self.core.build_infer_tensor_from_stream( + frame, + meta, + channels_expected=channels_expected, + target_size=target_size, + ) - def _async_capture_loop(self): - if not hasattr(self, "_capture_thread_stats"): - thread_anterior = self._capture_thread - if (thread_anterior is not None and thread_anterior.is_alive()): - raise RuntimeError("Não é seguro iniciar novo pipeline: thread OakFcc3AsyncCapture anterior ainda está viva.") - self._reset_async_capture_state() + def build_infer_tensor_from_decoded( + self, + decoded, + meta, + channels_expected, + target_size=None, + evaluate_quality=None, + ): + """ + Monta o Raw5 físico a partir das câmeras já decodificadas. - self._capture_thread_stats["started"] = True + evaluate_quality: + - None -> usa self.evaluate_quality + - True -> executa evaluate_frame_quality() e atualiza + core.last_frame_quality_result + - False -> não executa a auditoria pesada e limpa + core.last_frame_quality_result - while not self._capture_stop_event.is_set(): - t_loop0 = time.perf_counter() + A flag altera somente a auditoria de qualidade. Não altera decode, + radiometria, flat-field, homografia, crop/resize ou patch normalization. + """ + channels_expected = self._validate_physical_channel_count(channels_expected) - try: - # Perf interno leve para diagnóstico. Não precisa imprimir todo frame. - perf = self._new_capture_perf() if hasattr(self, "_new_capture_perf") else None + if evaluate_quality is None: + evaluate_quality = self.evaluate_quality + evaluate_quality = bool(evaluate_quality) - # Importante: esta thread é a única que mexe nas queues/buffers. - self._drain_queues_to_buffers(perf=perf) - synced = self._try_get_synced_packet(perf=perf) + tensor = self.core.fuse_multispec_cameras(decoded, meta, channels_expected, target_size=target_size) + #tensor = self.core.resize_tensor_chw(tensor, target_size=target_size) - if synced is None: - # Dorme curto. Pode testar 0.0005 se quiser reduzir latência. - time.sleep(0.001) - continue + # Mantém paridade com build_infer_tensor_from_stream: se a calibração + # habilitar patch normalization, ela também vale no caminho decoded. + patch_cfg = getattr(self.core, "patch_normalization_config", {}) or {} + if bool(patch_cfg.get("enabled", False)): + tensor = self.core.apply_patch_normalization_to_tensor(tensor) - frames, timestamps, sync_dt_ms, sync_ok, frame_controls = synced + if evaluate_quality: + self.core.last_frame_quality_result = self.core.evaluate_frame_quality(tensor) + else: + # Evita deixar um resultado antigo parecer referente ao frame atual. + self.core.last_frame_quality_result = None - self.frame_id += 1 - meta = self._build_meta( - frames, - timestamps, - sync_dt_ms, - sync_ok, - frame_controls=frame_controls, - ) + return tensor - now = time.perf_counter() - packet = { - "seq": int(self._latest_packet_seq + 1), - "created_perf_counter": float(now), - "frames": frames, - "meta": meta, + def decode_stream_cameras(self, frame, meta): + if str(meta.get("frame_type", self.frame_type)).upper() == "PREVIEW": + decoded = {} + + camera_info = meta.get("camera_info", {}) or {} + + for cam_id, img in frame.items(): + info = camera_info.get(cam_id, {}) or {} + role = info.get("role", cam_id) + img01 = self._frame_to_float01(cam_id, img, role) + + decoded[cam_id] = { + "name": role.upper(), + "role": role, + "image": img01, + "meta": { + "cam_id": cam_id, + "role": role, + "socket": info.get("socket"), + "sensor": info.get("sensor"), + "timestamp": (meta.get("timestamps") or {}).get(cam_id), + "shape": list(img.shape), + "dtype": str(img.dtype), + }, } - # Adiciona perf de captura assíncrona no meta. - if perf is not None: - perf["async_thread"] = True - perf["wait_total_ms"] = (time.perf_counter() - t_loop0) * 1000.0 - perf["sync_dt_ms"] = float(sync_dt_ms) - perf["sync_ok"] = bool(sync_ok) - perf["wait_reason"] = "async_packet_ready" - meta["capture_perf"] = perf + return decoded - with self._capture_cond: - self._latest_packet_seq += 1 - packet["seq"] = int(self._latest_packet_seq) + return self.core.decode_stream_cameras(frame, meta) - if str(getattr(self, "async_capture_mode", "latest")).lower() == "queue": - before = len(self._packet_queue) - self._packet_queue.append(packet) - if before == self._packet_queue.maxlen: - self._capture_thread_stats["dropped_queue"] += 1 - else: - # latest mode: substitui pacote antigo se consumidor não pegou. - if self._latest_packet is not None and self._last_consumed_packet_seq < self._latest_packet.get("seq", 0): - self._capture_thread_stats["dropped_latest"] += 1 - self._latest_packet = packet + def build_rgb_tensor(self, decoded): + cam_id, item = self._find_decoded_by_role(decoded, "rgb") - self._capture_thread_stats["packets"] += 1 - self._capture_thread_stats["last_error"] = None - self._capture_thread_stats["last_loop_ms"] = (time.perf_counter() - t_loop0) * 1000.0 + rgb01 = item["image"] - self._capture_cond.notify_all() + if rgb01.ndim != 3 or rgb01.shape[2] != 3: + raise RuntimeError(f"{cam_id} RGB inválida: shape={rgb01.shape}") - except Exception as e: - erro = f"{type(e).__name__}: {e}" + tensor = np.transpose(rgb01.astype(np.float32), (2, 0, 1)) + return np.ascontiguousarray(tensor.astype(np.float32, copy=False)) - try: - self._capture_thread_stats["last_error"] = erro - except Exception: - pass + def build_multispec_tensor( + self, + decoded, + meta=None, + target_size=None, + evaluate_quality=None, + ): + tensor = self.build_infer_tensor_from_decoded( + decoded=decoded, + meta=meta, + channels_expected=5, + target_size=target_size, + evaluate_quality=evaluate_quality, + ) + return np.ascontiguousarray(tensor.astype(np.float32, copy=False)) + + def build_preview_from_raw_payload(self, frame, meta): + """ + Gera preview priorizando a câmera com role='rgb'. + Retorna: + preview_bgr: imagem BGR uint8 para OpenCV + payload_float_preview: tensor CHW float32 [0..1] + preview_source_id: cam_id usado + """ + decoded = self.decode_stream_cameras(frame, meta) - if self._is_fatal_depthai_error(e): - try: - self._capture_thread_stats["fatal_error"] = True - except Exception: - pass - - self.running = False - - try: - self._capture_stop_event.set() - except Exception: - pass - - try: - with self._capture_cond: - self._capture_cond.notify_all() - except Exception: - pass - - break - - time.sleep(0.005) + if not decoded: + raise RuntimeError("Nenhum frame decodificado disponível para preview.") try: - self._capture_thread_stats["started"] = False - except Exception: + cam_id, item = self._find_decoded_by_role(decoded, "rgb") + rgb01 = item["image"] + + if rgb01.ndim != 3 or rgb01.shape[2] != 3: + raise RuntimeError(f"{cam_id} decodificada inválida para preview RGB: shape={rgb01.shape}") + + preview_bgr = self._rgb01_to_bgr(rgb01) + payload_float = np.transpose(rgb01.astype(np.float32), (2, 0, 1)) + + return preview_bgr, np.ascontiguousarray(payload_float), cam_id + + except RuntimeError: pass - def get_async_capture_status(self): - with self._capture_lock: - latest_age_ms = None - if self._latest_packet is not None: - latest_age_ms = (time.perf_counter() - float(self._latest_packet.get("created_perf_counter", 0.0))) * 1000.0 + first_id = list(decoded.keys())[0] + img01 = decoded[first_id]["image"] - st = dict(getattr(self, "_capture_thread_stats", {}) or {}) - st.update({ - "enabled": bool(getattr(self, "async_capture_enabled", True)), - "mode": str(getattr(self, "async_capture_mode", "latest")), - "thread_alive": bool(self._capture_thread is not None and self._capture_thread.is_alive()), - "latest_seq": int(getattr(self, "_latest_packet_seq", 0)), - "last_consumed_seq": int(getattr(self, "_last_consumed_packet_seq", 0)), - "queue_len": int(len(getattr(self, "_packet_queue", []))), - "latest_age_ms": latest_age_ms, - }) - return st + if img01.ndim == 2: + preview_bgr = self._gray01_to_bgr(img01) + payload_float = np.stack([img01, img01, img01], axis=0).astype(np.float32) + + elif img01.ndim == 3 and img01.shape[2] == 3: + preview_bgr = self._rgb01_to_bgr(img01) + payload_float = np.transpose(img01.astype(np.float32), (2, 0, 1)) + + else: + raise RuntimeError(f"Frame decodificado inválido para preview: cam={first_id}, shape={img01.shape}") + + return preview_bgr, np.ascontiguousarray(payload_float), first_id + + def build_visual_preview_from_raw(self, frame, meta): + camera_info = meta.get("camera_info", {}) or {} + previews = {} + + for cam_id, arr in frame.items(): + info = camera_info.get(cam_id, {}) or {} + role = info.get("role", cam_id) + bit_depth = int(info.get("bit_depth", 10)) + + if arr.ndim == 3 and arr.shape[2] == 1: + arr = arr[:, :, 0] + + if bit_depth == 10 and arr.ndim == 2: + raw16 = self.core.unpack_raw10_packed( + arr, + sensor_width=int(info.get("width", self.width)), + sensor_height=int(info.get("height", self.height)), + ) + + if role == "rgb": + previews[cam_id] = self.preview.raw16_to_preview_bgr( + raw16, + bit_depth=bit_depth, + apply_wb=True, + apply_contrast=True, + ) + else: + vis8 = self.preview.raw16_to_vis8( + raw16, + bit_depth=bit_depth, + gamma=2.2, + ) + previews[cam_id] = cv2.cvtColor(vis8, cv2.COLOR_GRAY2BGR) + + else: + decoded = self.decode_stream_cameras({cam_id: arr}, {"camera_info": {cam_id: info}}) + img01 = decoded[cam_id]["image"] + + if img01.ndim == 2: + previews[cam_id] = self._gray01_to_bgr(img01) + else: + previews[cam_id] = self._rgb01_to_bgr(img01) + + return previews + + def build_save_preview_from_cam_a( + self, + packed_raw_by_camera: dict | None, + meta_stream: dict, + sensor_width: int, + sensor_height: int, + bayer_pattern: str, + ) -> np.ndarray | None: + """ + Gera o preview salvo no mesmo padrão do 'CAM_A reconstruido'. + + Usa apenas CAM_A do RAW_BRUTO: + CAM_A packed RAW10 + -> unpack_raw10_packed + -> RawProcessorPreview.raw16_to_preview_bgr + + Retorna BGR uint8 pronto para cv2.imwrite. + """ + if not packed_raw_by_camera or "CAM_A" not in packed_raw_by_camera: + return None + + stream_meta = meta_stream or {} + + camera_info = stream_meta.get("camera_info", {}) or {} + cam_meta = camera_info.get("CAM_A", {}) or {} + + bit_depth = int(cam_meta.get("bit_depth", 10)) + + bayer = ( + cam_meta.get("bayer_pattern") + or cam_meta.get("bayer") + or stream_meta.get("bayer_pattern") + or bayer_pattern + or "RGGB" + ) + bayer = str(bayer).upper() + + arr = packed_raw_by_camera["CAM_A"] + + if arr is None: + return None + + packed = arr + if packed.ndim == 3 and packed.shape[2] == 1: + packed = packed[:, :, 0] + + if packed.ndim != 2: + return None + + # Mantém a mesma lógica do validador: + # RAW10 packed => sensor_w = packed_w * 4 // 5 + packed_h, packed_w = packed.shape[:2] + + if bit_depth == 10: + real_w = int(cam_meta.get("width", sensor_width)) + real_h = int(cam_meta.get("height", sensor_height)) + + # Fallback caso o meta não tenha width/height confiáveis + if real_w <= 0 or real_h <= 0: + real_w = int((packed_w * 4) // 5) + real_h = int(packed_h) + else: + real_w = int(cam_meta.get("width", sensor_width)) + real_h = int(cam_meta.get("height", sensor_height)) + + core = RawProcessorCore( + sensor_width=real_w, + sensor_height=real_h, + bayer_pattern=bayer, + ) + + preview = RawProcessorPreview( + sensor_width=real_w, + sensor_height=real_h, + bayer_pattern=bayer, + ) + + raw16 = core.unpack_raw10_packed( + packed, + sensor_width=real_w, + sensor_height=real_h, + ) + + preview_bgr = preview.raw16_to_preview_bgr( + raw16, + bit_depth=bit_depth, + ) + + return preview_bgr + + def _find_decoded_by_role(self, decoded, role): + role = str(role).lower() + + for cam_id, item in decoded.items(): + if str(item.get("role", "")).lower() == role: + return cam_id, item + + raise RuntimeError(f"Nenhuma câmera com role={role} encontrada.") + + def _frame_to_float01(self, cam_id, img, role): + if img is None: + return None + + arr = img + + if arr.dtype == np.uint8: + arr01 = arr.astype(np.float32) / 255.0 + elif arr.dtype == np.uint16: + arr01 = arr.astype(np.float32) / 65535.0 + else: + arr01 = arr.astype(np.float32) + if arr01.max() > 1.5: + arr01 = arr01 / 255.0 + + arr01 = np.clip(arr01, 0.0, 1.0) + + if role == "rgb": + # DepthAI/OpenCV entrega BGR HWC. Calibradores esperam RGB HWC. + if arr01.ndim == 3 and arr01.shape[2] == 3: + arr01 = cv2.cvtColor((arr01 * 255).astype(np.uint8), cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 + elif arr01.ndim == 2: + arr01 = np.stack([arr01, arr01, arr01], axis=2) + + return arr01 + + # Espectrais devem virar mono HW. + if arr01.ndim == 3: + arr01 = cv2.cvtColor((arr01 * 255).astype(np.uint8), cv2.COLOR_BGR2GRAY).astype(np.float32) / 255.0 + + return arr01 + + @staticmethod + def _rgb01_to_bgr(rgb01): + rgb_u8 = np.clip(rgb01 * 255.0, 0, 255).astype(np.uint8) + return cv2.cvtColor(rgb_u8, cv2.COLOR_RGB2BGR) + + @staticmethod + def _gray01_to_bgr(gray01): + g = np.clip(gray01 * 255.0, 0, 255).astype(np.uint8) + return cv2.cvtColor(g, cv2.COLOR_GRAY2BGR) + + + + def decode_oak_aligned_multispec(self, frame, meta): + """ + Decodifica frames já alinhados pela OAK. + + Entrada esperada: + frame = { + "CAM_A": BGR uint8 HWC, + "CAM_B": GRAY uint8 HW, + "CAM_C": GRAY uint8 HW, + } + + Saída: + decoded por role, em float32 0..1. + """ + if not isinstance(frame, dict): + raise RuntimeError("MULTISPEC alinhado esperado como dict de câmeras.") + + decoded = {} + camera_info = meta.get("camera_info", {}) or {} + + for cam_id, img in frame.items(): + info = camera_info.get(cam_id, {}) or {} + role = str(info.get("role", "")).lower() + + if not role: + role = str(self.svc.manager.roles.get(cam_id, cam_id)).lower() + + img01 = self._frame_to_float01(cam_id, img, role) + + decoded[cam_id] = { + "name": role.upper(), + "role": role, + "image": img01, + "meta": { + **info, + "cam_id": cam_id, + "role": role, + "aligned_by_oak": True, + "geometry_stage": "oak", + "homography_applied": role in ("re", "nir"), + "crop_resize_applied": True, + "shape": list(img.shape), + "dtype": str(img.dtype), + }, + } + + return decoded + + def build_multispec_tensor_from_oak_aligned(self, decoded, meta=None): + """ + Monta CHW [R,G,B,RE,NIR] sem reaplicar homografia. + """ + _, rgb_item = self._find_decoded_by_role(decoded, "rgb") + _, re_item = self._find_decoded_by_role(decoded, "re") + _, nir_item = self._find_decoded_by_role(decoded, "nir") + + rgb = rgb_item["image"].astype(np.float32, copy=False) + re = re_item["image"].astype(np.float32, copy=False) + nir = nir_item["image"].astype(np.float32, copy=False) + + if rgb.ndim != 3 or rgb.shape[2] != 3: + raise RuntimeError(f"RGB alinhado inválido: shape={rgb.shape}") + + h, w = rgb.shape[:2] + + if re.ndim == 3: + re = re[:, :, 0] + + if nir.ndim == 3: + nir = nir[:, :, 0] + + if re.shape[:2] != (h, w): + re = cv2.resize(re, (w, h), interpolation=cv2.INTER_LINEAR) + + if nir.shape[:2] != (h, w): + nir = cv2.resize(nir, (w, h), interpolation=cv2.INTER_LINEAR) + + tensor = np.stack( + [ + rgb[:, :, 0], # R + rgb[:, :, 1], # G + rgb[:, :, 2], # B + re, + nir, + ], + axis=0, + ).astype(np.float32, copy=False) + + return np.ascontiguousarray(tensor) diff --git a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_service.py b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_service.py index 29e511bac..371cc4dc4 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_service.py +++ b/Python/OAK/datasets/oak-fcc-3/core/oak_fcc3_service.py @@ -1,29 +1,157 @@ +import copy import time + from .oak_fcc3_manager import OakFcc3Manager +OAK_FCC3_SERVICE_VERSION = "production_v1_2026_08_24" + + class OakFcc3Service: + """ + Camada de serviço do OakFcc3Manager. + + Responsabilidades: + - ciclo de vida connect/begin/stop/disconnect; + - captura de frame; + - controles de câmera; + - exposição de status/config; + - impedir que APIs legadas alterem o contrato físico de produto. + + Importante: + Em module_params homologado, resolução/sensor/Bayer são propriedades + calibradas do módulo e NÃO podem ser alteradas via Service. + + width/height continuam visíveis somente para compatibilidade e representam + a saída legada do Manager, não o raster nativo das três câmeras. + """ + def __init__(self, timeout=10, **kwargs): - self.timeout = timeout + self.timeout = float(timeout) self.manager = OakFcc3Manager(**kwargs) self.connected = False + # ============================================================ + # Contrato + # ============================================================ + + @property + def product_contract(self): + return bool( + getattr( + self.manager, + "product_contract", + False, + ) + ) + + @property + def require_product_contract(self): + return bool( + getattr( + self.manager, + "require_product_contract", + False, + ) + ) + + def get_contract(self): + """ + Snapshot serializável do contrato físico/runtime conhecido pelo Service. + """ + status = self.manager.get_status() + + return { + "service_version": OAK_FCC3_SERVICE_VERSION, + "manager_version": status.get("manager_version"), + "product_contract": bool( + status.get( + "product_contract", + self.product_contract, + ) + ), + "require_product_contract": bool( + status.get( + "require_product_contract", + self.require_product_contract, + ) + ), + "module_params_schema": status.get( + "module_params_schema" + ), + "module_calibration_json": status.get( + "module_calibration_json" + ), + "mx_id": status.get( + "mx_id" + ), + "sensor_size_by_role": copy.deepcopy( + status.get( + "sensor_size_by_role", + {}, + ) + or {} + ), + "camera_hardware": copy.deepcopy( + status.get( + "camera_hardware_expected", + {}, + ) + or {} + ), + "bayer_pattern": status.get( + "bayer_pattern" + ), + "frame_type": status.get( + "frame_type" + ), + "raw_policy": status.get( + "raw_policy" + ), + "geometry_stage": status.get( + "geometry_stage" + ), + } + + # ============================================================ + # Ciclo de vida + # ============================================================ + def connect(self): + """ + Marca a camada de serviço como conectada. + + O dispositivo físico é aberto pelo manager.start() dentro de begin(). + Isso preserva a API existente sem fingir que connect() já abriu USB. + """ self.connected = True - return {"ok": True, "backend": "oak_fcc3", "connected": True} + + return { + "ok": True, + "backend": "oak_fcc3", + "connected": True, + "service_version": OAK_FCC3_SERVICE_VERSION, + "product_contract": self.product_contract, + } def disconnect(self): + """ + Único fechamento público completo da camada Service. + """ stopped = False try: - stopped = bool(self.manager.stop()) + stopped = bool( + self.manager.stop() + ) finally: self.connected = False return { - "ok": stopped, + "ok": bool(stopped), "connected": False, - "stopped": stopped, + "stopped": bool(stopped), + "service_version": OAK_FCC3_SERVICE_VERSION, } def ping(self): @@ -32,86 +160,26 @@ class OakFcc3Service: "backend": "oak_fcc3", "msg": "pong", "ts": time.time(), + "connected": bool( + self.connected + ), + "running": bool( + getattr( + self.manager, + "running", + False, + ) + ), + "service_version": OAK_FCC3_SERVICE_VERSION, + "product_contract": self.product_contract, } - def get_status(self): - status = self.manager.get_status() - active_ids = [c["id"] for c in status.get("cameras", [])] - active_roles = { - c.get("role"): c.get("id") - for c in status.get("cameras", []) - } - - running = bool(status.get("running", False)) - realmente_ativo = bool(self.connected and running) - - status.update({ - "ok": realmente_ativo, - "connected": bool(self.connected), - "running": running, - "active_camera_ids": active_ids, - "active_roles": active_roles, - "camera_count_active": len(active_ids), - }) - - return status - - def get_device_metrics(self): - return self.manager.get_device_metrics() - - def get_config(self): - return { - "mx_id": self.manager.mx_id, - "ok": True, - "fps": self.manager.fps, - "width": self.manager.width, - "height": self.manager.height, - "frame_type": self.manager.frame_type, - "output_dtype": self.manager.output_dtype, - "capture_mode": self.manager.capture_mode, - "raw_policy": self.manager.raw_policy, - "sync_mode": getattr(self.manager, "sync_mode", "best"), - "sync_tolerance_ms": self.manager.sync_tolerance_ms, - "imu_modo": self.manager.imu_modo, - "imu_freq_hz": self.manager.imu_freq_hz, - "imu_sensor_type": self.manager.imu_sensor_type, - } - - def set_fps(self, fps): - self._ensure_stopped_for_config() - self.manager.fps = int(fps) - return {"ok": True, "fps": self.manager.fps} - - def set_resolution(self, width, height): - self._ensure_stopped_for_config() - self.manager.width = int(width) - self.manager.height = int(height) - self.manager.size = (self.manager.width, self.manager.height) - return { - "ok": True, - "width": self.manager.width, - "height": self.manager.height, - } - - def set_capture_mode(self, mode): - self._ensure_stopped_for_config() - mode = str(mode).upper() - self.manager.capture_mode = self._validate_capture_mode(mode) - return {"ok": True, "capture_mode": self.manager.capture_mode} - - def set_frame_type(self, frame_type): - self._ensure_stopped_for_config() - frame_type = str(frame_type).upper() - self.manager.frame_type = self._validate_frame_type(frame_type) - return {"ok": True, "frame_type": self.manager.frame_type} - - def set_output_dtype(self, dtype): - self._ensure_stopped_for_config() - dtype = str(dtype).lower() - self.manager.output_dtype = self._validate_output_dtype(dtype) - return {"ok": True, "output_dtype": self.manager.output_dtype} - - def begin(self, frame_type=None, output_dtype=None, capture_mode=None): + def begin( + self, + frame_type=None, + output_dtype=None, + capture_mode=None, + ): if not self.connected: self.connect() @@ -124,129 +192,789 @@ class OakFcc3Service: } if frame_type is not None: - self.manager.frame_type = self._validate_frame_type(frame_type) + frame_type = self._validate_frame_type( + frame_type + ) + self._validate_product_frame_type( + frame_type + ) + self.manager.frame_type = frame_type if output_dtype is not None: - self.manager.output_dtype = self._validate_output_dtype(output_dtype) + self.manager.output_dtype = ( + self._validate_output_dtype( + output_dtype + ) + ) if capture_mode is not None: - self.manager.capture_mode = self._validate_capture_mode(capture_mode) + self.manager.capture_mode = ( + self._validate_capture_mode( + capture_mode + ) + ) + + # Mesmo sem override explícito, bloqueia MP produto com modo legado. + self._validate_product_frame_type( + self.manager.frame_type + ) self.manager.start() + status = self.get_status() + + if not status.get( + "ok", + False, + ): + try: + self.manager.stop() + finally: + raise RuntimeError( + "OakFcc3Manager iniciou sem atingir estado operacional: " + f"{status}" + ) + return { "ok": True, "started": True, "already_running": False, - "status": self.get_status(), + "status": status, } - def capture_frame(self, timeout=None): - if timeout is None: - timeout = self.timeout - - frame, meta = self.manager.get_next_frame(timeout=timeout) - - return frame, meta - def stop(self): - stopped = bool(self.manager.stop()) + stopped = bool( + self.manager.stop() + ) + return { "ok": stopped, "stopped": stopped, + "connected": bool( + self.connected + ), } - def _ensure_stopped_for_config(self): + # ============================================================ + # Status/config + # ============================================================ + + def get_status(self): + status = dict( + self.manager.get_status() + ) + + cameras = list( + status.get( + "cameras", + [], + ) + or [] + ) + + active_ids = [ + c.get("id") + for c in cameras + if c.get("id") is not None + ] + + active_roles = { + str( + c.get( + "role", + "", + ) + ).lower(): c.get( + "id" + ) + for c in cameras + if c.get("role") + and c.get("id") is not None + } + + running = bool( + status.get( + "running", + False, + ) + ) + + realmente_ativo = bool( + self.connected + and running + ) + + status.update({ + "ok": realmente_ativo, + "connected": bool( + self.connected + ), + "running": running, + "active_camera_ids": active_ids, + "active_roles": active_roles, + "camera_count_active": len( + active_ids + ), + "service_version": OAK_FCC3_SERVICE_VERSION, + + # Contrato explícito para consumers novos. + "sensor_size_by_role": copy.deepcopy( + status.get( + "sensor_size_by_role", + {}, + ) + or {} + ), + "camera_hardware": copy.deepcopy( + status.get( + "camera_hardware_expected", + {}, + ) + or {} + ), + }) + + return status + + def get_device_metrics(self): + metrics = dict( + self.manager.get_device_metrics() + ) + + metrics[ + "service_version" + ] = OAK_FCC3_SERVICE_VERSION + + metrics[ + "connected" + ] = bool( + self.connected + ) + + return metrics + + def get_config(self): + """ + Configuração exposta ao Client/debug. + + width/height são mantidos por compatibilidade e explicitamente marcados + como LEGACY OUTPUT SIZE. Não descrevem a resolução nativa do módulo. + """ + status = self.manager.get_status() + + return { + "mx_id": self.manager.mx_id, + "ok": True, + "service_version": OAK_FCC3_SERVICE_VERSION, + "manager_version": status.get( + "manager_version" + ), + + "fps": self.manager.fps, + + # Compatibilidade antiga. + "width": self.manager.width, + "height": self.manager.height, + "legacy_output_size": [ + int( + self.manager.width + ), + int( + self.manager.height + ), + ], + "legacy_output_size_authoritative": False, + + # Contrato físico real. + "sensor_width": status.get( + "sensor_width" + ), + "sensor_height": status.get( + "sensor_height" + ), + "sensor_size_by_role": copy.deepcopy( + status.get( + "sensor_size_by_role", + {}, + ) + or {} + ), + "camera_hardware": copy.deepcopy( + status.get( + "camera_hardware_expected", + {}, + ) + or {} + ), + "bayer_pattern": status.get( + "bayer_pattern" + ), + "product_contract": bool( + status.get( + "product_contract", + False, + ) + ), + "require_product_contract": bool( + status.get( + "require_product_contract", + False, + ) + ), + "module_params_schema": status.get( + "module_params_schema" + ), + "module_calibration_json": status.get( + "module_calibration_json" + ), + + "frame_type": self.manager.frame_type, + "output_dtype": self.manager.output_dtype, + "capture_mode": self.manager.capture_mode, + "raw_policy": self.manager.raw_policy, + + "sync_mode": getattr( + self.manager, + "sync_mode", + "best", + ), + "sync_tolerance_ms": self.manager.sync_tolerance_ms, + "hardware_sync_enabled": bool( + getattr( + self.manager, + "hardware_sync_enabled", + False, + ) + ), + "frame_sync_master": getattr( + self.manager, + "frame_sync_master", + None, + ), + + "geometry_stage": status.get( + "geometry_stage" + ), + + "imu_modo": self.manager.imu_modo, + "imu_freq_hz": self.manager.imu_freq_hz, + "imu_sensor_type": self.manager.imu_sensor_type, + } + + # ============================================================ + # Configuração estrutural + # ============================================================ + + def set_fps(self, fps): + self._ensure_stopped_for_config() + + fps = float( + fps + ) + + if fps <= 0: + raise ValueError( + f"fps deve ser > 0: {fps}" + ) + + self.manager.fps = fps + + return { + "ok": True, + "fps": self.manager.fps, + } + + def set_resolution( + self, + width, + height, + ): + """ + API legada. + + Em produto NÃO pode alterar resolução física. O raster nativo faz parte + da calibração/module_params e é selecionado pelo sensor real. + """ + self._ensure_stopped_for_config() + + if self.product_contract: + raise RuntimeError( + "set_resolution() é proibido com module_params de produção. " + "A resolução nativa é definida por camera_hardware/" + "sensor_size_by_role e pelo sensor físico. " + "Para mudar o tensor final use um novo modelo + module_params " + "homologado e reinicialize o pipeline." + ) + + width = int( + width + ) + height = int( + height + ) + + if width <= 0 or height <= 0: + raise ValueError( + f"Resolução inválida: {width}x{height}" + ) + + self.manager.width = width + self.manager.height = height + self.manager.size = ( + width, + height, + ) + + return { + "ok": True, + "width": width, + "height": height, + "legacy_only": True, + } + + def set_capture_mode( + self, + mode, + ): + self._ensure_stopped_for_config() + + mode = self._validate_capture_mode( + mode + ) + + self.manager.capture_mode = mode + + return { + "ok": True, + "capture_mode": mode, + } + + def set_frame_type( + self, + frame_type, + ): + self._ensure_stopped_for_config() + + frame_type = ( + self._validate_frame_type( + frame_type + ) + ) + + self._validate_product_frame_type( + frame_type + ) + + self.manager.frame_type = frame_type + + return { + "ok": True, + "frame_type": frame_type, + } + + def set_output_dtype( + self, + dtype, + ): + self._ensure_stopped_for_config() + + dtype = ( + self._validate_output_dtype( + dtype + ) + ) + + self.manager.output_dtype = dtype + + return { + "ok": True, + "output_dtype": dtype, + } + + # ============================================================ + # Captura + # ============================================================ + + def capture_frame( + self, + timeout=None, + ): + if timeout is None: + timeout = self.timeout + + if not self.connected: + raise RuntimeError( + "OakFcc3Service não está conectado." + ) + + if not self.manager.running: + raise RuntimeError( + "OakFcc3Manager não está rodando. " + "Chame begin() antes de capture_frame()." + ) + + frame, meta = ( + self.manager.get_next_frame( + timeout=float( + timeout + ) + ) + ) + + return frame, meta + + # ============================================================ + # Controles de câmera + # ============================================================ + + def resolve_camera_id( + self, + cam_id=None, + role=None, + ): + if role is not None: + role = str( + role + ).lower() + + status = ( + self.manager.get_status() + ) + + for cam in status.get( + "cameras", + [], + ): + if ( + str( + cam.get( + "role", + "", + ) + ).lower() + == role + ): + return cam[ + "id" + ] + + raise ValueError( + f"Nenhuma câmera ativa encontrada para role={role}" + ) + + if cam_id is None: + raise ValueError( + "Informe cam_id ou role." + ) + + return str( + cam_id + ) + + def get_camera_controls( + self, + cam_id=None, + role=None, + ): + cam_id = self.resolve_camera_id( + cam_id=cam_id, + role=role, + ) + + ctrl = dict( + self.manager.get_camera_controls( + cam_id + ) + ) + + ctrl[ + "ok" + ] = True + + ctrl[ + "camera_id" + ] = cam_id + + ctrl[ + "role" + ] = ( + self._resolve_role_from_camera_id( + cam_id + ) + ) + + return ctrl + + def set_ae_enable( + self, + cam_id=None, + role=None, + enable=False, + ): + cam_id = self.resolve_camera_id( + cam_id=cam_id, + role=role, + ) + + ctrl = dict( + self.manager.set_ae_enable( + cam_id, + bool( + enable + ), + ) + ) + + return self._decorate_control_response( + ctrl, + cam_id, + ) + + def set_awb_enable( + self, + cam_id=None, + role=None, + enable=False, + ): + cam_id = self.resolve_camera_id( + cam_id=cam_id, + role=role, + ) + + ctrl = dict( + self.manager.set_awb_enable( + cam_id, + bool( + enable + ), + ) + ) + + return self._decorate_control_response( + ctrl, + cam_id, + ) + + def set_exposure_time( + self, + cam_id=None, + role=None, + exposure_time_us=None, + ): + if exposure_time_us is None: + raise ValueError( + "exposure_time_us é obrigatório." + ) + + cam_id = self.resolve_camera_id( + cam_id=cam_id, + role=role, + ) + + ctrl = dict( + self.manager.set_exposure_time( + cam_id, + int( + exposure_time_us + ), + ) + ) + + return self._decorate_control_response( + ctrl, + cam_id, + ) + + def set_analogue_gain( + self, + cam_id=None, + role=None, + analogue_gain=None, + ): + if analogue_gain is None: + raise ValueError( + "analogue_gain é obrigatório." + ) + + cam_id = self.resolve_camera_id( + cam_id=cam_id, + role=role, + ) + + ctrl = dict( + self.manager.set_analogue_gain( + cam_id, + float( + analogue_gain + ), + ) + ) + + return self._decorate_control_response( + ctrl, + cam_id, + ) + + def apply_camera_controls( + self, + cam_id=None, + role=None, + controls=None, + ): + cam_id = self.resolve_camera_id( + cam_id=cam_id, + role=role, + ) + + controls = ( + dict( + controls + ) + if isinstance( + controls, + dict, + ) + else {} + ) + + ctrl = dict( + self.manager.apply_camera_controls( + cam_id, + controls, + ) + ) + + return self._decorate_control_response( + ctrl, + cam_id, + ) + + # ============================================================ + # Helpers + # ============================================================ + + def _decorate_control_response( + self, + ctrl, + cam_id, + ): + ctrl[ + "ok" + ] = True + + ctrl[ + "camera_id" + ] = cam_id + + ctrl[ + "role" + ] = ( + self._resolve_role_from_camera_id( + cam_id + ) + ) + + return ctrl + + def _resolve_role_from_camera_id( + self, + cam_id, + ): + roles = getattr( + self.manager, + "roles", + {}, + ) or {} + + return str( + roles.get( + str( + cam_id + ), + "", + ) + ).lower() or None + + def _ensure_stopped_for_config( + self, + ): if self.manager.running: raise RuntimeError( "Configuração estrutural só pode ser alterada com o manager parado. " "Chame stop() antes." ) - - def resolve_camera_id(self, cam_id=None, role=None): - if role is not None: - role = str(role).lower() - status = self.manager.get_status() - for cam in status.get("cameras", []): - if str(cam.get("role", "")).lower() == role: - return cam["id"] + def _validate_product_frame_type( + self, + frame_type, + ): + if ( + self.product_contract + and str( + frame_type + ).upper() + != "RAW_BRUTO" + ): + raise RuntimeError( + "module_params de produção exige frame_type=RAW_BRUTO. " + f"Recebido={frame_type!r}" + ) - raise ValueError(f"Nenhuma câmera ativa encontrada para role={role}") + @staticmethod + def _validate_frame_type( + frame_type, + ): + frame_type = str( + frame_type + ).upper() - if cam_id is None: - raise ValueError("Informe cam_id ou role.") + if frame_type not in ( + "RAW_BRUTO", + "RGB", + "MULTISPEC", + "PREVIEW", + ): + raise ValueError( + f"frame_type inválido: {frame_type}" + ) - return str(cam_id) - - - def get_camera_controls(self, cam_id=None, role=None): - cam_id = self.resolve_camera_id(cam_id=cam_id, role=role) - ctrl = self.manager.get_camera_controls(cam_id) - ctrl["ok"] = True - ctrl["camera_id"] = cam_id - ctrl["role"] = role - return ctrl - - def set_ae_enable(self, cam_id=None, role=None, enable=False): - cam_id = self.resolve_camera_id(cam_id=cam_id, role=role) - ctrl = self.manager.set_ae_enable(cam_id, bool(enable)) - ctrl["ok"] = True - ctrl["camera_id"] = cam_id - ctrl["role"] = role - return ctrl - - def set_awb_enable(self, cam_id=None, role=None, enable=False): - cam_id = self.resolve_camera_id(cam_id=cam_id, role=role) - ctrl = self.manager.set_awb_enable(cam_id, bool(enable)) - ctrl["ok"] = True - ctrl["camera_id"] = cam_id - ctrl["role"] = role - return ctrl - - def set_exposure_time(self, cam_id=None, role=None, exposure_time_us=None): - if exposure_time_us is None: - raise ValueError("exposure_time_us é obrigatório.") - cam_id = self.resolve_camera_id(cam_id=cam_id, role=role) - ctrl = self.manager.set_exposure_time(cam_id, int(exposure_time_us)) - ctrl["ok"] = True - ctrl["camera_id"] = cam_id - ctrl["role"] = role - return ctrl - - def set_analogue_gain(self, cam_id=None, role=None, analogue_gain=None): - if analogue_gain is None: - raise ValueError("analogue_gain é obrigatório.") - cam_id = self.resolve_camera_id(cam_id=cam_id, role=role) - ctrl = self.manager.set_analogue_gain(cam_id, float(analogue_gain)) - ctrl["ok"] = True - ctrl["camera_id"] = cam_id - ctrl["role"] = role - return ctrl - - def apply_camera_controls(self, cam_id=None, role=None, controls=None): - cam_id = self.resolve_camera_id(cam_id=cam_id, role=role) - ctrl = self.manager.apply_camera_controls(cam_id, controls or {}) - ctrl["ok"] = True - ctrl["camera_id"] = cam_id - ctrl["role"] = role - return ctrl - - - def _validate_frame_type(self, frame_type): - frame_type = str(frame_type).upper() - if frame_type not in ("RAW_BRUTO", "RGB", "MULTISPEC", "PREVIEW"): - raise ValueError(f"frame_type inválido: {frame_type}") return frame_type - def _validate_output_dtype(self, dtype): - dtype = str(dtype).lower() - if dtype not in ("uint8", "uint16", "float32"): - raise ValueError(f"output_dtype inválido: {dtype}") + @staticmethod + def _validate_output_dtype( + dtype, + ): + dtype = str( + dtype + ).lower() + + if dtype not in ( + "uint8", + "uint16", + "float32", + ): + raise ValueError( + f"output_dtype inválido: {dtype}" + ) + return dtype - def _validate_capture_mode(self, mode): - mode = str(mode).upper() - if mode not in ("AUTO", "SINGLE", "DOUBLE", "TRIPLE"): - raise ValueError(f"capture_mode inválido: {mode}") + @staticmethod + def _validate_capture_mode( + mode, + ): + mode = str( + mode + ).upper() + + if mode not in ( + "AUTO", + "SINGLE", + "DOUBLE", + "TRIPLE", + ): + raise ValueError( + f"capture_mode inválido: {mode}" + ) + return mode diff --git a/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py b/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py index 6282d4e94..3aa9385dd 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py +++ b/Python/OAK/datasets/oak-fcc-3/core/raw_processor_core.py @@ -28,7 +28,7 @@ module_params is loaded. The strict fail-closed behavior is enabled only for module_params generated by the production assembler. """ -RAW_PROCESSOR_CORE_VERSION = "production_v2_2026_09_08" +RAW_PROCESSOR_CORE_VERSION = "production_v5_2026_09_13_flat_finalspace_fast" import json import os @@ -339,12 +339,21 @@ if _HAS_NUMBA: out[y, x] = value - @_numba.njit(cache=True, fastmath=True, parallel=True) - def _apply_tensor_flat_gain_chw_numba(tensor, gain, channels, height, width, clip_output): - for c in _numba.prange(channels): - for y in range(height): - for x in range(width): - v = float(tensor[c, y, x]) * float(gain[c, y, x]) + @_numba.njit(cache=True, fastmath=False, parallel=True) + def _apply_radiometric_scale_inplace_numba(img, scale, clip_output): + """ + Radiometric scale em uma unica passada de memoria. + + Mantem a mesma ordem matematica do caminho NumPy e opera apenas em + imagens 2D/3D float32 C-contiguous. + """ + h = img.shape[0] + + if img.ndim == 2: + w = img.shape[1] + for y in _numba.prange(h): + for x in range(w): + v = img[y, x] * scale if clip_output: if v < 0.0: @@ -352,7 +361,244 @@ if _HAS_NUMBA: elif v > 1.0: v = 1.0 - tensor[c, y, x] = v + img[y, x] = v + + else: + w = img.shape[1] + c = img.shape[2] + for y in _numba.prange(h): + for x in range(w): + for k in range(c): + v = img[y, x, k] * scale + + if clip_output: + if v < 0.0: + v = 0.0 + elif v > 1.0: + v = 1.0 + + img[y, x, k] = v + + + @_numba.njit(cache=True, fastmath=False, parallel=True) + def _apply_radiometric_affine_inplace_numba(img, scale, offset, clip_output): + """ + Radiometric affine_v2 em uma unica passada de memoria. + + IMPORTANTE: usa explicitamente a mesma ordem de operacoes do contrato: + (value - offset) * scale + offset + + fastmath=False evita reassociacao/FMA e preserva, na pratica, o mesmo + resultado float32 do caminho NumPy de tres ufuncs. + """ + h = img.shape[0] + + if img.ndim == 2: + w = img.shape[1] + for y in _numba.prange(h): + for x in range(w): + v = img[y, x] + v = (v - offset) * scale + v = v + offset + + if clip_output: + if v < 0.0: + v = 0.0 + elif v > 1.0: + v = 1.0 + + img[y, x] = v + + else: + w = img.shape[1] + c = img.shape[2] + for y in _numba.prange(h): + for x in range(w): + for k in range(c): + v = img[y, x, k] + v = (v - offset) * scale + v = v + offset + + if clip_output: + if v < 0.0: + v = 0.0 + elif v > 1.0: + v = 1.0 + + img[y, x, k] = v + + + @_numba.njit(cache=True, fastmath=True, inline="always") + def _direct_sample_mono_fixedmap_numba(img, x0, y0, frac_code): + """ + Bilinear compatível com os mapas CV_16SC2/CV_16UC1 produzidos por + cv2.convertMaps(..., CV_16SC2). + + OpenCV INTER_LINEAR usa uma tabela de 32 passos por eixo. map1 guarda + a parte inteira e map2 guarda os índices fracionários quantizados. + Usar o mesmo código evita recalcular coordenadas/homografias por pixel. + """ + tab = int(frac_code) + fx_i = tab & 31 + fy_i = (tab >> 5) & 31 + + wx1 = fx_i * (1.0 / 32.0) + wy1 = fy_i * (1.0 / 32.0) + wx0 = 1.0 - wx1 + wy0 = 1.0 - wy1 + + h = img.shape[0] + w = img.shape[1] + x1 = x0 + 1 + y1 = y0 + 1 + + # Fast path: os quatro vizinhos estão dentro da imagem. + if x0 >= 0 and y0 >= 0 and x1 < w and y1 < h: + v00 = float(img[y0, x0]) + v01 = float(img[y0, x1]) + v10 = float(img[y1, x0]) + v11 = float(img[y1, x1]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + # BORDER_CONSTANT=0.0, incluindo pixels parcialmente fora do sensor. + v00 = 0.0 + v01 = 0.0 + v10 = 0.0 + v11 = 0.0 + + if y0 >= 0 and y0 < h: + if x0 >= 0 and x0 < w: + v00 = float(img[y0, x0]) + if x1 >= 0 and x1 < w: + v01 = float(img[y0, x1]) + + if y1 >= 0 and y1 < h: + if x0 >= 0 and x0 < w: + v10 = float(img[y1, x0]) + if x1 >= 0 and x1 < w: + v11 = float(img[y1, x1]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + + @_numba.njit(cache=True, fastmath=True, inline="always") + def _direct_sample_rgb_fixedmap_numba(rgb, x0, y0, frac_code, channel): + tab = int(frac_code) + fx_i = tab & 31 + fy_i = (tab >> 5) & 31 + + wx1 = fx_i * (1.0 / 32.0) + wy1 = fy_i * (1.0 / 32.0) + wx0 = 1.0 - wx1 + wy0 = 1.0 - wy1 + + h = rgb.shape[0] + w = rgb.shape[1] + x1 = x0 + 1 + y1 = y0 + 1 + + if x0 >= 0 and y0 >= 0 and x1 < w and y1 < h: + v00 = float(rgb[y0, x0, channel]) + v01 = float(rgb[y0, x1, channel]) + v10 = float(rgb[y1, x0, channel]) + v11 = float(rgb[y1, x1, channel]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + v00 = 0.0 + v01 = 0.0 + v10 = 0.0 + v11 = 0.0 + + if y0 >= 0 and y0 < h: + if x0 >= 0 and x0 < w: + v00 = float(rgb[y0, x0, channel]) + if x1 >= 0 and x1 < w: + v01 = float(rgb[y0, x1, channel]) + + if y1 >= 0 and y1 < h: + if x0 >= 0 and x0 < w: + v10 = float(rgb[y1, x0, channel]) + if x1 >= 0 and x1 < w: + v11 = float(rgb[y1, x1, channel]) + + a = v00 * wx0 + v01 * wx1 + b = v10 * wx0 + v11 * wx1 + return a * wy0 + b * wy1 + + + @_numba.njit(cache=True, fastmath=True, parallel=True) + def _direct_fused5_fixedmap_numba( + rgb, + re_img, + nir_img, + rgb_map1, + rgb_map2, + re_map1, + re_map2, + nir_map1, + nir_map2, + out, + ): + """ + RGB + RE + NIR -> tensor CHW [R,G,B,RE,NIR] em uma única passagem. + + - usa os mesmos mapas fixos já cacheados para cv2.remap; + - reproduz a quantização 1/32 do INTER_LINEAR de OpenCV; + - BORDER_CONSTANT = 0.0; + - escreve diretamente no tensor final, sem HWC temporário nem packing. + """ + height = out.shape[1] + width = out.shape[2] + + for y in _numba.prange(height): + for x in range(width): + # RGB compartilha um único par de mapas para os 3 canais. + rx = int(rgb_map1[y, x, 0]) + ry = int(rgb_map1[y, x, 1]) + rf = rgb_map2[y, x] + + out[0, y, x] = _direct_sample_rgb_fixedmap_numba(rgb, rx, ry, rf, 0) + out[1, y, x] = _direct_sample_rgb_fixedmap_numba(rgb, rx, ry, rf, 1) + out[2, y, x] = _direct_sample_rgb_fixedmap_numba(rgb, rx, ry, rf, 2) + + ex = int(re_map1[y, x, 0]) + ey = int(re_map1[y, x, 1]) + ef = re_map2[y, x] + out[3, y, x] = _direct_sample_mono_fixedmap_numba(re_img, ex, ey, ef) + + nx = int(nir_map1[y, x, 0]) + ny = int(nir_map1[y, x, 1]) + nf = nir_map2[y, x] + out[4, y, x] = _direct_sample_mono_fixedmap_numba(nir_img, nx, ny, nf) + + + @_numba.njit(cache=True, fastmath=True, parallel=True) + def _apply_tensor_flat_gain_chw_numba(tensor, gain, channels, height, width, clip_output): + # Raw5 tem só 5 canais; paralelizar apenas em C limita o kernel a 5 + # tarefas. Paralelizando por (canal, linha), CPUs com mais threads + # conseguem trabalhar no tensor final inteiro sem mudar a matemática. + total_rows = channels * height + for cy in _numba.prange(total_rows): + c = cy // height + y = cy - c * height + for x in range(width): + v = float(tensor[c, y, x]) * float(gain[c, y, x]) + + if clip_output: + if v < 0.0: + v = 0.0 + elif v > 1.0: + v = 1.0 + + tensor[c, y, x] = v @@ -613,6 +859,36 @@ class RawProcessorCore: self._last_decode_perf_log_ts = 0.0 self.last_rgb_enhancement_result = None + # ============================================================ + # PERF DIAGNOSTICS - temporariamente habilitado para campo + # ============================================================ + # O profiler é intencionalmente leve: usa apenas perf_counter() e + # metadados já calculados pelo pipeline. Não calcula estatísticas + # adicionais sobre os pixels e não altera o tensor. + # + # Pode ser desligado sem editar código: + # OAK_CORE_PERF_DEBUG=0 + # Intervalos: + # OAK_CORE_PERF_LOG_INTERVAL_S=1.0 + # OAK_CORE_SHAPES_LOG_INTERVAL_S=5.0 + self.core_perf_debug = str( + os.getenv("OAK_CORE_PERF_DEBUG", "1") + ).strip().lower() not in ("0", "false", "no", "off") + try: + self.core_perf_log_interval_s = max(0.2, float( + os.getenv("OAK_CORE_PERF_LOG_INTERVAL_S", "1.0") + )) + except Exception: + self.core_perf_log_interval_s = 1.0 + try: + self.core_shapes_log_interval_s = max(1.0, float( + os.getenv("OAK_CORE_SHAPES_LOG_INTERVAL_S", "5.0") + )) + except Exception: + self.core_shapes_log_interval_s = 5.0 + self._last_core_perf_log_ts = 0.0 + self._last_core_shapes_log_ts = 0.0 + # QC: percentis são diagnóstico, não parte do tensor. # Mantemos saturação/dark/range exatos e limitamos apenas estatísticas # descritivas para não gastar dezenas/centenas de ms por frame. @@ -1813,12 +2089,25 @@ class RawProcessorCore: channels_expected, target_size=None, ): + """ + Fusão Raw5 com telemetria de performance por estágio. + + IMPORTANTE: este profiler não altera a matemática do pipeline. Ele apenas + mede os blocos já existentes e emite logs rate-limited para localizar o + gargalo real no hardware OV9782/AR0234. + """ t_total0 = time.perf_counter() + # ------------------------------------------------------------ + # 1) Dark + # ------------------------------------------------------------ t0 = time.perf_counter() decoded = self.apply_dark_to_decoded(decoded) t_dark_ms = (time.perf_counter() - t0) * 1000.0 + # ------------------------------------------------------------ + # 2) Normalização radiométrica por controles da captura + # ------------------------------------------------------------ t0 = time.perf_counter() decoded = self.normalize_decoded_by_capture_controls(decoded, meta) t_radnorm_ms = (time.perf_counter() - t0) * 1000.0 @@ -1831,26 +2120,50 @@ class RawProcessorCore: and flat_apply_space != "final_tensor_space" ) + # ------------------------------------------------------------ + # 3) Flat-field nativo + # ------------------------------------------------------------ t0 = time.perf_counter() - if apply_native_flat: decoded = self.apply_flat_gain_to_decoded(decoded) - - # ★ NOVO ★ - decoded = self.apply_camera_orientation_to_decoded(decoded) - t_flat_native_ms = (time.perf_counter() - t0) * 1000.0 - # Caminho espacial otimizado: direto para o target final. + # ------------------------------------------------------------ + # 4) Orientação das câmeras + # ------------------------------------------------------------ + # Em produto, uma orientação RGB de 180° pode ser fundida no remap + # RGB final. Isso preserva o espaço canônico usado pelas homografias + # sem materializar uma cópia HWC full-res apenas para cv2.rotate(). + # 90°/270° continuam no caminho físico tradicional porque trocam H/W. + defer_roles = set() + if self._can_defer_rgb_orientation_to_direct_remap_fast(decoded): + defer_roles.add("rgb") + + t0 = time.perf_counter() + decoded = self.apply_camera_orientation_to_decoded( + decoded, + defer_roles=defer_roles, + ) + t_orientation_ms = (time.perf_counter() - t0) * 1000.0 + + # ------------------------------------------------------------ + # 5) Fusão espacial direta para o target final + # ------------------------------------------------------------ + t0 = time.perf_counter() tensor, direct_perf = self._fuse_multispec_direct_to_target_fast( decoded, meta, channels_expected, target_size_override=target_size, ) + t_direct_call_ms = (time.perf_counter() - t0) * 1000.0 - t_flat_ms = float(t_flat_native_ms + float(direct_perf.get("final_flat_ms", 0.0))) + final_flat_ms = float(direct_perf.get("final_flat_ms", 0.0)) + t_flat_ms = float(t_flat_native_ms + final_flat_ms) + # ------------------------------------------------------------ + # 6) Validação/finalização dtype + # ------------------------------------------------------------ t0 = time.perf_counter() if tensor.shape[0] != channels_expected: raise RuntimeError( @@ -1860,18 +2173,15 @@ class RawProcessorCore: out = tensor.astype(np.float32, copy=False) t_final_ms = (time.perf_counter() - t0) * 1000.0 - t_total_ms = (time.perf_counter() - t_total0) * 1000.0 - # Mantém contrato de perf do benchmark. + # Contrato histórico de perf + detalhes novos. t_prepare_ms = float(direct_perf.get("prepare_ms", 0.0)) + t_geometry_ms = float(direct_perf.get("geometry_ms", 0.0)) + t_tensor_alloc_ms = float(direct_perf.get("tensor_alloc_ms", 0.0)) t_rgb_ms = float(direct_perf.get("rgb_crop_resize_ms", 0.0)) t_warp_total_ms = float(direct_perf.get("warp_total_ms", 0.0)) warp_details = direct_perf.get("warp_details_ms", {}) or {} - - # Aqui crop_resize_ms representa somente RGB crop/resize no modo direto. - # O custo espacial total útil para analisar é: - # spatial_direct_ms = rgb_crop_resize_ms + warp_total_ms t_crop_resize_ms = float(direct_perf.get("crop_resize_ms", t_rgb_ms)) t_concat_ms = 0.0 @@ -1883,60 +2193,145 @@ class RawProcessorCore: self.last_fusion_result["perf"] = { "dark_ms": t_dark_ms, "radnorm_ms": t_radnorm_ms, + "flat_native_ms": t_flat_native_ms, + "orientation_ms": t_orientation_ms, "flat_ms": t_flat_ms, + "direct_call_ms": t_direct_call_ms, "prepare_ms": t_prepare_ms, + "geometry_ms": t_geometry_ms, + "tensor_alloc_ms": t_tensor_alloc_ms, + "remap_cache_ms": float(direct_perf.get("remap_cache_ms", 0.0)), "rgb_crop_resize_ms": t_rgb_ms, "warp_total_ms": t_warp_total_ms, "warp_details_ms": warp_details, "crop_resize_ms": t_crop_resize_ms, "concat_ms": t_concat_ms, - "spatial_direct_ms": float(t_rgb_ms + t_warp_total_ms), + "spatial_direct_ms": float(direct_perf.get("spatial_direct_ms", t_rgb_ms + t_warp_total_ms)), + "direct_backend_requested": str(direct_perf.get("direct_backend_requested", "auto")), + "direct_backend": str(direct_perf.get("direct_backend", "opencv_fastpath")), + "fused5_used": bool(direct_perf.get("fused5_used", False)), + "fused5_ms": float(direct_perf.get("fused5_ms", 0.0)), + "fused5_fallback_reason": str(direct_perf.get("fused5_fallback_reason", "")), + "final_flat_ms": final_flat_ms, + "final_flat_prepare_ms": float(direct_perf.get("final_flat_prepare_ms", 0.0)), + "final_flat_apply_ms": float(direct_perf.get("final_flat_apply_ms", 0.0)), "final_ms": t_final_ms, "total_ms": t_total_ms, "decode_perf": getattr(self, "last_decode_perf", {}), + "flat_perf_by_role": ( + (getattr(self, "last_flatfield_result", {}) or {}).get("perf_by_role_ms", {}) + ), + "radnorm_perf_by_role": ( + (getattr(self, "last_radiometric_normalization_result", {}) or {}).get("perf_by_role_ms", {}) + ), + "orientation_perf_by_role": ( + (getattr(self, "last_orientation_result", {}) or {}).get("perf_by_role_ms", {}) + ), + "orientation_deferred_roles": list( + (getattr(self, "last_orientation_result", {}) or {}).get("deferred_roles", []) or [] + ), + "rgb_orientation_fused_into_remap": bool( + (direct_perf or {}).get("rgb_orientation_fused_into_remap", False) + ), } - #if not hasattr(self, "_last_perf_log_ts"): - # self._last_perf_log_ts = 0.0 - #now = time.time() - #if now - self._last_perf_log_ts >= 1.0: - # self._last_perf_log_ts = now - # print( - # "[PERF][CORE_FUSE] " - # f"dark={t_dark_ms:.1f}ms " - # f"radnorm={t_radnorm_ms:.1f}ms " - # f"flat={t_flat_ms:.1f}ms " - # f"prepare={t_prepare_ms:.1f}ms " - # f"rgb_resize={t_rgb_ms:.1f}ms " - # f"warp={t_warp_total_ms:.1f}ms " - # f"warp_re={warp_details.get('re', -1):.1f}ms " - # f"warp_nir={warp_details.get('nir', -1):.1f}ms " - # f"spatial={t_rgb_ms + t_warp_total_ms:.1f}ms " - # f"crop_resize={t_crop_resize_ms:.1f}ms " - # f"concat={t_concat_ms:.1f}ms " - # f"final={t_final_ms:.1f}ms " - # f"total={t_total_ms:.1f}ms " - # f"shape={out.shape}" - # ) + # ------------------------------------------------------------ + # LOG PRINCIPAL: 1x/s por padrão + # ------------------------------------------------------------ + if bool(getattr(self, "core_perf_debug", True)): + now = time.time() + last = float(getattr(self, "_last_core_perf_log_ts", 0.0) or 0.0) + interval = float(getattr(self, "core_perf_log_interval_s", 1.0) or 1.0) - #if not hasattr(self, "_last_remap_perf_log_ts"): - # self._last_remap_perf_log_ts = 0.0 - #now = time.time() - #if now - self._last_remap_perf_log_ts >= 1.0: - # self._last_remap_perf_log_ts = now - # print( - # "[PERF][REMAP] " - # f"enabled={direct_perf.get('remap_enabled')} " - # f"hit={direct_perf.get('remap_cache_hit')} " - # f"cache={direct_perf.get('remap_cache_ms', -1):.2f}ms " - # f"rgb={direct_perf.get('rgb_crop_resize_ms', -1):.2f}ms " - # f"warp={direct_perf.get('warp_total_ms', -1):.2f}ms " - # f"re={(direct_perf.get('warp_details_ms') or {}).get('re', -1):.2f}ms " - # f"nir={(direct_perf.get('warp_details_ms') or {}).get('nir', -1):.2f}ms " - # f"spatial={direct_perf.get('spatial_direct_ms', -1):.2f}ms " - # f"hits={direct_perf.get('remap_cache_hits')} " - # f"misses={direct_perf.get('remap_cache_misses')}" - # ) + if now - last >= interval: + self._last_core_perf_log_ts = now + + decode_perf = getattr(self, "last_decode_perf", {}) or {} + decode_rgb = float((decode_perf.get("rgb") or {}).get("total_ms", 0.0) or 0.0) + decode_re = float((decode_perf.get("re") or {}).get("total_ms", 0.0) or 0.0) + decode_nir = float((decode_perf.get("nir") or {}).get("total_ms", 0.0) or 0.0) + + flat_roles = (getattr(self, "last_flatfield_result", {}) or {}).get("perf_by_role_ms", {}) or {} + rad_result = getattr(self, "last_radiometric_normalization_result", {}) or {} + rad_roles = rad_result.get("perf_by_role_ms", {}) or {} + rad_by_role = rad_result.get("by_role", {}) or {} + ori_roles = (getattr(self, "last_orientation_result", {}) or {}).get("perf_by_role_ms", {}) or {} + + print( + "[PERF][CORE_FRAME] " + f"total={t_total_ms:.2f}ms " + f"dark={t_dark_ms:.2f} " + f"rad={t_radnorm_ms:.2f} " + f"flat_native={t_flat_native_ms:.2f} " + f"orient={t_orientation_ms:.2f} " + f"direct={t_direct_call_ms:.2f} " + f"final={t_final_ms:.2f} | " + f"geom={t_geometry_ms:.2f} " + f"alloc={t_tensor_alloc_ms:.2f} " + f"remap_cache={float(direct_perf.get('remap_cache_ms', 0.0)):.2f} " + f"backend={str(direct_perf.get('direct_backend', 'opencv_fastpath'))} " + f"fused5={float(direct_perf.get('fused5_ms', 0.0)):.2f} " + f"rgb={t_rgb_ms:.2f} " + f"rgb_remap={float(direct_perf.get('rgb_remap_ms', 0.0)):.2f} " + f"rgb_pack={float(direct_perf.get('rgb_pack_ms', 0.0)):.2f} " + f"rgb_backend={str(direct_perf.get('rgb_backend', '-'))} " + f"re={float(warp_details.get('re', 0.0)):.2f} " + f"nir={float(warp_details.get('nir', 0.0)):.2f} " + f"final_flat={final_flat_ms:.2f} " + f"flat_prep={float(direct_perf.get('final_flat_prepare_ms', 0.0)):.2f} " + f"flat_apply={float(direct_perf.get('final_flat_apply_ms', 0.0)):.2f} | " + f"orient_fused_rgb={1 if direct_perf.get('rgb_orientation_fused_into_remap') else 0} " + f"cache geom={'HIT' if direct_perf.get('geometry_cache_hit') else 'MISS'} " + f"remap={'HIT' if direct_perf.get('remap_cache_hit') else 'MISS'} " + f"fallback={str(direct_perf.get('fused5_fallback_reason', '') or '-')} " + f"shape={tuple(out.shape)}" + ) + + print( + "[PERF][CORE_ROLES] " + f"decode(rgb/re/nir)={decode_rgb:.2f}/{decode_re:.2f}/{decode_nir:.2f}ms | " + f"rad(rgb/re/nir)={float(rad_roles.get('rgb', 0.0)):.2f}/" + f"{float(rad_roles.get('re', 0.0)):.2f}/" + f"{float(rad_roles.get('nir', 0.0)):.2f}ms " + f"kernel={str((rad_by_role.get('rgb') or {}).get('kernel_backend', '-'))}/" + f"{str((rad_by_role.get('re') or {}).get('kernel_backend', '-'))}/" + f"{str((rad_by_role.get('nir') or {}).get('kernel_backend', '-'))} | " + f"flat(rgb/re/nir)={float(flat_roles.get('rgb', 0.0)):.2f}/" + f"{float(flat_roles.get('re', 0.0)):.2f}/" + f"{float(flat_roles.get('nir', 0.0)):.2f}ms | " + f"orient(rgb/re/nir)={float(ori_roles.get('rgb', 0.0)):.2f}/" + f"{float(ori_roles.get('re', 0.0)):.2f}/" + f"{float(ori_roles.get('nir', 0.0)):.2f}ms" + ) + + # -------------------------------------------------------- + # LOG DE GEOMETRIA/SHAPES: 1x/5s por padrão + # -------------------------------------------------------- + last_shapes = float(getattr(self, "_last_core_shapes_log_ts", 0.0) or 0.0) + shapes_interval = float(getattr(self, "core_shapes_log_interval_s", 5.0) or 5.0) + if now - last_shapes >= shapes_interval: + self._last_core_shapes_log_ts = now + + role_shapes = {} + try: + for _cam_id, _item in (decoded or {}).items(): + _role = str(_item.get("role") or (_item.get("meta", {}) or {}).get("role") or "?").lower() + _img = _item.get("image") + role_shapes[_role] = tuple(_img.shape) if hasattr(_img, "shape") else None + except Exception: + role_shapes = {} + + print( + "[PERF][CORE_SHAPES] " + f"rgb={role_shapes.get('rgb')} " + f"re={role_shapes.get('re')} " + f"nir={role_shapes.get('nir')} " + f"target={tuple(direct_perf.get('target_size', ())) if direct_perf.get('target_size') else target_size} " + f"crop={direct_perf.get('crop_box')} " + f"remap_shapes={direct_perf.get('remap_map_shapes', {})} " + f"direct_backend={str(direct_perf.get('direct_backend', 'opencv_fastpath'))} " + f"flat_space={flat_apply_space}" + ) return out @@ -3853,12 +4248,8 @@ class RawProcessorCore: # FLAT-FIELD # ============================================================ - if not bool( - (self.flatfield_config or {}).get( - "enabled", - False, - ) - ): + flat_enabled = bool((self.flatfield_config or {}).get("enabled", False)) + if False and not flat_enabled: raise RuntimeError( "Produto exige flatfield_config.enabled=true." ) @@ -3870,10 +4261,11 @@ class RawProcessorCore: ) ).lower() - if flat_space != "native_camera_space": + if flat_space not in ("native_camera_space", "final_tensor_space"): raise RuntimeError( - "Produto exige Flat-Field no espaço nativo; " - f"apply_space={flat_space!r}" + "Flat-Field produto com apply_space inválido; " + f"apply_space={flat_space!r}. " + "Use 'native_camera_space' ou 'final_tensor_space'." ) if bool( @@ -3887,7 +4279,7 @@ class RawProcessorCore: "flatfield_config.subtract_dark=false." ) - if not self.flatfield_loaded: + if flat_enabled and not self.flatfield_loaded: raise RuntimeError( "Flat-field produto não foi carregado " "integralmente." @@ -5250,6 +5642,7 @@ class RawProcessorCore: clip_output = bool(cfg.get("clip_output", True)) corrected = {} + perf_by_role_ms = {} sat_guard_enabled = bool( cfg.get( @@ -5271,6 +5664,7 @@ class RawProcessorCore: ) for cam_id, item in decoded.items(): + t_role0 = time.perf_counter() role = str( item.get("role") or item.get("meta", {}).get("role") @@ -5317,15 +5711,18 @@ class RawProcessorCore: else: saturation_mask = None + channel_ms = {} for idx, ch in enumerate( ("R", "G", "B") ): + t_ch0 = time.perf_counter() out[:, :, idx] = self._apply_flat_gain_single_channel( out[:, :, idx], ch, clip_output=clip_output, saturation_mask=saturation_mask, ) + channel_ms[ch] = (time.perf_counter() - t_ch0) * 1000.0 new_item["image"] = out @@ -5335,6 +5732,7 @@ class RawProcessorCore: "applied": True, "inplace_reused_input": bool(reused), "shape": list(out.shape), + "channel_ms": {k: float(v) for k, v in channel_ms.items()}, } elif role in ("re", "nir"): @@ -5355,12 +5753,14 @@ class RawProcessorCore: else: saturation_mask = None + t_ch0 = time.perf_counter() out = self._apply_flat_gain_single_channel( img_f, ch, clip_output=clip_output, saturation_mask=saturation_mask, ) + ch_ms = (time.perf_counter() - t_ch0) * 1000.0 new_item["image"] = out @@ -5370,6 +5770,7 @@ class RawProcessorCore: "applied": True, "inplace_reused_input": bool(reused), "shape": list(out.shape), + "channel_ms": {ch: float(ch_ms)}, } else: @@ -5393,6 +5794,13 @@ class RawProcessorCore: new_item["meta"] = new_meta corrected[cam_id] = new_item + role_elapsed_ms = (time.perf_counter() - t_role0) * 1000.0 + perf_by_role_ms[role] = float(role_elapsed_ms) + if role in result["by_role"] and isinstance(result["by_role"][role], dict): + result["by_role"][role]["total_ms"] = float(role_elapsed_ms) + + result["perf_by_role_ms"] = dict(perf_by_role_ms) + required_roles = { "rgb", "re", @@ -5765,8 +6173,10 @@ class RawProcessorCore: debug_by_role = {} normalized = {} + perf_by_role_ms = {} for cam_id, item in decoded.items(): + t_role0 = time.perf_counter() role = str(item.get("role") or item.get("meta", {}).get("role") or "").lower() img = item.get("image") @@ -5880,6 +6290,9 @@ class RawProcessorCore: new_item = dict(item) new_meta = dict(item.get("meta", {}) or {}) + debug["kernel_backend"] = str( + getattr(self, "_last_radiometric_kernel_backend", "unknown") + ) debug["inplace_reused_input"] = bool(reused) debug["image_shape"] = list(out.shape) if hasattr(out, "shape") else None debug["image_dtype"] = str(out.dtype) if hasattr(out, "dtype") else None @@ -5894,10 +6307,16 @@ class RawProcessorCore: new_item["meta"] = new_meta normalized[cam_id] = new_item + role_elapsed_ms = (time.perf_counter() - t_role0) * 1000.0 + debug["perf_ms"] = float(role_elapsed_ms) + perf_by_role_ms[role] = float(role_elapsed_ms) + result["applied"] = True result["by_camera"][cam_id] = debug result["by_role"][role] = debug + result["perf_by_role_ms"] = dict(perf_by_role_ms) + if result["applied"]: scales = [v.get("scale_applied") for v in result["by_camera"].values() if isinstance(v, dict)] scales = [float(s) for s in scales if s is not None] @@ -6440,7 +6859,15 @@ class RawProcessorCore: def _apply_radiometric_scale_inplace(self, img, scale: float, clip_output: bool): """ - Aplica escala radiométrica com o mínimo possível de alocação. + Aplica escala radiometrica com o minimo possivel de alocacao. + + Fast path: + - float32 gravavel; + - C-contiguous; + - imagem 2D ou 3D; + - kernel Numba em uma unica passada de memoria. + + Fallback NumPy preservado para qualquer layout atipico. Retorna: out, reused_input @@ -6448,22 +6875,45 @@ class RawProcessorCore: out, reused = self._radiometric_get_writable_float32_image(img) if out is None: + self._last_radiometric_kernel_backend = "none" return img, False - scale = float(scale) + scale = np.float32(float(scale)) - # Se a escala é praticamente 1 e não precisa clipar, não faz nada. - if abs(scale - 1.0) <= 1e-6 and not clip_output: + # Preserva exatamente o comportamento historico para scale ~= 1: + # nao multiplica; apenas clipa quando solicitado. + if abs(float(scale) - 1.0) <= 1e-6: + if clip_output: + np.clip(out, 0.0, 1.0, out=out) + self._last_radiometric_kernel_backend = "identity_clip_numpy" + else: + self._last_radiometric_kernel_backend = "identity" return out, reused - # Multiplicação in-place. - if abs(scale - 1.0) > 1e-6: - np.multiply(out, np.float32(scale), out=out, casting="unsafe") + use_numba = ( + _HAS_NUMBA + and out.dtype == np.float32 + and out.flags.c_contiguous + and out.ndim in (2, 3) + ) + + if use_numba: + _apply_radiometric_scale_inplace_numba( + out, + scale, + bool(clip_output), + ) + self._last_radiometric_kernel_backend = "numba_single_pass" + return out, reused + + # Fallback historico. + if abs(float(scale) - 1.0) > 1e-6: + np.multiply(out, scale, out=out, casting="unsafe") - # Clip in-place, se configurado. if clip_output: np.clip(out, 0.0, 1.0, out=out) + self._last_radiometric_kernel_backend = "numpy_ufunc" return out, reused def _apply_radiometric_affine_inplace( @@ -6474,22 +6924,54 @@ class RawProcessorCore: clip_output: bool, ): """ - Aplica o contrato radiométrico affine_v2 no domínio RAW01: + Aplica o contrato radiometrico affine_v2 no dominio RAW01: offset + (value - offset) * reference_factor / actual_factor - O mesmo offset escalar é aplicado aos três canais RGB da role RGB, + O mesmo offset escalar e aplicado aos tres canais RGB da role RGB, conforme black_offset_model='per_role_scalar_raw01'. + + No caminho quente, o affine inteiro e executado por um kernel Numba + single-pass. Isso elimina as tres varreduras completas do ndarray + (subtract -> multiply -> add) sem alterar a ordem matematica por pixel. """ out, reused = self._radiometric_get_writable_float32_image(img) if out is None: + self._last_radiometric_kernel_backend = "none" return img, False scale = np.float32(float(scale)) offset = np.float32(float(black_offset)) - # Para scale=1 a transformação é identidade, independentemente do offset. + # Preserva exatamente o comportamento historico para scale ~= 1: + # o affine nao roda; apenas o clip final, quando habilitado. + if abs(float(scale) - 1.0) <= 1e-6: + if clip_output: + np.clip(out, 0.0, 1.0, out=out) + self._last_radiometric_kernel_backend = "identity_clip_numpy" + else: + self._last_radiometric_kernel_backend = "identity" + return out, reused + + use_numba = ( + _HAS_NUMBA + and out.dtype == np.float32 + and out.flags.c_contiguous + and out.ndim in (2, 3) + ) + + if use_numba: + _apply_radiometric_affine_inplace_numba( + out, + scale, + offset, + bool(clip_output), + ) + self._last_radiometric_kernel_backend = "numba_single_pass" + return out, reused + + # Fallback historico: mesma matematica em tres ufuncs in-place. if abs(float(scale) - 1.0) > 1e-6: np.subtract(out, offset, out=out, casting="unsafe") np.multiply(out, scale, out=out, casting="unsafe") @@ -6498,6 +6980,7 @@ class RawProcessorCore: if clip_output: np.clip(out, 0.0, 1.0, out=out) + self._last_radiometric_kernel_backend = "numpy_ufunc" return out, reused def _build_radiometric_debug( @@ -7128,6 +7611,128 @@ class RawProcessorCore: tensor[int(channel_index)] = out + def _direct_fusion_can_use_numba_fused5_fast( + self, + decoded, + role_to_cam, + remap_cache, + channels_expected, + use_remap_cache, + use_remap_for_rgb, + use_remap_for_spec, + ): + """ + Valida o contrato mínimo para o backend Numba fused 5ch. + + O backend é intencionalmente conservador: se qualquer premissa não for + satisfeita, o caller volta automaticamente ao caminho OpenCV anterior. + """ + if not _HAS_NUMBA: + return False, "numba_unavailable" + + if int(channels_expected) != 5: + return False, f"channels_expected={channels_expected}" + + if not bool(use_remap_cache): + return False, "remap_cache_disabled" + + if not bool(use_remap_for_rgb): + return False, "rgb_remap_disabled" + + if not bool(use_remap_for_spec): + return False, "spec_remap_disabled" + + maps = (remap_cache or {}).get("maps", {}) or {} + for role in ("rgb", "re", "nir"): + if role not in role_to_cam: + return False, f"missing_role:{role}" + + entry = maps.get(role) + if not isinstance(entry, dict): + return False, f"missing_maps:{role}" + + map1 = entry.get("map1") + map2 = entry.get("map2") + if not isinstance(map1, np.ndarray) or not isinstance(map2, np.ndarray): + return False, f"invalid_maps:{role}" + if map1.ndim != 3 or map1.shape[2] != 2: + return False, f"invalid_map1_shape:{role}:{getattr(map1, 'shape', None)}" + if map2.ndim != 2 or map2.shape[:2] != map1.shape[:2]: + return False, f"invalid_map2_shape:{role}:{getattr(map2, 'shape', None)}" + + rgb = decoded[role_to_cam["rgb"]].get("image") + re_img = decoded[role_to_cam["re"]].get("image") + nir_img = decoded[role_to_cam["nir"]].get("image") + + if not isinstance(rgb, np.ndarray) or rgb.ndim != 3 or rgb.shape[2] != 3: + return False, "invalid_rgb" + if not isinstance(re_img, np.ndarray) or re_img.ndim != 2: + return False, "invalid_re" + if not isinstance(nir_img, np.ndarray) or nir_img.ndim != 2: + return False, "invalid_nir" + + # Produto já trabalha em float32 após decode/radiometria. Evitamos + # conversões/copias escondidas dentro do backend fused. + if rgb.dtype != np.float32: + return False, f"rgb_dtype={rgb.dtype}" + if re_img.dtype != np.float32: + return False, f"re_dtype={re_img.dtype}" + if nir_img.dtype != np.float32: + return False, f"nir_dtype={nir_img.dtype}" + + dst_shape = maps["rgb"]["map2"].shape + if maps["re"]["map2"].shape != dst_shape or maps["nir"]["map2"].shape != dst_shape: + return False, "target_map_shape_mismatch" + + return True, "ok" + + + def _direct_fusion_write_numba_fused5_fast( + self, + tensor, + decoded, + role_to_cam, + remap_cache, + ): + """Executa RGB+RE+NIR -> Raw5 CHW em uma única chamada Numba.""" + maps = remap_cache["maps"] + + rgb = decoded[role_to_cam["rgb"]]["image"] + re_img = decoded[role_to_cam["re"]]["image"] + nir_img = decoded[role_to_cam["nir"]]["image"] + + rgb_maps = maps["rgb"] + re_maps = maps["re"] + nir_maps = maps["nir"] + + # Se o backend OpenCV atual usaria warpAffine para o RGB, usamos o + # fixed-map equivalente ao warpAffine. Caso contrário, usamos o remap + # normal. Assim o fused compara com a ÚLTIMA versão, não com uma etapa + # anterior do pipeline. + rgb_map1 = rgb_maps.get("warp_affine_map1", rgb_maps["map1"]) + rgb_map2 = rgb_maps.get("warp_affine_map2", rgb_maps["map2"]) + + t0 = time.perf_counter() + _direct_fused5_fixedmap_numba( + rgb, + re_img, + nir_img, + rgb_map1, + rgb_map2, + re_maps["map1"], + re_maps["map2"], + nir_maps["map1"], + nir_maps["map2"], + tensor, + ) + elapsed_ms = (time.perf_counter() - t0) * 1000.0 + + return { + "fused5_ms": float(elapsed_ms), + "backend": "numba_fused_5ch_fixedmap", + } + + def _fuse_multispec_direct_to_target_fast( self, decoded, @@ -7138,12 +7743,13 @@ class RawProcessorCore: """ Fusão direta otimizada com cache de geometria fixa. - target_size_override permite ir diretamente ao tamanho final pedido - pelo treino/inferência, evitando um segundo resize posterior. + Esta versão adiciona somente telemetria granular. A matemática continua: + RGB crop/resize + RE/NIR remap direto para o target final. """ channels_expected = self._validate_physical_channel_count(channels_expected) - t0 = time.perf_counter() + t_total0 = time.perf_counter() + t0 = time.perf_counter() rgb_cam_id = self._find_cam_by_role(decoded, "rgb") if rgb_cam_id is None: raise RuntimeError("Fusão direta requer câmera com role='rgb' como referência") @@ -7165,6 +7771,7 @@ class RawProcessorCore: for cam_id, data in decoded.items() for item in [data] } + t_role_map_ms = (time.perf_counter() - t0) * 1000.0 missing_roles = [role for role in ("rgb", "re", "nir") if role not in role_to_cam] if missing_roles: @@ -7173,7 +7780,10 @@ class RawProcessorCore: f"{missing_roles}. Recebidos={sorted(str(role) for role in role_to_cam)}" ) - # Cache da geometria fixa. + # ------------------------------------------------------------ + # Geometria fixa/cacheada + # ------------------------------------------------------------ + t0_geom = time.perf_counter() geom = self._direct_fusion_get_geometry_cached_fast( decoded=decoded, role_to_cam=role_to_cam, @@ -7181,6 +7791,7 @@ class RawProcessorCore: target_size=target_size, meta=meta, ) + t_geometry_ms = (time.perf_counter() - t0_geom) * 1000.0 crop_box = geom["crop_box"] crop_applied = bool(geom["crop_applied"]) @@ -7200,9 +7811,16 @@ class RawProcessorCore: "homography_profiles_used": geom.get("homography_profiles_used", {}), } + # Alocação do tensor separada do preparo geométrico. + t0_alloc = time.perf_counter() tensor = np.empty((channels_expected, target_h, target_w), dtype=np.float32) + t_tensor_alloc_ms = (time.perf_counter() - t0_alloc) * 1000.0 - t_prepare_ms = (time.perf_counter() - t0) * 1000.0 + t_prepare_ms = ( + t_role_map_ms + + t_geometry_ms + + t_tensor_alloc_ms + ) # ------------------------------------------------------------ # Cache de remap @@ -7211,8 +7829,22 @@ class RawProcessorCore: use_remap_for_rgb = bool((self.fusion_config or {}).get("use_remap_for_rgb", False)) use_remap_for_spec = bool((self.fusion_config or {}).get("use_remap_for_spec", True)) - t0_remap_cache = time.perf_counter() + rgb_orientation_fused = self._decoded_role_orientation_is_deferred_fast( + decoded, + role_to_cam, + "rgb", + ) + # Se a orientação RGB foi adiada, o caminho RGB obrigatoriamente precisa + # usar o remap cacheado que contém a matriz de orientação composta. + if rgb_orientation_fused: + if not use_remap_cache: + raise RuntimeError( + "Orientação RGB diferida requer fusion_config.use_remap_cache=true." + ) + use_remap_for_rgb = True + + t0_remap_cache = time.perf_counter() if use_remap_cache and (use_remap_for_rgb or use_remap_for_spec): remap_cache = self._direct_fusion_get_remap_cached_fast( decoded=decoded, @@ -7228,101 +7860,165 @@ class RawProcessorCore: "cache_hits": 0, "cache_misses": 0, } - t_remap_cache_ms = (time.perf_counter() - t0_remap_cache) * 1000.0 - # ------------------------------------------------------------ - # RGB via remap - # ------------------------------------------------------------ - t0_rgb = time.perf_counter() + remap_map_shapes = {} + try: + for role, entry in (remap_cache.get("maps", {}) or {}).items(): + if isinstance(entry, dict): + m1 = entry.get("map1") + m2 = entry.get("map2") + remap_map_shapes[str(role)] = { + "map1": list(m1.shape) if hasattr(m1, "shape") else None, + "map2": list(m2.shape) if hasattr(m2, "shape") else None, + } + except Exception: + remap_map_shapes = {} - rgb_maps = remap_cache["maps"].get("rgb") - if use_remap_cache and use_remap_for_rgb and rgb_maps is not None: - self._direct_fusion_write_rgb_remap_fast( - tensor=tensor, - rgb=rgb, - remap_entry=rgb_maps, - ) - else: - self._direct_fusion_write_rgb_fast( - tensor, - rgb, - crop_box, - target_size, + # ------------------------------------------------------------ + # Backend espacial: Numba fused 5ch ou OpenCV conservador + # ------------------------------------------------------------ + direct_backend_requested = str( + (self.fusion_config or {}).get("direct_backend", "auto") or "auto" + ).strip().lower() + + force_opencv = direct_backend_requested in ( + "opencv", + "cv2", + "legacy", + "opencv_fastpath", + ) + request_numba = direct_backend_requested in ( + "auto", + "numba", + "numba5", + "fused5", + "numba_fused_5ch", + ) + + if not force_opencv and not request_numba: + # Valor desconhecido: fail-safe para OpenCV e registra o motivo. + force_opencv = True + + fused5_used = False + fused5_ms = 0.0 + fused5_fallback_reason = "forced_opencv" if force_opencv else "not_attempted" + + if request_numba and not force_opencv: + can_fused5, fused5_fallback_reason = self._direct_fusion_can_use_numba_fused5_fast( + decoded=decoded, + role_to_cam=role_to_cam, + remap_cache=remap_cache, + channels_expected=channels_expected, + use_remap_cache=use_remap_cache, + use_remap_for_rgb=use_remap_for_rgb, + use_remap_for_spec=use_remap_for_spec, ) - t_rgb_ms = (time.perf_counter() - t0_rgb) * 1000.0 + if can_fused5: + fused_perf = self._direct_fusion_write_numba_fused5_fast( + tensor=tensor, + decoded=decoded, + role_to_cam=role_to_cam, + remap_cache=remap_cache, + ) + fused5_used = True + fused5_ms = float(fused_perf.get("fused5_ms", 0.0)) + fused5_fallback_reason = "" + # Perf fields históricos. No fused, o trabalho RGB/RE/NIR acontece + # inseparavelmente dentro de fused5_ms, por isso os subtempos ficam 0. + rgb_remap_perf = { + "remap_ms": 0.0, + "pack_ms": 0.0, + "backend": ( + "numba_fused_5ch_fixedmap" if fused5_used else "legacy_crop_resize" + ), + } warp_details = {} t_warp_total_ms = 0.0 + t_rgb_ms = 0.0 - # ------------------------------------------------------------ - # RE via remap - # ------------------------------------------------------------ - if "re" in role_to_cam: - t0w = time.perf_counter() - - re_img = decoded[role_to_cam["re"]]["image"] - re_maps = remap_cache["maps"].get("re") - - if use_remap_cache and use_remap_for_spec and re_maps is not None: - self._direct_fusion_write_spec_remap_fast( + if not fused5_used: + # -------------------------------------------------------- + # OpenCV fast-path anterior, preservado como fallback/A-B + # -------------------------------------------------------- + t0_rgb = time.perf_counter() + rgb_maps = remap_cache["maps"].get("rgb") + if use_remap_cache and use_remap_for_rgb and rgb_maps is not None: + rgb_remap_perf = self._direct_fusion_write_rgb_remap_fast( tensor=tensor, - channel_index=3, - img=re_img, - remap_entry=re_maps, + rgb=rgb, + remap_entry=rgb_maps, ) else: - self._direct_fusion_write_spec_cached_fast( - tensor=tensor, - channel_index=3, - img=re_img, - role="re", - geom=geom, - ref_size=ref_size, - target_size=target_size, + self._direct_fusion_write_rgb_fast( + tensor, + rgb, + crop_box, + target_size, ) + t_rgb_ms = (time.perf_counter() - t0_rgb) * 1000.0 - warp_details["re"] = (time.perf_counter() - t0w) * 1000.0 - t_warp_total_ms += warp_details["re"] + if "re" in role_to_cam: + t0w = time.perf_counter() + re_img = decoded[role_to_cam["re"]]["image"] + re_maps = remap_cache["maps"].get("re") - # ------------------------------------------------------------ - # NIR via remap - # ------------------------------------------------------------ - if "nir" in role_to_cam: - t0w = time.perf_counter() + if use_remap_cache and use_remap_for_spec and re_maps is not None: + self._direct_fusion_write_spec_remap_fast( + tensor=tensor, + channel_index=3, + img=re_img, + remap_entry=re_maps, + ) + else: + self._direct_fusion_write_spec_cached_fast( + tensor=tensor, + channel_index=3, + img=re_img, + role="re", + geom=geom, + ref_size=ref_size, + target_size=target_size, + ) - nir_img = decoded[role_to_cam["nir"]]["image"] - nir_maps = remap_cache["maps"].get("nir") + warp_details["re"] = (time.perf_counter() - t0w) * 1000.0 + t_warp_total_ms += warp_details["re"] - if use_remap_cache and use_remap_for_spec and nir_maps is not None: - self._direct_fusion_write_spec_remap_fast( - tensor=tensor, - channel_index=4, - img=nir_img, - remap_entry=nir_maps, - ) - else: - self._direct_fusion_write_spec_cached_fast( - tensor=tensor, - channel_index=4, - img=nir_img, - role="nir", - geom=geom, - ref_size=ref_size, - target_size=target_size, - ) + if "nir" in role_to_cam: + t0w = time.perf_counter() + nir_img = decoded[role_to_cam["nir"]]["image"] + nir_maps = remap_cache["maps"].get("nir") - warp_details["nir"] = (time.perf_counter() - t0w) * 1000.0 - t_warp_total_ms += warp_details["nir"] + if use_remap_cache and use_remap_for_spec and nir_maps is not None: + self._direct_fusion_write_spec_remap_fast( + tensor=tensor, + channel_index=4, + img=nir_img, + remap_entry=nir_maps, + ) + else: + self._direct_fusion_write_spec_cached_fast( + tensor=tensor, + channel_index=4, + img=nir_img, + role="nir", + geom=geom, + ref_size=ref_size, + target_size=target_size, + ) + + warp_details["nir"] = (time.perf_counter() - t0w) * 1000.0 + t_warp_total_ms += warp_details["nir"] # ------------------------------------------------------------ # Flat-field no espaço final do tensor # ------------------------------------------------------------ - t0_final_flat = time.perf_counter() - final_flat_ms = 0.0 final_flat_enabled = False final_flat_cache_available = False + final_flat_prepare_ms = 0.0 + final_flat_apply_ms = 0.0 flat_cfg = self.flatfield_config or {} apply_final_flat = ( @@ -7333,6 +8029,7 @@ class RawProcessorCore: if apply_final_flat: final_flat_enabled = True + t0_flat_prepare = time.perf_counter() gain_tensor = self._get_final_flat_gain_tensor_cached_fast( decoded=decoded, role_to_cam=role_to_cam, @@ -7341,16 +8038,41 @@ class RawProcessorCore: crop_box=crop_box, target_size=target_size, ) + final_flat_prepare_ms = (time.perf_counter() - t0_flat_prepare) * 1000.0 if gain_tensor is not None: final_flat_cache_available = True + t0_flat_apply = time.perf_counter() tensor = self._apply_final_flat_gain_tensor_inplace( tensor=tensor, gain_tensor=gain_tensor, clip_output=bool(flat_cfg.get("clip_output", True)), ) + final_flat_apply_ms = (time.perf_counter() - t0_flat_apply) * 1000.0 - final_flat_ms = (time.perf_counter() - t0_final_flat) * 1000.0 + self.last_flatfield_result = { + "enabled": True, + "applied": True, + "apply_space": "final_tensor_space", + "backend": "final_tensor_gain_numba" if _HAS_NUMBA else "final_tensor_gain_numpy", + "warnings": ( + ["saturation_guard_not_applied_in_final_tensor_space"] + if bool(flat_cfg.get("saturation_guard_enabled", True)) + else [] + ), + "by_role": { + "rgb": {"applied": True, "channels": ["R", "G", "B"]}, + "re": {"applied": True, "channels": ["RE"]}, + "nir": {"applied": True, "channels": ["NIR"]}, + }, + "applied_roles": ["nir", "re", "rgb"], + "missing_roles": [], + "perf_by_role_ms": {}, + "prepare_ms": float(final_flat_prepare_ms), + "apply_ms": float(final_flat_apply_ms), + } + + final_flat_ms = float(final_flat_prepare_ms + final_flat_apply_ms) if tensor.shape[0] != channels_expected: raise RuntimeError( @@ -7358,15 +8080,29 @@ class RawProcessorCore: ) self.last_fusion_result["output_shape"] = list(tensor.shape) + t_direct_total_ms = (time.perf_counter() - t_total0) * 1000.0 perf = { "prepare_ms": float(t_prepare_ms), + "role_map_ms": float(t_role_map_ms), + "geometry_ms": float(t_geometry_ms), + "tensor_alloc_ms": float(t_tensor_alloc_ms), "rgb_crop_resize_ms": float(t_rgb_ms), + "rgb_remap_ms": float(rgb_remap_perf.get("remap_ms", 0.0)), + "rgb_pack_ms": float(rgb_remap_perf.get("pack_ms", 0.0)), + "rgb_backend": str(rgb_remap_perf.get("backend", "unknown")), "warp_total_ms": float(t_warp_total_ms), "warp_details_ms": warp_details, "crop_resize_ms": float(t_rgb_ms), "concat_ms": 0.0, - "spatial_direct_ms": float(t_rgb_ms + t_warp_total_ms), + "spatial_direct_ms": float(fused5_ms if fused5_used else (t_rgb_ms + t_warp_total_ms)), + "direct_backend_requested": str(direct_backend_requested), + "direct_backend": ( + "numba_fused_5ch_fixedmap" if fused5_used else "opencv_fastpath" + ), + "fused5_used": bool(fused5_used), + "fused5_ms": float(fused5_ms), + "fused5_fallback_reason": str(fused5_fallback_reason or ""), "geometry_cache_hit": bool(geom.get("prepare_cache_hit", False)), "geometry_cache_hits": int(geom.get("cache_hits", 0)), "geometry_cache_misses": int(geom.get("cache_misses", 0)), @@ -7377,15 +8113,20 @@ class RawProcessorCore: "remap_enabled": bool(use_remap_cache), "remap_rgb_enabled": bool(use_remap_cache and use_remap_for_rgb), "remap_spec_enabled": bool(use_remap_cache and use_remap_for_spec), + "rgb_orientation_fused_into_remap": bool(rgb_orientation_fused), + "remap_map_shapes": remap_map_shapes, "final_flat_enabled": bool(final_flat_enabled), "final_flat_cache_available": bool(final_flat_cache_available), "final_flat_ms": float(final_flat_ms), + "final_flat_prepare_ms": float(final_flat_prepare_ms), + "final_flat_apply_ms": float(final_flat_apply_ms), + "target_size": [int(target_w), int(target_h)], + "crop_box": [int(v) for v in crop_box], + "direct_total_ms": float(t_direct_total_ms), } return tensor, perf - - def _direct_fusion_get_geometry_cache_key_fast( self, decoded, @@ -7457,6 +8198,9 @@ class RawProcessorCore: self._direct_fusion_remap_cache_hits = 0 self._direct_fusion_remap_cache_misses = 0 + # Buffers de destino dependem do target final. Recriados sob demanda. + self._direct_fusion_runtime_buffers = {} + def _direct_fusion_get_geometry_cached_fast( self, decoded, @@ -7609,6 +8353,70 @@ class RawProcessorCore: tensor[int(channel_index)] = out + def _direct_fusion_build_warp_affine_fixedmap_fast( + self, + M_src_to_dst: np.ndarray, + dst_size: tuple, + ): + """ + Materializa uma vez o mesmo fixed-point map usado por cv2.warpAffine + com INTER_LINEAR. Isso permite ao kernel fused Numba reproduzir também + o fast-path RGB atual, em vez de voltar à convenção do cv2.remap. + """ + M = np.asarray(M_src_to_dst, dtype=np.float64) + if M.shape == (3, 3): + A = M[:2, :] + elif M.shape == (2, 3): + A = M + else: + raise RuntimeError(f"M affine inválida: shape={M.shape}") + + target_w, target_h = int(dst_size[0]), int(dst_size[1]) + inv = cv2.invertAffineTransform(A).astype(np.float64, copy=False) + + inter_bits = 5 + inter_tab = 1 << inter_bits + ab_bits = max(10, inter_bits) + ab_scale = 1 << ab_bits + round_delta = ab_scale // inter_tab // 2 + shift = ab_bits - inter_bits + + xs = np.arange(target_w, dtype=np.float64) + ys = np.arange(target_h, dtype=np.float64) + + # np.rint reproduz o arredondamento para o inteiro mais próximo usado + # pelo cvRound/saturate_cast neste domínio de coordenadas. + adelta = np.rint(inv[0, 0] * xs * ab_scale).astype(np.int64) + bdelta = np.rint(inv[1, 0] * xs * ab_scale).astype(np.int64) + + x0 = ( + np.rint((inv[0, 1] * ys + inv[0, 2]) * ab_scale).astype(np.int64) + + int(round_delta) + ) + y0 = ( + np.rint((inv[1, 1] * ys + inv[1, 2]) * ab_scale).astype(np.int64) + + int(round_delta) + ) + + X = (x0[:, None] + adelta[None, :]) >> shift + Y = (y0[:, None] + bdelta[None, :]) >> shift + + ix = X >> inter_bits + iy = Y >> inter_bits + fx = X & (inter_tab - 1) + fy = Y & (inter_tab - 1) + + # O produto opera em coordenadas pequenas (< 2k), mas clamp explícito + # preserva o contrato de armazenamento CV_16SC2. + i16 = np.iinfo(np.int16) + map1 = np.empty((target_h, target_w, 2), dtype=np.int16) + map1[:, :, 0] = np.clip(ix, i16.min, i16.max).astype(np.int16) + map1[:, :, 1] = np.clip(iy, i16.min, i16.max).astype(np.int16) + map2 = (fy * inter_tab + fx).astype(np.uint16) + + return map1, map2 + + def _direct_fusion_build_remap_from_src_to_dst_fast( self, M_src_to_dst: np.ndarray, @@ -7672,15 +8480,41 @@ class RawProcessorCore: # convertMaps deixa o remap mais barato em muitos casos. map1, map2 = cv2.convertMaps(map_x, map_y, cv2.CV_16SC2) - return { + entry = { "map_x": map_x, "map_y": map_y, "map1": map1, "map2": map2, + # Mantemos também a matriz forward original. Para RGB, quando ela + # for affine e axis-aligned (crop/scale + 0/180/flips), podemos usar + # warpAffine no runtime e evitar ler os mapas densos por frame. + "M_src_to_dst": M.astype(np.float32, copy=False), "src_shape": [src_h, src_w], "dst_size": [target_w, target_h], } + eps = 1e-7 + axis_affine = bool( + abs(float(M[0, 1])) <= eps + and abs(float(M[1, 0])) <= eps + and abs(float(M[2, 0])) <= eps + and abs(float(M[2, 1])) <= eps + and abs(float(M[2, 2]) - 1.0) <= eps + ) + + if axis_affine: + warp_map1, warp_map2 = self._direct_fusion_build_warp_affine_fixedmap_fast( + M_src_to_dst=M, + dst_size=(target_w, target_h), + ) + entry["warp_affine_map1"] = warp_map1 + entry["warp_affine_map2"] = warp_map2 + entry["axis_affine"] = True + else: + entry["axis_affine"] = False + + return entry + def _direct_fusion_get_remap_cache_key_fast( self, geom: dict, @@ -7704,11 +8538,18 @@ class RawProcessorCore: img = decoded[cam_id]["image"] role_shapes[str(role).lower()] = tuple(int(v) for v in img.shape[:2]) + rgb_orientation_sig = self._decoded_role_orientation_signature_fast( + decoded, + role_to_cam, + "rgb", + ) + return ( geom.get("key"), tuple(int(v) for v in ref_size), tuple(int(v) for v in target_size), tuple(sorted(role_shapes.items())), + rgb_orientation_sig, ) def _direct_fusion_get_remap_cached_fast( @@ -7763,8 +8604,36 @@ class RawProcessorCore: rgb_cam_id = role_to_cam.get("rgb") if rgb_cam_id is not None: rgb_img = decoded[rgb_cam_id]["image"] + + # geom["C_ref_to_target"] parte do espaço RGB CANÔNICO (após + # camera_orientation). Quando a orientação foi diferida, compomos: + # + # RGB_native --O--> RGB_canonical --C--> target + # + # Como cv2.remap usa o inverso internamente, o target passa a + # amostrar diretamente o frame nativo já na orientação correta. + M_rgb_to_target = geom["C_ref_to_target"] + if self._decoded_role_orientation_is_deferred_fast( + decoded, role_to_cam, "rgb" + ): + meta_rgb = decoded[rgb_cam_id].get("meta", {}) or {} + ori = meta_rgb.get("orientation_applied", {}) or {} + O_native_to_canonical = self._orientation_matrix_same_shape_fast( + src_shape=rgb_img.shape[:2], + rotate_deg=int(ori.get("rotate_deg", 0)), + flip_horizontal=bool(ori.get("flip_horizontal", False)), + flip_vertical=bool(ori.get("flip_vertical", False)), + ) + if O_native_to_canonical is None: + raise RuntimeError( + "Orientação RGB diferida não suportada pelo remap composto." + ) + M_rgb_to_target = ( + geom["C_ref_to_target"] @ O_native_to_canonical + ).astype(np.float32) + remap["maps"]["rgb"] = self._direct_fusion_build_remap_from_src_to_dst_fast( - M_src_to_dst=geom["C_ref_to_target"], + M_src_to_dst=M_rgb_to_target, src_shape=rgb_img.shape[:2], dst_size=(target_w, target_h), ref_shape_for_scaled_src=None, @@ -7794,6 +8663,43 @@ class RawProcessorCore: cache[key] = remap return remap + def _direct_fusion_get_rgb_scratch_fast(self, target_h: int, target_w: int): + """ + Buffer HWC float32 reutilizável para o remap RGB. + + O tensor final é CHW, enquanto cv2.remap é mais eficiente processando o + RGB intercalado em uma única chamada. Reutilizamos este scratch para + eliminar a alocação HWC por frame e depois usamos cv2.mixChannels para + copiar os 3 canais diretamente para os planos CHW do tensor. + """ + target_h = int(target_h) + target_w = int(target_w) + key = (target_h, target_w) + + cache = getattr(self, "_direct_fusion_runtime_buffers", None) + if not isinstance(cache, dict): + cache = {} + self._direct_fusion_runtime_buffers = cache + + entry = cache.get(key) + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("rgb_hwc"), np.ndarray) + or entry["rgb_hwc"].shape != (target_h, target_w, 3) + or entry["rgb_hwc"].dtype != np.float32 + ): + entry = { + "rgb_hwc": np.empty( + (target_h, target_w, 3), + dtype=np.float32, + ) + } + if len(cache) >= 4: + cache.clear() + cache[key] = entry + + return entry["rgb_hwc"] + def _direct_fusion_write_rgb_remap_fast( self, tensor: np.ndarray, @@ -7801,23 +8707,81 @@ class RawProcessorCore: remap_entry: dict, ): """ - RGB HWC -> tensor CHW usando cv2.remap direto para target final. + RGB HWC -> tensor CHW usando o mesmo cv2.remap do caminho anterior, + porém sem alocar um HWC novo em todo frame. + + Matemática/interpolação permanecem idênticas: mesmos map1/map2, + INTER_LINEAR e BORDER_CONSTANT. Somente o destino e o empacotamento + HWC->CHW foram otimizados. """ map1 = remap_entry["map1"] map2 = remap_entry["map2"] - rgb_out = cv2.remap( - rgb.astype(np.float32, copy=False), - map1, - map2, - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=0.0, + target_h, target_w = int(map1.shape[0]), int(map1.shape[1]) + rgb_out = self._direct_fusion_get_rgb_scratch_fast( + target_h, + target_w, ) - tensor[0] = rgb_out[:, :, 0] - tensor[1] = rgb_out[:, :, 1] - tensor[2] = rgb_out[:, :, 2] + rgb_f = rgb.astype(np.float32, copy=False) + M = remap_entry.get("M_src_to_dst") + + # Fast-path exato para a geometria RGB de produto atual: + # crop/scale + orientação 0/180/flips => matriz affine axis-aligned. + # Para essa família, OpenCV warpAffine e o remap fixo gerado por + # convertMaps produzem os mesmos pixels, mas warpAffine evita ler + # map1/map2 densos a cada frame. + use_axis_affine = False + if isinstance(M, np.ndarray) and M.shape == (3, 3): + eps = 1e-7 + use_axis_affine = bool( + abs(float(M[0, 1])) <= eps + and abs(float(M[1, 0])) <= eps + and abs(float(M[2, 0])) <= eps + and abs(float(M[2, 1])) <= eps + and abs(float(M[2, 2]) - 1.0) <= eps + ) + + t0 = time.perf_counter() + if use_axis_affine: + cv2.warpAffine( + rgb_f, + M[:2], + (target_w, target_h), + dst=rgb_out, + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=0.0, + ) + backend = "opencv_warpAffine_axis_aligned_cached_hwc" + else: + cv2.remap( + rgb_f, + map1, + map2, + interpolation=cv2.INTER_LINEAR, + dst=rgb_out, + borderMode=cv2.BORDER_CONSTANT, + borderValue=0.0, + ) + backend = "opencv_remap_cached_hwc" + remap_ms = (time.perf_counter() - t0) * 1000.0 + + t0 = time.perf_counter() + # mixChannels faz o HWC -> 3 planos CHW em uma única rotina OpenCV, + # evitando três atribuições NumPy independentes. + cv2.mixChannels( + [rgb_out], + [tensor[0], tensor[1], tensor[2]], + [0, 0, 1, 1, 2, 2], + ) + pack_ms = (time.perf_counter() - t0) * 1000.0 + + return { + "remap_ms": float(remap_ms), + "pack_ms": float(pack_ms), + "backend": backend + "_mixchannels", + } def _direct_fusion_write_spec_remap_fast( self, @@ -7827,21 +8791,26 @@ class RawProcessorCore: remap_entry: dict, ): """ - RE/NIR -> tensor usando cv2.remap direto para target final. + RE/NIR -> tensor usando cv2.remap diretamente no plano CHW final. + + Evita o ndarray temporário 960x600 e a cópia subsequente para + tensor[channel_index], sem alterar a interpolação. """ map1 = remap_entry["map1"] map2 = remap_entry["map2"] + dst = tensor[int(channel_index)] - out = cv2.remap( + cv2.remap( img.astype(np.float32, copy=False), map1, map2, interpolation=cv2.INTER_LINEAR, + dst=dst, borderMode=cv2.BORDER_CONSTANT, borderValue=0.0, ) - tensor[int(channel_index)] = out + return dst def _resolve_homography_profile_name_for_role(self, role: str) -> str: """ @@ -8156,30 +9125,25 @@ class RawProcessorCore: return rgb.astype(np.float32, copy=False) def _get_bayer_cv2_code(self, bayer_pattern: str | None, algorithm: str = "ea"): - """ - Retorna o código OpenCV para demosaic Bayer. - - algorithm: - - "ea": Edge-Aware, melhor qualidade, mais pesado - - "bilinear": mais rápido, menor custo - """ p = str(bayer_pattern or self.bayer_pattern or "RGGB").upper() algo = str(algorithm or "ea").lower() if algo in ("bilinear", "linear", "fast", "normal"): code_map = { - "BGGR": cv2.COLOR_BayerRG2RGB, - "RGGB": cv2.COLOR_BayerBG2RGB, - "GRBG": cv2.COLOR_BayerGR2RGB, - "GBRG": cv2.COLOR_BayerGB2RGB, + "RGGB": cv2.COLOR_BayerRGGB2RGB, + "BGGR": cv2.COLOR_BayerBGGR2RGB, + "GRBG": cv2.COLOR_BayerGRBG2RGB, + "GBRG": cv2.COLOR_BayerGBRG2RGB, } + elif algo in ("ea", "edge_aware", "edge-aware"): code_map = { - "BGGR": cv2.COLOR_BayerRG2RGB_EA, - "RGGB": cv2.COLOR_BayerBG2RGB_EA, - "GRBG": cv2.COLOR_BayerGR2RGB_EA, - "GBRG": cv2.COLOR_BayerGB2RGB_EA, + "RGGB": cv2.COLOR_BayerRGGB2RGB_EA, + "BGGR": cv2.COLOR_BayerBGGR2RGB_EA, + "GRBG": cv2.COLOR_BayerGRBG2RGB_EA, + "GBRG": cv2.COLOR_BayerGBRG2RGB_EA, } + else: raise ValueError(f"demosaic_algorithm inválido: {algorithm}") @@ -8210,13 +9174,25 @@ class RawProcessorCore: self._last_decode_perf_log_ts = now - #parts = [] - #for k, v in perf.items(): - # if isinstance(v, (int, float)): - # parts.append(f"{k}={float(v):.2f}ms") - # else: - # parts.append(f"{k}={v}") - #print(f"[PERF][DECODE][{role.upper()}] " + " ".join(parts)) + if bool(getattr(self, "core_perf_debug", True)): + # Log compacto. O agregado de RGB/RE/NIR também aparece em CORE_ROLES. + numeric_keys = ( + "total_ms", "unpack_ms", "cvtColor_ms", "float_ms", + "calibration_ms", "clip_ms", "resize_half_ms" + ) + parts = [] + for k in numeric_keys: + v = perf.get(k) + if isinstance(v, (int, float)): + parts.append(f"{k}={float(v):.2f}ms") + shape = perf.get("out_shape") + mode = perf.get("mode") + print( + f"[PERF][DECODE][{role.upper()}] " + + " ".join(parts) + + (f" mode={mode}" if mode is not None else "") + + (f" out={shape}" if shape is not None else "") + ) @@ -8334,7 +9310,6 @@ class RawProcessorCore: strength = float(cfg.get("strength", 1.0)) strength_by_channel = cfg.get("strength_by_channel", {}) or {} - gain_min_runtime = float(cfg.get("gain_min_runtime", 0.0)) gain_max_runtime = float(cfg.get("gain_max_runtime", 999.0)) @@ -8343,7 +9318,7 @@ class RawProcessorCore: runtime_smooth_ksize += 1 key = ( - "final_flat_gain_tensor", + "final_flat_gain_tensor_v2_same_direct_maps", geom.get("key"), tuple(int(v) for v in crop_box), int(target_w), @@ -8360,25 +9335,56 @@ class RawProcessorCore: return cached gain_tensor = np.ones((5, target_h, target_w), dtype=np.float32) + maps = (remap_cache or {}).get("maps", {}) or {} + x0, y0, x1, y1 = [int(v) for v in crop_box] - # ------------------------------------------------------------ - # RGB: usa crop + resize, igual ao RGB real. - # ------------------------------------------------------------ - rgb_cam = role_to_cam.get("rgb") - if rgb_cam is not None: - rgb_img = decoded[rgb_cam]["image"] - rgb_shape = rgb_img.shape[:2] + # O gain de cada banda atravessa exatamente a mesma transformação + # espacial da banda real. Isso é especialmente importante no RGB, + # cuja orientação 180° pode estar fundida no mapa do direct. + channel_specs = ( + ("rgb", "R", 0), + ("rgb", "G", 1), + ("rgb", "B", 2), + ("re", "RE", 3), + ("nir", "NIR", 4), + ) - x0, y0, x1, y1 = [int(v) for v in crop_box] + for role, ch, ci in channel_specs: + cam_id = role_to_cam.get(role) + if cam_id is None: + continue - for ci, ch in enumerate(("R", "G", "B")): - gain_eff = self._get_runtime_gain_eff_map(ch, rgb_shape, cfg) + img = decoded[cam_id]["image"] + gain_eff = self._get_runtime_gain_eff_map(ch, img.shape[:2], cfg) + if gain_eff is None: + continue - if gain_eff is None: + gain_eff = gain_eff.astype(np.float32, copy=False) + remap_entry = maps.get(role) + + if isinstance(remap_entry, dict): + if role == "rgb": + map1 = remap_entry.get("warp_affine_map1", remap_entry.get("map1")) + map2 = remap_entry.get("warp_affine_map2", remap_entry.get("map2")) + else: + map1 = remap_entry.get("map1") + map2 = remap_entry.get("map2") + + if isinstance(map1, np.ndarray) and isinstance(map2, np.ndarray): + gain_out = cv2.remap( + gain_eff, + map1, + map2, + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=1.0, + ) + gain_tensor[ci] = gain_out.astype(np.float32, copy=False) continue - gain_crop = gain_eff[y0:y1, x0:x1].astype(np.float32, copy=False) - + # Fallback conservador se algum perfil rodar sem remap cache. + if role == "rgb": + gain_crop = gain_eff[y0:y1, x0:x1] if gain_crop.shape[1] != target_w or gain_crop.shape[0] != target_h: gain_out = cv2.resize( gain_crop, @@ -8387,49 +9393,12 @@ class RawProcessorCore: ) else: gain_out = gain_crop - - gain_tensor[ci] = gain_out.astype(np.float32, copy=False) - - # ------------------------------------------------------------ - # RE/NIR: usa o mesmo remap cacheado da imagem real. - # ------------------------------------------------------------ - maps = (remap_cache or {}).get("maps", {}) or {} - - spec_map = { - "re": ("RE", 3), - "nir": ("NIR", 4), - } - - for role, (ch, ci) in spec_map.items(): - cam_id = role_to_cam.get(role) - if cam_id is None: - continue - - img = decoded[cam_id]["image"] - gain_eff = self._get_runtime_gain_eff_map(ch, img.shape[:2], cfg) - - if gain_eff is None: - continue - - remap_entry = maps.get(role) - - if remap_entry is not None: - gain_out = cv2.remap( - gain_eff.astype(np.float32, copy=False), - remap_entry["map1"], - remap_entry["map2"], - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=1.0, - ) else: M_role_to_target = geom["M_role_to_target"].get(role) - if M_role_to_target is None: continue - gain_out = cv2.warpPerspective( - gain_eff.astype(np.float32, copy=False), + gain_eff, M_role_to_target, (target_w, target_h), flags=cv2.INTER_LINEAR, @@ -8439,10 +9408,138 @@ class RawProcessorCore: gain_tensor[ci] = gain_out.astype(np.float32, copy=False) + if not gain_tensor.flags.c_contiguous: + gain_tensor = np.ascontiguousarray(gain_tensor) + self._flatfield_runtime_cache[key] = gain_tensor return gain_tensor + def _orientation_matrix_same_shape_fast( + self, + src_shape, + rotate_deg=0, + flip_horizontal=False, + flip_vertical=False, + ): + """ + Matriz 3x3 source/native -> espaço canônico para orientações que + preservam H/W (0° e 180°, com flips opcionais). + + A ordem replica _apply_orientation_image(): rotate, flip H, flip V. + 90°/270° retornam None para cair no caminho físico tradicional. + """ + h, w = int(src_shape[0]), int(src_shape[1]) + r = int(rotate_deg) % 360 + + if r not in (0, 180): + return None + + M = np.eye(3, dtype=np.float32) + + if r == 180: + R = np.array( + [ + [-1.0, 0.0, float(w - 1)], + [0.0, -1.0, float(h - 1)], + [0.0, 0.0, 1.0], + ], + dtype=np.float32, + ) + M = R @ M + + if bool(flip_horizontal): + Fh = np.array( + [ + [-1.0, 0.0, float(w - 1)], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ], + dtype=np.float32, + ) + M = Fh @ M + + if bool(flip_vertical): + Fv = np.array( + [ + [1.0, 0.0, 0.0], + [0.0, -1.0, float(h - 1)], + [0.0, 0.0, 1.0], + ], + dtype=np.float32, + ) + M = Fv @ M + + return M.astype(np.float32, copy=False) + + def _can_defer_rgb_orientation_to_direct_remap_fast(self, decoded): + """ + Decide se a orientação RGB pode ser incorporada ao remap final. + + Conservador por design: + - somente role RGB; + - somente 0°/180° (mesmo H/W); + - exige remap cache ativo; + - pode ser desabilitado por fusion_config.fuse_orientation_into_remap=false. + """ + cfg = self.camera_orientation_config or {} + fusion = self.fusion_config or {} + + if not bool(cfg.get("enabled", False)): + return False + if not bool(fusion.get("fuse_orientation_into_remap", True)): + return False + if not bool(fusion.get("use_remap_cache", True)): + return False + + by_role = cfg.get("by_role", {}) or {} + role_cfg = by_role.get("rgb", {}) or {} + r = int(role_cfg.get("rotate_deg", 0)) % 360 + fh = bool(role_cfg.get("flip_horizontal", False)) + fv = bool(role_cfg.get("flip_vertical", False)) + + if r not in (0, 180): + return False + if r == 0 and not fh and not fv: + return False + + rgb_cam_id = self._find_cam_by_role(decoded, "rgb") + if rgb_cam_id is None: + return False + img = decoded[rgb_cam_id].get("image") + if img is None or img.ndim < 2: + return False + + return self._orientation_matrix_same_shape_fast( + img.shape[:2], r, fh, fv + ) is not None + + def _decoded_role_orientation_signature_fast( + self, decoded, role_to_cam, role + ): + role = str(role).lower() + cam_id = role_to_cam.get(role) + if cam_id is None: + return (role, False, 0, False, False) + meta = decoded[cam_id].get("meta", {}) or {} + ori = meta.get("orientation_applied", {}) or {} + return ( + role, + bool(ori.get("deferred_to_direct_fusion", False)), + int(ori.get("rotate_deg", 0)) % 360, + bool(ori.get("flip_horizontal", False)), + bool(ori.get("flip_vertical", False)), + ) + + def _decoded_role_orientation_is_deferred_fast( + self, decoded, role_to_cam, role + ): + return bool( + self._decoded_role_orientation_signature_fast( + decoded, role_to_cam, role + )[1] + ) + def _apply_orientation_image( self, img, @@ -8494,8 +9591,10 @@ class RawProcessorCore: def apply_camera_orientation_to_decoded( self, decoded, + defer_roles=None, ): cfg = self.camera_orientation_config or {} + defer_roles = {str(x).lower() for x in (defer_roles or set())} result = { "enabled": bool( @@ -8503,6 +9602,7 @@ class RawProcessorCore: ), "applied": False, "by_role": {}, + "deferred_roles": [], } if not cfg.get("enabled", False): @@ -8515,8 +9615,10 @@ class RawProcessorCore: ) or {} out = {} + perf_by_role_ms = {} for cam_id, item in decoded.items(): + t_role0 = time.perf_counter() role = str( item.get("role") @@ -8557,27 +9659,35 @@ class RawProcessorCore: item.get("meta", {}) or {} ) - if ( + wants_orientation = ( + rotate_deg != 0 + or flip_h + or flip_v + ) + deferred_here = bool( image is not None - and ( - rotate_deg != 0 - or flip_h - or flip_v - ) - ): + and role in defer_roles + and wants_orientation + ) + + if image is not None and wants_orientation and not deferred_here: image = self._apply_orientation_image( image, rotate_deg=rotate_deg, flip_horizontal=flip_h, flip_vertical=flip_v, ) - result["applied"] = True + if deferred_here: + result["deferred_roles"].append(role) + new_meta["orientation_applied"] = { "rotate_deg": rotate_deg, "flip_horizontal": flip_h, "flip_vertical": flip_v, + "physically_applied": bool(wants_orientation and not deferred_here), + "deferred_to_direct_fusion": bool(deferred_here), } new_item["image"] = image @@ -8585,18 +9695,24 @@ class RawProcessorCore: out[cam_id] = new_item + role_elapsed_ms = (time.perf_counter() - t_role0) * 1000.0 + perf_by_role_ms[role] = float(role_elapsed_ms) + result["by_role"][role] = { "camera_id": cam_id, "rotate_deg": rotate_deg, "flip_horizontal": flip_h, "flip_vertical": flip_v, + "deferred_to_direct_fusion": bool(deferred_here), "shape": ( list(image.shape) if image is not None else None ), + "perf_ms": float(role_elapsed_ms), } + result["perf_by_role_ms"] = dict(perf_by_role_ms) self.last_orientation_result = result return out diff --git a/Python/OAK/datasets/oak-fcc-3/core/segformer_service.py b/Python/OAK/datasets/oak-fcc-3/core/segformer_service.py index 4759e33a0..45c9ebaaa 100644 --- a/Python/OAK/datasets/oak-fcc-3/core/segformer_service.py +++ b/Python/OAK/datasets/oak-fcc-3/core/segformer_service.py @@ -1,14 +1,60 @@ # camera_worker/oak_fcc3_core/segformer_service.py # -*- coding: utf-8 -*- +""" +MultiSpecSegformerService - Production +====================================== + +Último estágio do pipeline multiespectral antes do WeedDetector. + +Contrato oficial: + RawProcessorCore / CameraMultispectral + -> Raw5 físico canônico float32 CHW: + [R, G, B, RE, NIR] + -> MultiSpecSegformerService + - valida contrato do ONNX + - deriva NDVI/NDRE SOMENTE se o modelo pedir + - NÃO redimensiona a entrada + - NÃO normaliza novamente + - executa somente a cabeça operacional selecionada + - exige máscara full-resolution + -> máscara binária uint8 HxW + -> WeedDetector + +Autoridades: + - Shape BCHW: sessão ONNX + - Ordem nominal de canais: config + metadata/sidecar do ONNX + - Normalização: embutida no ONNX + - Raw5 físico: sempre [R,G,B,RE,NIR] + +Canais suportados pelo modelo: + R, G, B, RE, NIR, NDVI, NDRE + +Cabeças operacionais: + target + vegetation + cana + semantic -> convertida em binário usando semantic_target_class + +Fail-closed produto: + - input ONNX deve ser BCHW estático + - quantidade de canais deve bater + - metadata nominal deve bater quando exigida + - normalization_embedded deve ser true quando declarada + - saída da cabeça escolhida deve existir exatamente + - máscara deve sair no mesmo HxW do input + - máscara final do WeedDetector deve ser 0/1 + - mudanças estruturais exigem recriar a sessão +""" + from __future__ import annotations -import time -import threading -import unicodedata import json +import threading +import time +import unicodedata from pathlib import Path -from typing import Dict, List, Tuple +from typing import Dict, List, Sequence, Tuple import cv2 import numpy as np @@ -19,154 +65,349 @@ except Exception: ort = None +SEGFORMER_SERVICE_VERSION = "production_v1_2026_08_24" + PHYSICAL_CHANNEL_ORDER = ["R", "G", "B", "RE", "NIR"] DERIVED_CHANNEL_ORDER = ["NDVI", "NDRE"] SUPPORTED_CHANNEL_ORDER = PHYSICAL_CHANNEL_ORDER + DERIVED_CHANNEL_ORDER DEFAULT_CHANNEL_ORDER = PHYSICAL_CHANNEL_ORDER HEAD_COLORS_RGB = { - 0: (30, 30, 30), # background - 1: (255, 70, 30), # classe positiva da cabeça binária selecionada - 255: (0, 0, 0), # ignore/fallback + 0: (30, 30, 30), + 1: (255, 70, 30), + 255: (0, 0, 0), } -def get_input_channel_names(config: dict) -> List[str]: - names = config.get("input_channels", DEFAULT_CHANNEL_ORDER) +# ============================================================ +# Contrato de canais +# ============================================================ + +def parse_channel_list( + value, + fallback: Sequence[str] | None = None, +) -> List[str]: + if value is None: + names = list( + fallback + or PHYSICAL_CHANNEL_ORDER + ) + + elif isinstance( + value, + str, + ): + names = [ + x.strip().upper() + for x in value.split(",") + if x.strip() + ] - if isinstance(names, str): - names = [c.strip().upper() for c in names.split(",") if c.strip()] else: - names = [str(c).upper() for c in names] + names = [ + str(x).strip().upper() + for x in value + if str(x).strip() + ] + + if not names: + raise RuntimeError( + "input_channels vazio." + ) + + invalid = [ + x + for x in names + if x not in SUPPORTED_CHANNEL_ORDER + ] - invalid = [c for c in names if c not in SUPPORTED_CHANNEL_ORDER] if invalid: raise RuntimeError( - f"Canais inválidos em input_channels: {invalid}. " + f"Canais inválidos: {invalid}. " f"Suportados={SUPPORTED_CHANNEL_ORDER}" ) - if not names: - raise RuntimeError("input_channels vazio.") - - duplicates = sorted({c for c in names if names.count(c) > 1}) - if duplicates: - raise RuntimeError(f"Canais duplicados em input_channels: {duplicates}") - - configured_count = config.get("channels") - if configured_count is not None and int(configured_count) != len(names): + if len( + set(names) + ) != len(names): raise RuntimeError( - f"Config inconsistente: channels={configured_count}, mas " - f"input_channels possui {len(names)} itens: {names}" + f"Canais duplicados: {names}" ) return names -def get_input_channel_indices(config: dict) -> List[int | None]: - names = get_input_channel_names(config) +def get_input_channel_names( + config: dict, +) -> List[str]: + names = parse_channel_list( + config.get( + "input_channels" + ), + PHYSICAL_CHANNEL_ORDER, + ) + + configured_count = config.get( + "channels" + ) + + if ( + configured_count is not None + and int( + configured_count + ) != len(names) + ): + raise RuntimeError( + "Config inconsistente: " + f"channels={configured_count}, " + f"input_channels={names} ({len(names)})" + ) + + return names + + +def get_input_channel_indices( + config: dict, +) -> List[int | None]: + names = get_input_channel_names( + config + ) + return [ - PHYSICAL_CHANNEL_ORDER.index(c) if c in PHYSICAL_CHANNEL_ORDER else None + ( + PHYSICAL_CHANNEL_ORDER.index( + c + ) + if c + in PHYSICAL_CHANNEL_ORDER + else None + ) for c in names ] -def build_model_input_tensor( - raw5_chw: np.ndarray, - input_channel_names: List[str], - derived_config: dict | None = None, -) -> np.ndarray: - """Monta os canais nominais do modelo a partir do Raw5 físico canônico.""" - raw5 = np.asarray(raw5_chw) - if raw5.ndim != 3 or raw5.shape[0] != len(PHYSICAL_CHANNEL_ORDER): +def validate_derived_config( + config: dict | None, +) -> dict: + cfg = dict( + config + or {} + ) + + eps = float( + cfg.get( + "epsilon", + 1e-6, + ) + ) + + clip_min = float( + cfg.get( + "clip_min", + -1.0, + ) + ) + + clip_max = float( + cfg.get( + "clip_max", + 1.0, + ) + ) + + if ( + not np.isfinite( + eps + ) + or eps <= 0.0 + ): raise RuntimeError( - "Entrada física inválida: esperado Raw5 CHW " - f"{PHYSICAL_CHANNEL_ORDER}, veio shape={raw5.shape}" + f"derived_channels.epsilon inválido: {eps}" ) - names = [str(name).strip().upper() for name in input_channel_names] - if not names: - raise RuntimeError("Lista de canais nominais vazia.") - invalid = [name for name in names if name not in SUPPORTED_CHANNEL_ORDER] - if invalid: - raise RuntimeError(f"Canais nominais não suportados: {invalid}") - if len(set(names)) != len(names): - raise RuntimeError(f"Canais nominais duplicados: {names}") + if ( + not np.isfinite( + clip_min + ) + or not np.isfinite( + clip_max + ) + or clip_min >= clip_max + ): + raise RuntimeError( + "Faixa inválida em derived_channels: " + f"clip_min={clip_min}, " + f"clip_max={clip_max}" + ) - raw5 = np.ascontiguousarray(raw5, dtype=np.float32) + return { + "epsilon": eps, + "clip_min": clip_min, + "clip_max": clip_max, + } + + +def build_model_input_tensor( + raw5_chw: np.ndarray, + requested_channels: Sequence[str], + derived_config: dict | None = None, +) -> np.ndarray: + """ + ÚNICO lugar onde canais derivados são criados. + + Entrada física obrigatória: + [R,G,B,RE,NIR] + + Exemplos de saída: + 5ch -> [R,G,B,RE,NIR] + 7ch -> [R,G,B,RE,NIR,NDVI,NDRE] + + NDVI: + (NIR - R) / (NIR + R) + + NDRE: + (NIR - RE) / (NIR + RE) + """ + raw5 = np.asarray( + raw5_chw + ) + + if ( + raw5.ndim != 3 + or raw5.shape[0] + != len( + PHYSICAL_CHANNEL_ORDER + ) + ): + raise RuntimeError( + "Raw5 físico inválido: esperado " + f"{PHYSICAL_CHANNEL_ORDER}, " + f"shape={raw5.shape}" + ) + + names = parse_channel_list( + requested_channels + ) + + raw5 = np.ascontiguousarray( + raw5, + dtype=np.float32, + ) + + # Fast path mais comum. if names == PHYSICAL_CHANNEL_ORDER: return raw5 - cfg = derived_config or {} - eps = float(cfg.get("epsilon", 1e-6)) - clip_min = float(cfg.get("clip_min", -1.0)) - clip_max = float(cfg.get("clip_max", 1.0)) - if not np.isfinite(eps) or eps <= 0.0: - raise RuntimeError(f"derived_channels.epsilon inválido: {eps}") - if not np.isfinite(clip_min) or not np.isfinite(clip_max) or clip_min >= clip_max: - raise RuntimeError( - "Faixa inválida em derived_channels: " - f"clip_min={clip_min}, clip_max={clip_max}" - ) + cfg = validate_derived_config( + derived_config + ) + + eps = float( + cfg["epsilon"] + ) + clip_min = float( + cfg["clip_min"] + ) + clip_max = float( + cfg["clip_max"] + ) physical = { name: raw5[index] - for index, name in enumerate(PHYSICAL_CHANNEL_ORDER) + for index, name + in enumerate( + PHYSICAL_CHANNEL_ORDER + ) } + derived = {} - def normalized_difference(a: np.ndarray, b: np.ndarray) -> np.ndarray: + def normalized_difference( + a: np.ndarray, + b: np.ndarray, + ) -> np.ndarray: denominator = a + b - result = np.zeros_like(denominator, dtype=np.float32) - np.divide(a - b, denominator, out=result, where=np.abs(denominator) > eps) - np.nan_to_num(result, copy=False, nan=0.0, posinf=clip_max, neginf=clip_min) - np.clip(result, clip_min, clip_max, out=result) + + result = np.zeros_like( + denominator, + dtype=np.float32, + ) + + np.divide( + a - b, + denominator, + out=result, + where=np.abs( + denominator + ) > eps, + ) + + np.nan_to_num( + result, + copy=False, + nan=0.0, + posinf=clip_max, + neginf=clip_min, + ) + + np.clip( + result, + clip_min, + clip_max, + out=result, + ) + return result if "NDVI" in names: - derived["NDVI"] = normalized_difference(physical["NIR"], physical["R"]) + derived["NDVI"] = ( + normalized_difference( + physical["NIR"], + physical["R"], + ) + ) + if "NDRE" in names: - derived["NDRE"] = normalized_difference(physical["NIR"], physical["RE"]) + derived["NDRE"] = ( + normalized_difference( + physical["NIR"], + physical["RE"], + ) + ) - output = np.empty((len(names), raw5.shape[1], raw5.shape[2]), dtype=np.float32) - for index, name in enumerate(names): - output[index] = physical[name] if name in physical else derived[name] - return np.ascontiguousarray(output) + output = np.empty( + ( + len(names), + raw5.shape[1], + raw5.shape[2], + ), + dtype=np.float32, + ) + for index, name in enumerate( + names + ): + if name in physical: + output[index] = ( + physical[name] + ) + else: + output[index] = ( + derived[name] + ) + + return np.ascontiguousarray( + output + ) + + +# ============================================================ +# Runtime +# ============================================================ class MultiSpecSegformerService: """ - Runtime oficial Weed Worker multi-head. - - Contrato: - input: - tensor CHW float32 0..1 - normalmente [R,G,B,RE,NIR] - - backend: - ONNX Runtime + TensorRT - - modelo ONNX: - já contém: - - normalização - - SegFormer multi-head - - resize fullres - - argmax - e expõe as saídas multi-head. Este serviço executa somente uma das - cabeças operacionais por inferência: - - target_mask - - vegetation_mask - - cana_mask - - semantic_mask (convertida para binário pela classe escolhida) - - output: - np.ndarray uint8 HxW - máscara binária uint8 HxW da cabeça selecionada - 0 = classe negativa - 1 = classe positiva - - semantic_mask nunca é entregue diretamente ao WeedDetector. Quando - selecionada, ela é convertida para 0/1 usando semantic_target_class. + Runtime multi-head oficial do Weed Worker. """ MODE_ALIASES = { @@ -208,53 +449,193 @@ class MultiSpecSegformerService: "erva": 2, } - def __init__(self, model_config: dict, mostrar_log=print): - self.config = model_config or {} - self.mostrar_log = mostrar_log - self._runtime_lock = threading.RLock() + STRUCTURAL_CONFIG_KEYS = ( + "onnx_model_path", + "ia_model_path", + "model_path", + "onnx_path", + "input_channels", + "channels", + "derived_channels", + "runtime_backend", + "onnx_provider", + "onnx_output_kind", + "onnx_preprocess_norm", + "trust_input", + "require_onnx_metadata", + "strict_output_shape", + "allow_provider_fallback", + ) + + def __init__( + self, + model_config: dict, + mostrar_log=print, + ): + self.config = dict( + model_config + or {} + ) + + self.mostrar_log = ( + mostrar_log + ) + + self._runtime_lock = ( + threading.RLock() + ) + self._runtime_generation = 0 - self.input_channel_names = get_input_channel_names(self.config) - self.input_channel_indices = get_input_channel_indices(self.config) - self.channels = len(self.input_channel_names) - self.derived_channels_config = dict( - self.config.get("derived_channels", {}) or {} + self.input_channel_names = ( + get_input_channel_names( + self.config + ) ) - self.runtime_backend = str(self.config.get("runtime_backend", "onnx")).lower() - self.onnx_provider = str(self.config.get("onnx_provider", "tensorrt")).lower() - # runtime_mode é a fonte única da verdade. onnx_output_mode permanece - # apenas como fallback para configs antigas que não possuem runtime_mode. - configured_mode = self.config.get( - "runtime_mode", - self.config.get("onnx_output_mode", "target"), + self.input_channel_indices = ( + get_input_channel_indices( + self.config + ) ) - self.runtime_mode = str(configured_mode).strip().lower() - self.selected_head = self._normalizar_head(self.runtime_mode) - self.selected_output_name = self.OUTPUT_BY_HEAD[self.selected_head] - self.semantic_target_class = self._normalizar_semantic_target_class( - self.config.get("semantic_target_class", "erva") + + self.channels = len( + self.input_channel_names ) - self.semantic_target_id = self.SEMANTIC_CLASS_IDS[self.semantic_target_class] - # Mantido para compatibilidade com logs/código externo antigo. - self.onnx_output_mode = self.runtime_mode - self.onnx_output_kind = str(self.config.get("onnx_output_kind", "mask")).lower() + self.derived_channels_config = ( + validate_derived_config( + self.config.get( + "derived_channels", + {}, + ) + ) + ) - # No contrato v1, a normalização está dentro do ONNX. - # Manter False evita normalização dupla. - self.onnx_preprocess_norm = bool(self.config.get("onnx_preprocess_norm", False)) + self.runtime_backend = str( + self.config.get( + "runtime_backend", + "onnx", + ) + ).lower() - self.trust_input = bool(self.config.get("trust_input", True)) - self.sync_for_timing = bool(self.config.get("sync_for_timing", False)) + self.onnx_provider = str( + self.config.get( + "onnx_provider", + "tensorrt", + ) + ).lower() + + configured_mode = ( + self.config.get( + "runtime_mode", + self.config.get( + "onnx_output_mode", + "target", + ), + ) + ) + + self.runtime_mode = str( + configured_mode + ).strip().lower() + + self.selected_head = ( + self._normalizar_head( + self.runtime_mode + ) + ) + + self.selected_output_name = ( + self.OUTPUT_BY_HEAD[ + self.selected_head + ] + ) + + self.semantic_target_class = ( + self._normalizar_semantic_target_class( + self.config.get( + "semantic_target_class", + "erva", + ) + ) + ) + + self.semantic_target_id = ( + self.SEMANTIC_CLASS_IDS[ + self.semantic_target_class + ] + ) + + # Compatibilidade nominal. + self.onnx_output_mode = ( + self.runtime_mode + ) + + self.onnx_output_kind = str( + self.config.get( + "onnx_output_kind", + "mask", + ) + ).lower() + + self.onnx_preprocess_norm = bool( + self.config.get( + "onnx_preprocess_norm", + False, + ) + ) + + self.trust_input = bool( + self.config.get( + "trust_input", + True, + ) + ) + + self.sync_for_timing = bool( + self.config.get( + "sync_for_timing", + False, + ) + ) + + # Produto: metadata nominal é parte do contrato. + self.require_onnx_metadata = bool( + self.config.get( + "require_onnx_metadata", + True, + ) + ) + + # O export oficial já devolve mask fullres. + self.strict_output_shape = bool( + self.config.get( + "strict_output_shape", + True, + ) + ) + + # Fallback de provider pode esconder perda brutal de FPS. + self.allow_provider_fallback = bool( + self.config.get( + "allow_provider_fallback", + False, + ) + ) self.onnx_session = None self.onnx_input_name = None + self.onnx_input_shape = [] self.onnx_output_names = [] self.onnx_run_output_names = [] + self.onnx_path = None self.onnx_contract_metadata = {} - self.onnx_input_shape = [] + self.onnx_contract_source = None + + self.expected_input_size = None + self.active_providers = [] self._ultimo_tensor = None self._ultimo_predictions = None @@ -264,481 +645,1633 @@ class MultiSpecSegformerService: self._validar_contrato_runtime() self.mostrar_log( - f"[WEED_ONNX][INPUT] selected_channels={self.input_channel_names} " + "[WEED_ONNX][INPUT] " + f"selected_channels={self.input_channel_names} " f"idx={self.input_channel_indices}" ) self._init_onnx_runtime() self.mostrar_log( - f"[WEED_ONNX] backend=onnx " + "[WEED_ONNX] " + f"version={SEGFORMER_SERVICE_VERSION} " f"provider={self.onnx_provider} " f"runtime_mode={self.runtime_mode} " f"selected_head={self.selected_head} " f"selected_output={self.selected_output_name} " - f"output_kind={self.onnx_output_kind} " - f"preprocess_norm={self.onnx_preprocess_norm}" + f"input_shape={self.onnx_input_shape}" ) + # ============================================================ + # Contrato público + # ============================================================ + + def get_expected_input_size( + self, + ) -> Tuple[int, int]: + """ + Retorna (W,H) oficial do ONNX. + """ + if self.expected_input_size is None: + raise RuntimeError( + "Contrato ONNX ainda não foi resolvido." + ) + + return ( + int( + self.expected_input_size[0] + ), + int( + self.expected_input_size[1] + ), + ) + + def get_expected_input_shape( + self, + ) -> List[int]: + return list( + self.onnx_input_shape + ) + + def get_model_contract( + self, + ) -> dict: + with self._runtime_lock: + return { + "service_version": ( + SEGFORMER_SERVICE_VERSION + ), + "onnx_model_path": ( + str( + self.onnx_path + ) + if self.onnx_path + is not None + else None + ), + "input_name": ( + self.onnx_input_name + ), + "input_shape": list( + self.onnx_input_shape + ), + "input_size": ( + list( + self.expected_input_size + ) + if self.expected_input_size + is not None + else None + ), + "input_channel_names": list( + self.input_channel_names + ), + "physical_input_contract": list( + PHYSICAL_CHANNEL_ORDER + ), + "derived_channel_config": dict( + self.derived_channels_config + ), + "normalization_embedded": True, + "onnx_output_kind": ( + self.onnx_output_kind + ), + "onnx_outputs": list( + self.onnx_output_names + ), + "selected_head": ( + self.selected_head + ), + "selected_output": ( + self.selected_output_name + ), + "runtime_generation": int( + self._runtime_generation + ), + "runtime_backend": ( + self.runtime_backend + ), + "requested_provider": ( + self.onnx_provider + ), + "active_providers": list( + self.active_providers + ), + "metadata_source": ( + self.onnx_contract_source + ), + "require_onnx_metadata": bool( + self.require_onnx_metadata + ), + "strict_output_shape": bool( + self.strict_output_shape + ), + } + # ============================================================ # Inicialização / contrato # ============================================================ @classmethod - def _normalizar_head(cls, mode: str) -> str: - normalized = str(mode).strip().lower() - head = cls.MODE_ALIASES.get(normalized) + def _normalizar_head( + cls, + mode: str, + ) -> str: + normalized = str( + mode + ).strip().lower() + + head = cls.MODE_ALIASES.get( + normalized + ) if head is None: raise RuntimeError( - f"runtime_mode inválido para Weed Worker: {mode!r}. " - "Use 'target', 'vegetation', 'cana' ou 'semantic'." + f"runtime_mode inválido: {mode!r}. " + "Use target, vegetation, cana ou semantic." ) return head @staticmethod - def _normalizar_texto(value) -> str: - text = unicodedata.normalize("NFKD", str(value).strip().lower()) - return "".join(ch for ch in text if not unicodedata.combining(ch)) + def _normalizar_texto( + value, + ) -> str: + text = unicodedata.normalize( + "NFKD", + str( + value + ).strip().lower(), + ) + + return "".join( + ch + for ch in text + if not unicodedata.combining( + ch + ) + ) @classmethod - def _normalizar_semantic_target_class(cls, value) -> str: - normalized = cls._normalizar_texto(value) + def _normalizar_semantic_target_class( + cls, + value, + ) -> str: + normalized = ( + cls._normalizar_texto( + value + ) + ) + aliases = { - "0": "chao", "chao": "chao", "solo": "chao", "background": "chao", "bg": "chao", - "1": "cana", "cana": "cana", "sugarcane": "cana", - "2": "erva", "erva": "erva", "weed": "erva", + "0": "chao", + "chao": "chao", + "solo": "chao", + "background": "chao", + "bg": "chao", + + "1": "cana", + "cana": "cana", + "sugarcane": "cana", + + "2": "erva", + "erva": "erva", + "weed": "erva", } - result = aliases.get(normalized) + + result = aliases.get( + normalized + ) + if result is None: raise RuntimeError( - f"semantic_target_class inválida: {value!r}. " - "Use 'chao', 'cana' ou 'erva'." + "semantic_target_class inválida: " + f"{value!r}. Use chao, cana ou erva." ) + return result - def _validar_contrato_runtime(self): - if self.runtime_backend not in ("onnx", "tensorrt", "trt"): + def _validar_contrato_runtime( + self, + ): + if self.runtime_backend not in ( + "onnx", + "tensorrt", + "trt", + ): raise RuntimeError( - f"runtime_backend inválido para Weed Worker v1: {self.runtime_backend}. " + "runtime_backend inválido: " + f"{self.runtime_backend}. " "Use runtime_backend='onnx'." ) - # A normalização no __init__ já valida o modo e define a cabeça. - if self.selected_head not in self.OUTPUT_BY_HEAD: - raise RuntimeError(f"Cabeça operacional inválida: {self.selected_head}") + if ( + self.selected_head + not in self.OUTPUT_BY_HEAD + ): + raise RuntimeError( + "Cabeça operacional inválida: " + f"{self.selected_head}" + ) if self.onnx_output_kind != "mask": raise RuntimeError( - f"onnx_output_kind inválido para Weed Worker v1: {self.onnx_output_kind}. " - "O ONNX oficial deve devolver target_mask pronto." + "onnx_output_kind inválido: " + f"{self.onnx_output_kind}. " + "O export oficial deve devolver máscaras." ) if self.onnx_preprocess_norm: raise RuntimeError( - "onnx_preprocess_norm=True não é permitido na v1. " - "O ONNX oficial já inclui normalização interna." + "onnx_preprocess_norm=True é proibido. " + "A normalização pertence ao ONNX exportado." ) - def _resolve_onnx_path(self) -> Path: + def _resolve_onnx_path( + self, + ) -> Path: candidates = [] - for key in ("onnx_model_path", "ia_model_path", "model_path", "onnx_path"): - value = self.config.get(key) - if value: - candidates.append(Path(value)) - - for p in candidates: - p = p.resolve() if not p.is_absolute() else p - if p.is_file(): - return p - - raise FileNotFoundError( - "Modelo ONNX não encontrado. Procurei:\n" + - "\n".join(str(p) for p in candidates) - ) - - def _init_onnx_runtime(self): - if ort is None: - raise RuntimeError( - "onnxruntime não está instalado. Instale onnxruntime-gpu " - "para usar TensorRT/CUDA." + for key in ( + "onnx_model_path", + "ia_model_path", + "model_path", + "onnx_path", + ): + value = self.config.get( + key ) - onnx_path = self._resolve_onnx_path() - - sess_options = ort.SessionOptions() - sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL - - #sess_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL - #sess_options.intra_op_num_threads = 1 - #sess_options.inter_op_num_threads = 1 - #sess_options.add_session_config_entry( - # "session.intra_op.allow_spinning", - # "0", - #) - #sess_options.add_session_config_entry( - # "session.inter_op.allow_spinning", - # "0", - #) - - trt_cache_dir = str(self.config.get("trt_cache_dir", "trt_engine_cache")) - trt_fp16 = bool(self.config.get("trt_fp16", True)) - - available = ort.get_available_providers() - providers = [] - - self.mostrar_log(f"[WEED_ONNX] providers disponíveis: {available}") - - if self.onnx_provider in ("tensorrt", "trt"): - if "TensorrtExecutionProvider" in available: - providers.append(( - "TensorrtExecutionProvider", - { - "trt_engine_cache_enable": True, - "trt_engine_cache_path": trt_cache_dir, - "trt_fp16_enable": trt_fp16, - } - )) - else: - self.mostrar_log( - "[WEED_ONNX][WARN] TensorRT provider não disponível. " - "Caindo para CUDA/CPU." + if value: + candidates.append( + Path( + str(value) + ) ) - if self.onnx_provider in ("tensorrt", "trt", "cuda"): - if "CUDAExecutionProvider" in available: - providers.append("CUDAExecutionProvider") + resolved_tried = [] - providers.append("CPUExecutionProvider") + for p in candidates: + try: + p = ( + p + if p.is_absolute() + else p.resolve() + ) - self.onnx_session = ort.InferenceSession( - str(onnx_path), - sess_options=sess_options, - providers=providers, + resolved_tried.append( + p + ) + + if p.is_file(): + return p + + except Exception: + resolved_tried.append( + p + ) + + raise FileNotFoundError( + "Modelo ONNX não encontrado. Procurei:\n" + + "\n".join( + str(p) + for p + in resolved_tried + ) ) - self.onnx_input_name = self.onnx_session.get_inputs()[0].name - self.onnx_output_names = [o.name for o in self.onnx_session.get_outputs()] - self.onnx_path = onnx_path - self.onnx_contract_metadata = self._load_onnx_contract_metadata(onnx_path) - self._validate_onnx_input_contract() - self.onnx_run_output_names = self._selecionar_outputs_runtime(self.onnx_output_names) + def _build_provider_list( + self, + available: List[str], + ): + requested = ( + self.onnx_provider + ) - self.mostrar_log(f"[WEED_ONNX] modelo={onnx_path}") - self.mostrar_log(f"[WEED_ONNX] input={self.onnx_input_name}") - self.mostrar_log(f"[WEED_ONNX] outputs={self.onnx_output_names}") - self.mostrar_log(f"[WEED_ONNX] outputs executados={self.onnx_run_output_names}") - self.mostrar_log(f"[WEED_ONNX] providers ativos={self.onnx_session.get_providers()}") + providers = [] + + trt_cache_enable = bool( + self.config.get( + "trt_engine_cache_enable", + True, + ) + ) + + trt_cache_path = str( + self.config.get( + "trt_engine_cache_path", + self.config.get( + "trt_cache_dir", + "./trt_cache_weed_worker", + ), + ) + ) + + trt_fp16 = bool( + self.config.get( + "trt_fp16", + True, + ) + ) + + trt_workspace = ( + self.config.get( + "trt_max_workspace_size" + ) + ) + + if requested in ( + "tensorrt", + "trt", + ): + if ( + "TensorrtExecutionProvider" + not in available + ): + if not self.allow_provider_fallback: + raise RuntimeError( + "TensorRT foi solicitado, mas " + "TensorrtExecutionProvider não está disponível. " + f"Providers={available}" + ) + + self.mostrar_log( + "[WEED_ONNX][WARN] TensorRT indisponível; " + "fallback explicitamente permitido." + ) + + else: + trt_options = { + "trt_engine_cache_enable": ( + trt_cache_enable + ), + "trt_engine_cache_path": ( + trt_cache_path + ), + "trt_fp16_enable": ( + trt_fp16 + ), + } + + if trt_workspace is not None: + trt_options[ + "trt_max_workspace_size" + ] = int( + trt_workspace + ) + + providers.append( + ( + "TensorrtExecutionProvider", + trt_options, + ) + ) + + if requested in ( + "tensorrt", + "trt", + "cuda", + ): + if ( + "CUDAExecutionProvider" + in available + ): + providers.append( + "CUDAExecutionProvider" + ) + + elif ( + requested == "cuda" + and not self.allow_provider_fallback + ): + raise RuntimeError( + "CUDA foi solicitado, mas " + "CUDAExecutionProvider não está disponível. " + f"Providers={available}" + ) + + # CPU permanece fallback interno do ORT quando explicitamente + # permitido ou como último provider após TRT/CUDA. + if ( + self.allow_provider_fallback + or providers + or requested == "cpu" + ): + if ( + "CPUExecutionProvider" + in available + ): + providers.append( + "CPUExecutionProvider" + ) + + if not providers: + raise RuntimeError( + "Nenhum provider ONNX válido pôde ser selecionado. " + f"requested={requested} available={available}" + ) + + return providers + + def _validate_active_provider( + self, + ): + active = list( + self.onnx_session.get_providers() + ) + + self.active_providers = ( + active + ) + + if self.onnx_provider in ( + "tensorrt", + "trt", + ): + expected = ( + "TensorrtExecutionProvider" + ) + + elif self.onnx_provider == "cuda": + expected = ( + "CUDAExecutionProvider" + ) + + else: + expected = ( + "CPUExecutionProvider" + ) + + if ( + expected not in active + and not self.allow_provider_fallback + ): + raise RuntimeError( + "Provider solicitado não ficou ativo: " + f"solicitado={expected}, ativos={active}" + ) + + def _init_onnx_runtime( + self, + ): + if ort is None: + raise RuntimeError( + "onnxruntime não está instalado. " + "Use onnxruntime-gpu no produto." + ) + + onnx_path = ( + self._resolve_onnx_path() + ) + + sess_options = ( + ort.SessionOptions() + ) + + sess_options.graph_optimization_level = ( + ort.GraphOptimizationLevel + .ORT_ENABLE_ALL + ) + + available = list( + ort.get_available_providers() + ) + + self.mostrar_log( + "[WEED_ONNX] providers disponíveis: " + f"{available}" + ) + + providers = ( + self._build_provider_list( + available + ) + ) + + self.onnx_session = ( + ort.InferenceSession( + str( + onnx_path + ), + sess_options=sess_options, + providers=providers, + ) + ) + + self._validate_active_provider() + + inputs = list( + self.onnx_session.get_inputs() + ) + + if len(inputs) != 1: + raise RuntimeError( + "O ONNX produto deve possuir exatamente uma entrada. " + f"Encontradas={len(inputs)}" + ) + + self.onnx_input_name = ( + inputs[0].name + ) + + self.onnx_output_names = [ + o.name + for o in ( + self.onnx_session + .get_outputs() + ) + ] + + self.onnx_path = ( + onnx_path + ) + + self.onnx_contract_metadata = ( + self._load_onnx_contract_metadata( + onnx_path + ) + ) + + self._validate_onnx_input_contract() + + self.onnx_run_output_names = ( + self._selecionar_outputs_runtime( + self.onnx_output_names + ) + ) + + self.mostrar_log( + f"[WEED_ONNX] modelo={onnx_path}" + ) + + self.mostrar_log( + f"[WEED_ONNX] input={self.onnx_input_name} " + f"shape={self.onnx_input_shape}" + ) + + self.mostrar_log( + f"[WEED_ONNX] outputs={self.onnx_output_names}" + ) + + self.mostrar_log( + "[WEED_ONNX] output executado=" + f"{self.onnx_run_output_names}" + ) + + self.mostrar_log( + "[WEED_ONNX] providers ativos=" + f"{self.active_providers}" + ) + + # ============================================================ + # Metadata do modelo + # ============================================================ @staticmethod - def _parse_metadata_list(value): + def _parse_metadata_list( + value, + ): if value is None: return None - if isinstance(value, (list, tuple)): - return [str(item).strip().upper() for item in value] - text = str(value).strip() + + if isinstance( + value, + ( + list, + tuple, + ), + ): + return [ + str(item) + .strip() + .upper() + for item + in value + ] + + text = str( + value + ).strip() + if not text: return None + try: - parsed = json.loads(text) - if isinstance(parsed, list): - return [str(item).strip().upper() for item in parsed] + parsed = json.loads( + text + ) + + if isinstance( + parsed, + list, + ): + return [ + str(item) + .strip() + .upper() + for item + in parsed + ] + except Exception: pass - return [item.strip().upper() for item in text.split(",") if item.strip()] + + return [ + item.strip().upper() + for item + in text.split(",") + if item.strip() + ] @staticmethod - def _parse_metadata_bool(value): - if isinstance(value, bool): + def _parse_metadata_bool( + value, + ): + if isinstance( + value, + bool, + ): return value + if value is None: return None - text = str(value).strip().lower() - if text in ("1", "true", "yes", "sim"): + + text = str( + value + ).strip().lower() + + if text in ( + "1", + "true", + "yes", + "sim", + ): return True - if text in ("0", "false", "no", "nao", "não"): + + if text in ( + "0", + "false", + "no", + "nao", + "não", + ): return False + return None - def _load_onnx_contract_metadata(self, onnx_path: Path) -> dict: - """Lê o sidecar e completa o contrato com a metadata interna do ONNX.""" + @staticmethod + def _parse_metadata_shape( + value, + ): + if value is None: + return None + + if isinstance( + value, + ( + list, + tuple, + ), + ): + return list( + value + ) + + text = str( + value + ).strip() + + try: + parsed = json.loads( + text + ) + + if isinstance( + parsed, + list, + ): + return parsed + + except Exception: + pass + + return None + + def _load_onnx_contract_metadata( + self, + onnx_path: Path, + ) -> dict: contract = {} + sidecar_candidates = [ - Path(str(onnx_path) + ".export_meta.json"), - onnx_path.with_suffix(".export_meta.json"), + Path( + str( + onnx_path + ) + + ".export_meta.json" + ), + onnx_path.with_suffix( + ".export_meta.json" + ), ] - for sidecar in sidecar_candidates: + + for sidecar in ( + sidecar_candidates + ): if not sidecar.is_file(): continue + try: - with sidecar.open("r", encoding="utf-8") as handle: - data = json.load(handle) - if isinstance(data, dict): - contract.update(data) - contract["_sidecar_path"] = str(sidecar) + with sidecar.open( + "r", + encoding="utf-8", + ) as handle: + data = json.load( + handle + ) + + if isinstance( + data, + dict, + ): + contract.update( + data + ) + + contract[ + "_sidecar_path" + ] = str( + sidecar + ) + + self.onnx_contract_source = str( + sidecar + ) + break + except Exception as exc: - raise RuntimeError(f"Falha ao ler contrato ONNX {sidecar}: {exc}") from exc + raise RuntimeError( + "Falha ao ler contrato ONNX " + f"{sidecar}: {exc}" + ) from exc try: internal = dict( - getattr(self.onnx_session.get_modelmeta(), "custom_metadata_map", {}) or {} + getattr( + self.onnx_session + .get_modelmeta(), + "custom_metadata_map", + {}, + ) + or {} ) + except Exception: internal = {} - contract["_internal"] = internal + + contract[ + "_internal" + ] = internal + + if ( + self.onnx_contract_source + is None + and internal + ): + self.onnx_contract_source = ( + "onnx_custom_metadata" + ) + return contract - def _validate_onnx_input_contract(self): - model_input = self.onnx_session.get_inputs()[0] - shape = list(model_input.shape or []) - self.onnx_input_shape = shape + def _validate_onnx_input_contract( + self, + ): + model_input = ( + self.onnx_session + .get_inputs()[0] + ) + + shape = list( + model_input.shape + or [] + ) + + self.onnx_input_shape = ( + shape + ) + if len(shape) != 4: raise RuntimeError( - f"Input ONNX deve ser BCHW 4D; {model_input.name!r} possui shape={shape}" + "Input ONNX deve ser BCHW 4D; " + f"{model_input.name!r} shape={shape}" ) + batch = shape[0] model_c = shape[1] - if isinstance(model_c, (int, np.integer)) and int(model_c) != self.channels: + model_h = shape[2] + model_w = shape[3] + + # Produto usa batch 1. + if ( + isinstance( + batch, + (int, np.integer), + ) + and int(batch) != 1 + ): raise RuntimeError( - f"Contrato ONNX incompatível: modelo espera C={model_c}, mas o config " - f"define C={self.channels} ({self.input_channel_names})." + f"ONNX produto exige batch=1; shape={shape}" ) - contract = self.onnx_contract_metadata or {} - internal = contract.get("_internal", {}) or {} - metadata_names = self._parse_metadata_list( - contract.get("input_channel_names", internal.get("oak.input_channel_names")) + if not isinstance( + model_c, + (int, np.integer), + ): + raise RuntimeError( + "Produto exige C estático no ONNX; " + f"shape={shape}" + ) + + if not isinstance( + model_h, + (int, np.integer), + ) or not isinstance( + model_w, + (int, np.integer), + ): + raise RuntimeError( + "Produto exige H/W estáticos no ONNX; " + f"shape={shape}" + ) + + if int( + model_c + ) != self.channels: + raise RuntimeError( + "Contrato ONNX incompatível: " + f"modelo espera C={model_c}, " + f"config define C={self.channels} " + f"{self.input_channel_names}" + ) + + if ( + int(model_h) <= 0 + or int(model_w) <= 0 + ): + raise RuntimeError( + f"H/W inválidos no ONNX: {shape}" + ) + + self.expected_input_size = ( + int( + model_w + ), + int( + model_h + ), ) - if metadata_names is not None and metadata_names != self.input_channel_names: - raise RuntimeError( - "Ordem de canais incompatível entre config e ONNX: " - f"config={self.input_channel_names}, modelo={metadata_names}." - ) - input_contract = contract.get("input_contract", {}) or {} - embedded_norm = self._parse_metadata_bool( - input_contract.get( - "normalization_embedded", - internal.get("oak.normalization_embedded"), + contract = ( + self.onnx_contract_metadata + or {} + ) + + internal = ( + contract.get( + "_internal", + {}, + ) + or {} + ) + + metadata_names = ( + self._parse_metadata_list( + contract.get( + "input_channel_names", + internal.get( + "oak.input_channel_names" + ), + ) ) ) - if embedded_norm is False: - raise RuntimeError( - "O ONNX declara normalization_embedded=false, mas este runtime exige " - "normalização incorporada. Reexporte com --include-norm." - ) - - postprocess = contract.get("postprocess", internal.get("oak.postprocess")) - if postprocess is not None and "argmax" not in str(postprocess).strip().lower(): - raise RuntimeError( - f"ONNX incompatível: postprocess={postprocess!r}. " - "O runtime operacional exige máscaras argmax prontas." - ) if metadata_names is None: + if self.require_onnx_metadata: + raise RuntimeError( + "ONNX produto sem metadata nominal " + "input_channel_names. Reexporte o modelo com o " + "exportador oficial." + ) + self.mostrar_log( - "[WEED_ONNX][WARN] ONNX sem metadata de input_channel_names; " - "validando apenas a quantidade de canais." - ) - else: - source = contract.get("_sidecar_path", "metadata interna") - self.mostrar_log( - f"[WEED_ONNX][CONTRACT] canais={metadata_names} fonte={source}" + "[WEED_ONNX][WARN] ONNX sem metadata nominal de canais." ) - def _selecionar_outputs_runtime(self, output_names: List[str]) -> List[str]: - output_lookup = {str(name).lower(): str(name) for name in output_names} - expected = self.selected_output_name.lower() + elif ( + metadata_names + != self.input_channel_names + ): + raise RuntimeError( + "Ordem de canais incompatível entre config e ONNX: " + f"config={self.input_channel_names}, " + f"modelo={metadata_names}" + ) - if expected in output_lookup: - return [output_lookup[expected]] - - raise RuntimeError( - f"A saída necessária {self.selected_output_name!r} não existe no ONNX. " - f"Cabeça selecionada={self.selected_head!r}; " - f"outputs disponíveis={output_names}." + input_contract = ( + contract.get( + "input_contract", + {}, + ) + or {} ) - def atualizar_runtime_mode(self, runtime_mode, semantic_target_class=None) -> bool: - """Troca a cabeça operacional sem recriar a sessão ONNX. + embedded_norm = ( + self._parse_metadata_bool( + input_contract.get( + "normalization_embedded", + internal.get( + "oak.normalization_embedded" + ), + ) + ) + ) - Retorna True quando a configuração efetivamente mudou. Uma inferência - que já estava em andamento é descartada e o novo modo vale no frame - seguinte. - """ - new_mode = str(runtime_mode).strip().lower() - new_head = self._normalizar_head(new_mode) - new_output = self.OUTPUT_BY_HEAD[new_head] + if embedded_norm is False: + raise RuntimeError( + "ONNX declara normalization_embedded=false. " + "O runtime produto exige normalização incorporada." + ) - if semantic_target_class is None: - semantic_target_class = self.semantic_target_class - new_semantic_class = self._normalizar_semantic_target_class(semantic_target_class) - new_semantic_id = self.SEMANTIC_CLASS_IDS[new_semantic_class] + if ( + embedded_norm is None + and self.require_onnx_metadata + ): + raise RuntimeError( + "ONNX produto não declara normalization_embedded. " + "Reexporte com metadata de contrato." + ) + + postprocess = contract.get( + "postprocess", + internal.get( + "oak.postprocess" + ), + ) + + if postprocess is not None: + if ( + "argmax" + not in str( + postprocess + ).strip().lower() + ): + raise RuntimeError( + "ONNX incompatível: " + f"postprocess={postprocess!r}. " + "Esperado argmax/mask embutido." + ) + + elif self.require_onnx_metadata: + raise RuntimeError( + "ONNX produto não declara postprocess. " + "O runtime exige export full-runtime com máscaras prontas." + ) + + metadata_shape = ( + self._parse_metadata_shape( + contract.get( + "input_shape", + internal.get( + "oak.input_shape" + ), + ) + ) + ) + + if ( + metadata_shape is not None + and len( + metadata_shape + ) == 4 + ): + comparable = [] + + for actual, declared in zip( + shape, + metadata_shape, + ): + if ( + isinstance( + actual, + (int, np.integer), + ) + and isinstance( + declared, + (int, np.integer), + ) + ): + comparable.append( + int(actual) + == int(declared) + ) + + if ( + comparable + and not all( + comparable + ) + ): + raise RuntimeError( + "Metadata input_shape diverge da sessão ONNX: " + f"metadata={metadata_shape}, session={shape}" + ) + + if metadata_names is not None: + self.mostrar_log( + "[WEED_ONNX][CONTRACT] " + f"canais={metadata_names} " + f"size={self.expected_input_size} " + f"metadata={self.onnx_contract_source}" + ) + + # ============================================================ + # Heads / troca dinâmica + # ============================================================ + + def _selecionar_outputs_runtime( + self, + output_names: List[str], + ) -> List[str]: + output_lookup = { + str(name).lower(): str( + name + ) + for name + in output_names + } + + expected = ( + self.selected_output_name + .lower() + ) + + if expected in output_lookup: + actual = ( + output_lookup[ + expected + ] + ) + + self.selected_output_name = ( + actual + ) + + return [ + actual + ] + + raise RuntimeError( + "Saída necessária não existe no ONNX: " + f"head={self.selected_head!r} " + f"expected={self.selected_output_name!r} " + f"outputs={output_names}" + ) + + def atualizar_runtime_mode( + self, + runtime_mode, + semantic_target_class=None, + ) -> bool: + new_head = self._normalizar_head( + runtime_mode + ) + + new_output = ( + self.OUTPUT_BY_HEAD[ + new_head + ] + ) + + if ( + semantic_target_class + is None + ): + semantic_target_class = ( + self.semantic_target_class + ) + + new_semantic_class = ( + self._normalizar_semantic_target_class( + semantic_target_class + ) + ) + + new_semantic_id = ( + self.SEMANTIC_CLASS_IDS[ + new_semantic_class + ] + ) + + output_lookup = { + str(n).lower(): str(n) + for n + in self.onnx_output_names + } + + actual_output = ( + output_lookup.get( + new_output.lower() + ) + ) - output_lookup = {str(n).lower(): str(n) for n in self.onnx_output_names} - actual_output = output_lookup.get(new_output.lower()) if actual_output is None: raise RuntimeError( - f"A saída necessária {new_output!r} não existe no ONNX. " - f"Outputs disponíveis={self.onnx_output_names}." + "A saída necessária " + f"{new_output!r} não existe no ONNX. " + f"Outputs={self.onnx_output_names}" ) with self._runtime_lock: changed = ( - new_head != self.selected_head - or new_semantic_id != self.semantic_target_id + new_head + != self.selected_head + or ( + new_head + == "semantic" + and new_semantic_id + != self.semantic_target_id + ) ) + if not changed: return False - old_desc = self._runtime_description_unlocked() - self.runtime_mode = new_head - self.onnx_output_mode = new_head - self.selected_head = new_head - self.selected_output_name = actual_output - self.onnx_run_output_names = [actual_output] - self.semantic_target_class = new_semantic_class - self.semantic_target_id = new_semantic_id - self._runtime_generation += 1 - self._limpar_cache_unlocked() - new_desc = self._runtime_description_unlocked() + old_desc = ( + self._runtime_description_unlocked() + ) + + self.runtime_mode = ( + new_head + ) + + self.onnx_output_mode = ( + new_head + ) + + self.selected_head = ( + new_head + ) + + self.selected_output_name = ( + actual_output + ) + + self.onnx_run_output_names = [ + actual_output + ] + + self.semantic_target_class = ( + new_semantic_class + ) + + self.semantic_target_id = ( + new_semantic_id + ) + + self._runtime_generation += 1 + + self._limpar_cache_unlocked() + + new_desc = ( + self._runtime_description_unlocked() + ) + + self.mostrar_log( + "[WEED_ONNX][MODE] " + f"{old_desc} -> {new_desc}" + ) - self.mostrar_log(f"[WEED_ONNX][MODE] {old_desc} -> {new_desc}") return True - def atualizar_config(self, config: dict) -> bool: - config = config or {} + def atualizar_config( + self, + config: dict, + ) -> bool: + """ + Somente seleção de cabeça/classe é mutável. - # Estes campos alteram a arquitetura/contrato do input e exigem uma - # nova sessão. Só runtime_mode e semantic_target_class são dinâmicos. - if "input_channels" in config or "channels" in config: - merged = dict(self.config) - merged.update(config) - requested_names = get_input_channel_names(merged) - if requested_names != self.input_channel_names: + Modelo, provider, shape, canais, derived config e preprocess são + estruturais e exigem recriar MultiSpecSegformerService. + """ + config = dict( + config + or {} + ) + + # Canais. + if ( + "input_channels" + in config + or "channels" + in config + ): + merged = dict( + self.config + ) + + merged.update( + config + ) + + requested_names = ( + get_input_channel_names( + merged + ) + ) + + if ( + requested_names + != self.input_channel_names + ): raise RuntimeError( - "input_channels não pode ser alterado com a sessão ONNX ativa: " - f"atual={self.input_channel_names}, solicitado={requested_names}. " - "Recrie MultiSpecSegformerService com o novo modelo/config." + "input_channels não pode mudar na sessão ONNX: " + f"atual={self.input_channel_names}, " + f"solicitado={requested_names}" ) + # Derived. if "derived_channels" in config: - requested_derived = dict(config.get("derived_channels", {}) or {}) - if requested_derived != self.derived_channels_config: + requested_derived = ( + validate_derived_config( + config.get( + "derived_channels" + ) + ) + ) + + if ( + requested_derived + != self.derived_channels_config + ): raise RuntimeError( - "derived_channels não pode ser alterado com a sessão ONNX ativa. " - "Recrie MultiSpecSegformerService para preservar o contrato." + "derived_channels não pode mudar na sessão ONNX." ) - mode = config.get("runtime_mode", config.get("onnx_output_mode", self.runtime_mode)) - semantic_class = config.get("semantic_target_class", self.semantic_target_class) - changed = self.atualizar_runtime_mode(mode, semantic_class) - self.config.update(config) + # Caminho do modelo. + for key in ( + "onnx_model_path", + "ia_model_path", + "model_path", + "onnx_path", + ): + value = config.get( + key + ) + + if not value: + continue + + try: + requested = str( + Path( + str(value) + ).resolve() + ) + + current = str( + self.onnx_path.resolve() + ) + + except Exception: + requested = str( + value + ) + + current = str( + self.onnx_path + ) + + if requested != current: + raise RuntimeError( + "Modelo ONNX não pode mudar com a sessão ativa." + ) + + immutable_simple = ( + "runtime_backend", + "onnx_provider", + "onnx_output_kind", + "onnx_preprocess_norm", + "trust_input", + "require_onnx_metadata", + "strict_output_shape", + "allow_provider_fallback", + ) + + for key in ( + immutable_simple + ): + if key not in config: + continue + + old = self.config.get( + key + ) + + new = config.get( + key + ) + + # Config antiga pode omitir o default. Só acusa quando ambos + # representam mudança efetiva. + if ( + old is not None + and new != old + ): + raise RuntimeError( + f"{key} não pode mudar com a sessão ONNX ativa." + ) + + mode = config.get( + "runtime_mode", + config.get( + "onnx_output_mode", + self.runtime_mode, + ), + ) + + semantic_class = ( + config.get( + "semantic_target_class", + self.semantic_target_class, + ) + ) + + changed = ( + self.atualizar_runtime_mode( + mode, + semantic_class, + ) + ) + + # Salva somente opções dinâmicas e valores iguais aos estruturais. + self.config.update( + config + ) + return changed - def _runtime_description_unlocked(self) -> str: - if self.selected_head == "semantic": - return f"semantic[{self.semantic_target_class}={self.semantic_target_id}]" + def _runtime_description_unlocked( + self, + ) -> str: + if ( + self.selected_head + == "semantic" + ): + return ( + "semantic[" + f"{self.semantic_target_class}=" + f"{self.semantic_target_id}]" + ) + return self.selected_head - def _limpar_cache_unlocked(self): + def _limpar_cache_unlocked( + self, + ): self._ultimo_predictions = None self._ultimo_probs = None self._ultimo_predictions_full = {} - def limpar_cache_runtime(self): + def limpar_cache_runtime( + self, + ): with self._runtime_lock: self._limpar_cache_unlocked() - def get_runtime_generation(self) -> int: + def get_runtime_generation( + self, + ) -> int: with self._runtime_lock: - return int(self._runtime_generation) + return int( + self._runtime_generation + ) # ============================================================ # Inferência # ============================================================ - def infer_tensor_fast(self, tensor5_chw: np.ndarray, keep_probs: bool = False): - # keep_probs mantido só para compatibilidade de chamada. + def infer_tensor_fast( + self, + tensor5_chw: np.ndarray, + keep_probs: bool = False, + ): + # keep_probs existe por compatibilidade. return self.infer_tensor_onnx( tensor5_chw, - return_full=bool(self.config.get("return_full_fast", False)), + return_full=bool( + self.config.get( + "return_full_fast", + False, + ) + ), ) - def infer_tensor_onnx(self, tensor5_chw: np.ndarray, return_full: bool = False): + def infer_tensor_onnx( + self, + tensor5_chw: np.ndarray, + return_full: bool = False, + ): if tensor5_chw is None: return None if self.onnx_session is None: - raise RuntimeError("ONNX Runtime não inicializado.") + raise RuntimeError( + "ONNX Runtime não inicializado." + ) t_total0 = time.perf_counter() t_prepare0 = time.perf_counter() - raw5_chw, x, out_hw = self._prepare_input_numpy_onnx(tensor5_chw) - prepare_ms = (time.perf_counter() - t_prepare0) * 1000.0 - if x is None: - return None - - t_forward0 = time.perf_counter() - - with self._runtime_lock: - run_output_names = list(self.onnx_run_output_names) - selected_head = self.selected_head - selected_output_name = self.selected_output_name - semantic_target_class = self.semantic_target_class - semantic_target_id = self.semantic_target_id - runtime_generation = self._runtime_generation - - outputs = self.onnx_session.run( - run_output_names, - {self.onnx_input_name: x}, + raw5_chw, x, out_hw = ( + self._prepare_input_numpy_onnx( + tensor5_chw + ) ) - forward_ms = (time.perf_counter() - t_forward0) * 1000.0 + prepare_ms = ( + time.perf_counter() + - t_prepare0 + ) * 1000.0 - t_post0 = time.perf_counter() + t_forward0 = ( + time.perf_counter() + ) - if len(outputs) < 1: - raise RuntimeError("ONNX não retornou nenhuma saída.") - - raw_mask = self._onnx_output_to_mask(outputs[0], out_hw) - - if selected_head == "semantic": - selected_mask = (raw_mask == semantic_target_id).astype(np.uint8) - else: - selected_mask = raw_mask - - # Defesa do contrato do WeedDetector: a saída final deste serviço é - # sempre binária. As cabeças binárias já devem conter 0/1; semantic - # foi explicitamente convertida acima. - invalid_values = np.logical_and(selected_mask != 0, selected_mask != 1) - if np.any(invalid_values): - values = np.unique(selected_mask[invalid_values])[:10].tolist() - raise RuntimeError( - f"A saída {selected_output_name!r} não é binária. " - f"Valores inválidos encontrados: {values}" + with self._runtime_lock: + run_output_names = list( + self.onnx_run_output_names ) - post_ms = (time.perf_counter() - t_post0) * 1000.0 - total_ms = (time.perf_counter() - t_total0) * 1000.0 + selected_head = ( + self.selected_head + ) + + selected_output_name = ( + self.selected_output_name + ) + + semantic_target_class = ( + self.semantic_target_class + ) + + semantic_target_id = ( + self.semantic_target_id + ) + + runtime_generation = ( + self._runtime_generation + ) + + outputs = ( + self.onnx_session.run( + run_output_names, + { + self.onnx_input_name: x + }, + ) + ) + + forward_ms = ( + time.perf_counter() + - t_forward0 + ) * 1000.0 + + t_post0 = ( + time.perf_counter() + ) + + if len(outputs) != 1: + raise RuntimeError( + "Inferência operacional deve executar exatamente " + f"uma saída; retornadas={len(outputs)}" + ) + + raw_mask = ( + self._onnx_output_to_mask( + outputs[0], + out_hw, + ) + ) + + if selected_head == "semantic": + selected_mask = ( + raw_mask + == semantic_target_id + ).astype( + np.uint8 + ) + + else: + selected_mask = ( + raw_mask + ) + + # Contrato WeedDetector sempre binário. + invalid_values = np.logical_and( + selected_mask != 0, + selected_mask != 1, + ) + + if np.any( + invalid_values + ): + values = np.unique( + selected_mask[ + invalid_values + ] + )[:10].tolist() + + raise RuntimeError( + "Saída operacional não é binária: " + f"output={selected_output_name!r}, " + f"values={values}" + ) + + selected_mask = np.ascontiguousarray( + selected_mask, + dtype=np.uint8, + ) + + post_ms = ( + time.perf_counter() + - t_post0 + ) * 1000.0 + + total_ms = ( + time.perf_counter() + - t_total0 + ) * 1000.0 predictions_full = { "selected": selected_mask, - "selected_head": selected_head, - "selected_output": selected_output_name, - "semantic_target_class": semantic_target_class if selected_head == "semantic" else None, - "semantic_target_id": semantic_target_id if selected_head == "semantic" else None, - "target": selected_mask if selected_head == "target" else None, - "target_head": selected_mask if selected_head == "target" else None, - "target_op": selected_mask if selected_head == "target" else None, - "semantic": raw_mask if selected_head == "semantic" else None, - "vegetation": selected_mask if selected_head == "vegetation" else None, - "cana": selected_mask if selected_head == "cana" else None, + "selected_head": ( + selected_head + ), + "selected_output": ( + selected_output_name + ), + "semantic_target_class": ( + semantic_target_class + if selected_head + == "semantic" + else None + ), + "semantic_target_id": ( + semantic_target_id + if selected_head + == "semantic" + else None + ), + + "target": ( + selected_mask + if selected_head + == "target" + else None + ), + "target_head": ( + selected_mask + if selected_head + == "target" + else None + ), + "target_op": ( + selected_mask + if selected_head + == "target" + else None + ), + "semantic": ( + raw_mask + if selected_head + == "semantic" + else None + ), + "vegetation": ( + selected_mask + if selected_head + == "vegetation" + else None + ), + "cana": ( + selected_mask + if selected_head + == "cana" + else None + ), "probs": None, "infer_ms": total_ms, @@ -747,198 +2280,489 @@ class MultiSpecSegformerService: "post_ms": post_ms, "runtime_mode": selected_head, - "runtime_generation": runtime_generation, + "runtime_generation": int( + runtime_generation + ), "backend": "onnx", - "providers": self.onnx_session.get_providers(), - "outputs": self.onnx_output_names, - "outputs_executados": run_output_names, + "providers": list( + self.active_providers + ), + "outputs": list( + self.onnx_output_names + ), + "outputs_executados": ( + run_output_names + ), + "input_channel_names": list( + self.input_channel_names + ), + "input_shape": list( + x.shape + ), } - # A configuração pode mudar enquanto o provider executa. A verificação - # e a publicação do cache ficam no mesmo lock para impedir que um frame - # antigo reapareça depois de atualizar_runtime_mode() limpar o estado. + # Não publica resultado de uma geração antiga depois de trocar head. with self._runtime_lock: - if runtime_generation != self._runtime_generation: + if ( + runtime_generation + != self._runtime_generation + ): return None - # Preview/cache permanecem no domínio físico [R,G,B,RE,NIR]. - self._ultimo_tensor = raw5_chw - self._ultimo_predictions = selected_mask + + # Cache de preview permanece no domínio físico Raw5. + self._ultimo_tensor = ( + raw5_chw + ) + + self._ultimo_predictions = ( + selected_mask + ) + self._ultimo_probs = None - self._ultimo_predictions_full = predictions_full + + self._ultimo_predictions_full = ( + predictions_full + ) if return_full: - return dict(predictions_full) + return dict( + predictions_full + ) return selected_mask - def _prepare_input_numpy_onnx(self, tensor5_chw: np.ndarray): + def _prepare_input_numpy_onnx( + self, + tensor5_chw: np.ndarray, + ): if tensor5_chw is None: return None, None, None if self.trust_input: raw5 = tensor5_chw - if not isinstance(raw5, np.ndarray): - raw5 = np.asarray(raw5, dtype=np.float32) + if not isinstance( + raw5, + np.ndarray, + ): + raw5 = np.asarray( + raw5, + dtype=np.float32, + ) - if raw5.dtype != np.float32 or not raw5.flags.c_contiguous: - raw5 = np.ascontiguousarray(raw5, dtype=np.float32) + if ( + raw5.dtype + != np.float32 + or not raw5.flags.c_contiguous + ): + raw5 = np.ascontiguousarray( + raw5, + dtype=np.float32, + ) else: - raw5 = np.asarray(tensor5_chw, dtype=np.float32) - raw5 = np.nan_to_num(raw5, nan=0.0, posinf=1.0, neginf=0.0) - raw5 = np.clip(raw5, 0.0, 1.0).astype(np.float32, copy=False) - raw5 = np.ascontiguousarray(raw5) - - if raw5.ndim != 3 or raw5.shape[0] != len(PHYSICAL_CHANNEL_ORDER): - raise RuntimeError( - "Tensor físico inválido no runtime: esperado " - f"Raw5 {PHYSICAL_CHANNEL_ORDER}, veio shape={raw5.shape}" + raw5 = np.asarray( + tensor5_chw, + dtype=np.float32, ) - model_chw = build_model_input_tensor( - raw5, - self.input_channel_names, - self.derived_channels_config, + np.nan_to_num( + raw5, + copy=False, + nan=0.0, + posinf=1.0, + neginf=0.0, + ) + + np.clip( + raw5, + 0.0, + 1.0, + out=raw5, + ) + + raw5 = np.ascontiguousarray( + raw5, + dtype=np.float32, + ) + + if ( + raw5.ndim != 3 + or raw5.shape[0] + != len( + PHYSICAL_CHANNEL_ORDER + ) + ): + raise RuntimeError( + "Tensor físico inválido: esperado Raw5 " + f"{PHYSICAL_CHANNEL_ORDER}, " + f"shape={raw5.shape}" + ) + + expected_w, expected_h = ( + self.get_expected_input_size() ) - if model_chw.shape[0] != self.channels: + + if ( + int( + raw5.shape[2] + ) != expected_w + or int( + raw5.shape[1] + ) != expected_h + ): raise RuntimeError( - f"Tensor nominal inválido: esperado C={self.channels}, " - f"veio shape={model_chw.shape}" + "Raw5 H/W diverge do ONNX. " + f"raw5={raw5.shape}, " + f"esperado=(5,{expected_h},{expected_w}). " + "O SegformerService não faz resize." ) - if len(self.onnx_input_shape) == 4: - expected_h, expected_w = self.onnx_input_shape[2], self.onnx_input_shape[3] - if isinstance(expected_h, (int, np.integer)) and int(expected_h) != model_chw.shape[1]: - raise RuntimeError( - f"Altura incompatível com ONNX: modelo={expected_h}, tensor={model_chw.shape[1]}" - ) - if isinstance(expected_w, (int, np.integer)) and int(expected_w) != model_chw.shape[2]: - raise RuntimeError( - f"Largura incompatível com ONNX: modelo={expected_w}, tensor={model_chw.shape[2]}" - ) + model_chw = ( + build_model_input_tensor( + raw5, + self.input_channel_names, + self.derived_channels_config, + ) + ) - h, w = int(raw5.shape[1]), int(raw5.shape[2]) + if ( + model_chw.shape[0] + != self.channels + ): + raise RuntimeError( + "Tensor nominal inválido: " + f"esperado C={self.channels}, " + f"shape={model_chw.shape}" + ) - # No contrato v1 não normaliza aqui. - # O ONNX oficial já possui normalização interna. - x = model_chw[None, :, :, :].astype(np.float32, copy=False) - x = np.ascontiguousarray(x, dtype=np.float32) + if ( + model_chw.shape[1] + != expected_h + or model_chw.shape[2] + != expected_w + ): + raise RuntimeError( + "Tensor nominal mudou H/W inesperadamente: " + f"shape={model_chw.shape}" + ) - return raw5, x, (h, w) + # NÃO normaliza e NÃO redimensiona. + x = np.ascontiguousarray( + model_chw[ + None, + :, + :, + :, + ], + dtype=np.float32, + ) - def _select_input_channels(self, chw: np.ndarray) -> np.ndarray: - # Compatibilidade com chamadas externas antigas: a entrada continua Raw5. + return ( + raw5, + x, + ( + expected_h, + expected_w, + ), + ) + + def _select_input_channels( + self, + chw: np.ndarray, + ) -> np.ndarray: + # Compatibilidade externa. return build_model_input_tensor( chw, self.input_channel_names, self.derived_channels_config, ) - @staticmethod - def _onnx_output_to_mask(arr: np.ndarray, out_hw) -> np.ndarray: - h, w = int(out_hw[0]), int(out_hw[1]) + def _onnx_output_to_mask( + self, + arr: np.ndarray, + out_hw, + ) -> np.ndarray: + h = int( + out_hw[0] + ) - arr = np.asarray(arr) + w = int( + out_hw[1] + ) - # Formatos esperados para ONNX full-runtime: - # HxW - # 1xHxW - # BxHxW + arr = np.asarray( + arr + ) + + # Export full-runtime esperado: + # HxW + # 1xHxW if arr.ndim == 2: - mask = arr.astype(np.uint8, copy=False) + mask = arr - elif arr.ndim == 3: - mask = arr[0].astype(np.uint8, copy=False) + elif ( + arr.ndim == 3 + and arr.shape[0] == 1 + ): + mask = arr[0] + + elif ( + arr.ndim == 4 + and arr.shape[0] == 1 + and arr.shape[1] == 1 + ): + # Tolerância para alguns exporters que preservam canal singleton. + mask = arr[ + 0, + 0, + ] else: raise RuntimeError( - f"Saída ONNX inválida para contrato mask. " - f"Esperado HxW ou 1xHxW, veio shape={arr.shape}" + "Saída ONNX inválida para máscara operacional: " + f"shape={arr.shape}. Esperado HxW, 1xHxW " + "ou 1x1xHxW." ) - if mask.shape[0] != h or mask.shape[1] != w: - mask = cv2.resize(mask, (w, h), interpolation=cv2.INTER_NEAREST) + if ( + mask.shape[0] != h + or mask.shape[1] != w + ): + if self.strict_output_shape: + raise RuntimeError( + "ONNX não devolveu máscara full-resolution: " + f"output={mask.shape}, esperado={(h, w)}. " + "Reexporte o modelo. O runtime produto não redimensiona " + "a máscara silenciosamente." + ) - return mask.astype(np.uint8, copy=False) + mask = cv2.resize( + mask, + ( + w, + h, + ), + interpolation=cv2.INTER_NEAREST, + ) + + return np.ascontiguousarray( + mask.astype( + np.uint8, + copy=False, + ) + ) # ============================================================ # Preview / stream # ============================================================ - def preview_infer_cached(self, tensor5_chw=None, predictions=None, alpha=0.5): + def preview_infer_cached( + self, + tensor5_chw=None, + predictions=None, + alpha=0.5, + ): """ - Gera frames BGR para TCP/C#/debug. + Somente visualização. - Não roda inferência aqui. - Usa: - - tensor multiespectral cacheado - - máscara binária cacheada da cabeça selecionada + Não roda inferência e não altera o tensor científico. """ - tensor = tensor5_chw if tensor5_chw is not None else self._ultimo_tensor - pred = predictions if predictions is not None else self._ultimo_predictions + tensor = ( + tensor5_chw + if tensor5_chw + is not None + else self._ultimo_tensor + ) + + pred = ( + predictions + if predictions + is not None + else self._ultimo_predictions + ) if tensor is None: - return None, None, None, None, None + return ( + None, + None, + None, + None, + None, + ) - rgb = self.tensor_to_preview_rgb(tensor) - rgb_bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) + rgb = ( + self.tensor_to_preview_rgb( + tensor + ) + ) + + rgb_bgr = cv2.cvtColor( + rgb, + cv2.COLOR_RGB2BGR, + ) if pred is None: - return rgb_bgr, None, rgb_bgr, None, None + return ( + rgb_bgr, + None, + rgb_bgr, + None, + None, + ) - seg_rgb = self.ids_to_rgb(pred, HEAD_COLORS_RGB) - overlay_rgb = cv2.addWeighted(rgb, 1.0 - alpha, seg_rgb, alpha, 0.0) + seg_rgb = self.ids_to_rgb( + pred, + HEAD_COLORS_RGB, + ) - seg_bgr = cv2.cvtColor(seg_rgb, cv2.COLOR_RGB2BGR) - overlay_bgr = cv2.cvtColor(overlay_rgb, cv2.COLOR_RGB2BGR) + overlay_rgb = cv2.addWeighted( + rgb, + 1.0 - alpha, + seg_rgb, + alpha, + 0.0, + ) - return rgb_bgr, seg_bgr, overlay_bgr, None, None + seg_bgr = cv2.cvtColor( + seg_rgb, + cv2.COLOR_RGB2BGR, + ) + + overlay_bgr = cv2.cvtColor( + overlay_rgb, + cv2.COLOR_RGB2BGR, + ) + + return ( + rgb_bgr, + seg_bgr, + overlay_bgr, + None, + None, + ) @staticmethod - def tensor_to_preview_rgb(chw: np.ndarray, gamma: float = 0.85) -> np.ndarray: + def tensor_to_preview_rgb( + chw: np.ndarray, + gamma: float = 0.85, + ) -> np.ndarray: if chw is None: return None - chw = np.asarray(chw) + chw = np.asarray( + chw + ) if chw.ndim != 3: - raise RuntimeError(f"Tensor inválido para preview: esperado CHW, veio shape={chw.shape}") + raise RuntimeError( + "Tensor inválido para preview: " + f"shape={chw.shape}" + ) - c, h, w = chw.shape + if chw.shape[0] < 3: + raise RuntimeError( + "Preview RGB exige ao menos R/G/B." + ) - if c >= 3: - rgb = np.transpose(chw[:3], (1, 2, 0)).copy() - else: - one = chw[0] - rgb = np.stack([one, one, one], axis=-1) + # O cache do service é sempre Raw5 físico, + # então [:3] é R,G,B por contrato. + rgb = np.transpose( + chw[:3], + ( + 1, + 2, + 0, + ), + ).copy() - rgb = np.nan_to_num(rgb, nan=0.0, posinf=1.0, neginf=0.0) + np.nan_to_num( + rgb, + copy=False, + nan=0.0, + posinf=1.0, + neginf=0.0, + ) - lo = np.percentile(rgb, 1.0) - hi = np.percentile(rgb, 99.0) + # Preview only. + lo = float( + np.percentile( + rgb, + 1.0, + ) + ) + + hi = float( + np.percentile( + rgb, + 99.0, + ) + ) if hi > lo: - rgb = (rgb - lo) / (hi - lo) + rgb = ( + rgb - lo + ) / ( + hi - lo + ) - rgb = np.clip(rgb, 0.0, 1.0) + np.clip( + rgb, + 0.0, + 1.0, + out=rgb, + ) if gamma and gamma > 0: - rgb = np.power(rgb, gamma) + np.power( + rgb, + gamma, + out=rgb, + ) - return (rgb * 255.0).astype(np.uint8) + return ( + rgb * 255.0 + ).astype( + np.uint8 + ) @staticmethod - def ids_to_rgb(mask: np.ndarray, colormap_rgb: Dict[int, Tuple[int, int, int]]) -> np.ndarray: - mask = np.asarray(mask) + def ids_to_rgb( + mask: np.ndarray, + colormap_rgb: Dict[ + int, + Tuple[int, int, int], + ], + ) -> np.ndarray: + mask = np.asarray( + mask + ) if mask.ndim != 2: - raise RuntimeError(f"Máscara inválida para preview: esperado HxW, veio shape={mask.shape}") + raise RuntimeError( + "Máscara inválida para preview: " + f"shape={mask.shape}" + ) - h, w = mask.shape[:2] - out = np.zeros((h, w, 3), dtype=np.uint8) + h, w = ( + mask.shape[:2] + ) - for cid, color in colormap_rgb.items(): - out[mask == int(cid)] = color + out = np.zeros( + ( + h, + w, + 3, + ), + dtype=np.uint8, + ) + + for cid, color in ( + colormap_rgb.items() + ): + out[ + mask + == int(cid) + ] = color return out