feat(BEVFusion-Camera): release BEVFusion-camera 2.7.x

autoware_ml/configs/detection3d/dataset/t4dataset/gen2_base.py

@@ -0,0 +1,221 @@
+custom_imports = dict(
+    imports=[
+        "autoware_ml.detection3d.datasets.t4dataset",
+        "autoware_ml.detection3d.evaluation.t4metric.t4metric",
+        "autoware_ml.detection3d.evaluation.t4metric.t4metric_v2",
+    ]
+)
+
+# dataset type setting
+dataset_type = "T4Dataset"
+info_train_file_name = "t4dataset_gen2_base_infos_train.pkl"
+info_val_file_name = "t4dataset_gen2_base_infos_val.pkl"
+info_test_file_name = "t4dataset_gen2_base_infos_test.pkl"
+
+info_train_statistics_file_name = "t4dataset_gen2_base_statistics_train.parquet"
+info_val_statistics_file_name = "t4dataset_gen2_base_statistics_val.parquet"
+info_test_statistics_file_name = "t4dataset_gen2_base_statistics_test.parquet"
+
+# dataset scene setting
+dataset_version_list = [
+    "db_jpntaxigen2_v1",
+    "db_jpntaxigen2_v2",
+    "db_j6gen2_v1",
+    "db_j6gen2_v2",
+    "db_j6gen2_v3",
+    "db_j6gen2_v4",
+    "db_j6gen2_v5",
+    "db_j6gen2_v6",
+    "db_j6gen2_v7",
+    "db_j6gen2_v8",
+    "db_j6gen2_v9",
+    "db_j6gen2_v10",
+    "db_j6gen2_v11",
+    "db_j6gen2_v12",
+    "db_largebus_v1",
+    "db_largebus_v2",
+    "db_largebus_v3",
+]
+
+# TODO (KokSeang): This will be removed to avoid repeatitive computation
+# Dataset set, test info files, and enable/disable evaluation of prefix
+dataset_test_groups = {
+    "j6gen2_base": ("t4dataset_j6gen2_base_infos_test.pkl", False),
+    "j6gen2": ("t4dataset_j6gen2_infos_test.pkl", False),
+    "largebus": ("t4dataset_largebus_infos_test.pkl", False),
+    "jpntaxi_gen2": ("t4dataset_jpntaxi_gen2_infos_test.pkl", False),
+    "gen2_base": ("t4dataset_gen2_base_infos_test.pkl", True),
+}
+
+# dataset format setting
+data_prefix = dict(
+    pts="",
+    CAM_FRONT="",
+    CAM_FRONT_WIDE="",
+    CAM_FRONT_LEFT="",
+    CAM_FRONT_LEFT_WIDE="",
+    CAM_FRONT_RIGHT="",
+    CAM_FRONT_RIGHT_WIDE="",
+    CAM_BACK="",
+    CAM_BACK_RIGHT="",
+    CAM_BACK_RIGHT_WIDE="",
+    CAM_BACK_LEFT="",
+    CAM_BACK_LEFT_WIDE="",
+    sweeps="",
+)
+camera_types = {
+    "CAM_FRONT",
+    "CAM_FRONT_WIDE",
+    "CAM_FRONT_RIGHT",
+    "CAM_FRONT_RIGHT_WIDE",
+    "CAM_FRONT_LEFT",
+    "CAM_FRONT_LEFT_WIDE",
+    "CAM_BACK",
+    "CAM_BACK_LEFT",
+    "CAM_BACK_LEFT_WIDE",
+    "CAM_BACK_RIGHT",
+    "CAM_BACK_RIGHT_WIDE",
+}
+
+# class setting
+name_mapping = {
+    # DBv1.0
+    "vehicle.car": "car",
+    "vehicle.construction": "truck",
+    "vehicle.emergency (ambulance & police)": "car",
+    "vehicle.motorcycle": "bicycle",
+    "vehicle.trailer": "trailer",
+    "vehicle.truck": "truck",
+    "vehicle.bicycle": "bicycle",
+    "vehicle.bus (bendy & rigid)": "bus",
+    "pedestrian.adult": "pedestrian",
+    "pedestrian.child": "pedestrian",
+    "pedestrian.construction_worker": "pedestrian",
+    "pedestrian.personal_mobility": "pedestrian",
+    "pedestrian.police_officer": "pedestrian",
+    "pedestrian.stroller": "pedestrian",
+    "pedestrian.wheelchair": "pedestrian",
+    "movable_object.barrier": "barrier",
+    "movable_object.debris": "barrier",
+    "movable_object.pushable_pullable": "barrier",
+    "movable_object.trafficcone": "traffic_cone",
+    "movable_object.traffic_cone": "traffic_cone",
+    "animal": "animal",
+    "static_object.bicycle_rack": "bicycle_rack",
+    # DBv1.1 and UCv2.0
+    "car": "car",
+    "truck": "truck",
+    "bus": "bus",
+    "trailer": "trailer",
+    "motorcycle": "bicycle",
+    "bicycle": "bicycle",
+    "police_car": "car",
+    "pedestrian": "pedestrian",
+    "police_officer": "pedestrian",
+    "forklift": "car",
+    "construction_worker": "pedestrian",
+    "stroller": "pedestrian",
+    # DBv2.0 and DBv3.0
+    "animal": "animal",
+    "movable_object.barrier": "barrier",
+    "movable_object.pushable_pullable": "barrier",
+    "movable_object.traffic_cone": "traffic_cone",
+    "pedestrian.adult": "pedestrian",
+    "pedestrian.child": "pedestrian",
+    "pedestrian.construction_worker": "pedestrian",
+    "pedestrian.personal_mobility": "pedestrian",
+    "pedestrian.police_officer": "pedestrian",
+    "pedestrian.stroller": "pedestrian",
+    "pedestrian.wheelchair": "pedestrian",
+    "static_object.bicycle rack": "bicycle rack",
+    "static_object.bollard": "bollard",
+    "vehicle.ambulance": "car",  # Define vehicle.ambulance as car since vehicle.emergency (ambulance & police) is defined as car
+    "vehicle.bicycle": "bicycle",
+    "vehicle.bus": "bus",
+    "vehicle.car": "car",
+    "vehicle.construction": "truck",
+    "vehicle.fire": "truck",
+    "vehicle.motorcycle": "bicycle",
+    "vehicle.police": "car",
+    "vehicle.trailer": "trailer",
+    "vehicle.truck": "truck",
+    # DBv1.3
+    "ambulance": "car",
+    "kart": "car",
+    "wheelchair": "pedestrian",
+    "personal_mobility": "pedestrian",
+    "fire_truck": "truck",
+    "semi_trailer": "trailer",
+    "tractor_unit": "truck",
+    "construction_vehicle": "truck",
+    "traffic_cone": "traffic_cone",
+    "trafficcone": "traffic_cone",
+    "barrier": "barrier",
+    "other_vehicle": "car",
+    "other_pedestrian": "pedestrian",
+}
+
+class_names = ["car", "truck", "bus", "bicycle", "pedestrian", "traffic_cone", "barrier"]
+num_class = len(class_names)
+metainfo = dict(classes=class_names)
+
+merge_objects = [
+    ("truck", ["truck", "trailer"]),
+]
+merge_type = "extend_longer"  # One of ["extend_longer","union", None]
+
+# visualization
+class_colors = {
+    "car": (30, 144, 255),
+    "truck": (140, 0, 255),
+    "construction_vehicle": (255, 255, 0),
+    "bus": (111, 255, 111),
+    "trailer": (0, 255, 255),
+    "barrier": (0, 0, 0),
+    "motorcycle": (100, 0, 30),
+    "bicycle": (255, 0, 30),
+    "pedestrian": (255, 200, 200),
+    "traffic_cone": (120, 120, 120),
+}
+camera_panels = [
+    "data/CAM_FRONT_LEFT",
+    "data/CAM_FRONT",
+    "data/CAM_FRONT_RIGHT",
+    "data/CAM_BACK_LEFT",
+    "data/CAM_BACK",
+    "data/CAM_BACK_RIGHT",
+]
+
+# Add filter attributes
+filter_attributes = [
+    ("vehicle.bicycle", "vehicle_state.parked"),
+    ("vehicle.bicycle", "cycle_state.without_rider"),
+    ("vehicle.bicycle", "motorcycle_state.without_rider"),
+    ("vehicle.motorcycle", "vehicle_state.parked"),
+    ("vehicle.motorcycle", "cycle_state.without_rider"),
+    ("vehicle.motorcycle", "motorcycle_state.without_rider"),
+    ("bicycle", "vehicle_state.parked"),
+    ("bicycle", "cycle_state.without_rider"),
+    ("bicycle", "motorcycle_state.without_rider"),
+    ("motorcycle", "vehicle_state.parked"),
+    ("motorcycle", "cycle_state.without_rider"),
+    ("motorcycle", "motorcycle_state.without_rider"),
+]
+
+evaluator_metric_configs = dict(
+    evaluation_task="detection",
+    target_labels=class_names,
+    center_distance_bev_thresholds=[0.5, 1.0, 2.0, 4.0],
+    # plane_distance_thresholds is required for the pass fail evaluation
+    plane_distance_thresholds=[2.0, 4.0],
+    iou_2d_thresholds=None,
+    iou_3d_thresholds=None,
+    label_prefix="autoware",
+    # bev minimum distance ranges for each range bucket, must be the same length as max_distance,
+    # they will form bev distance ranges in [(min_distance[0], max_distance[0]), (min_distance[1], max_distance[1]), ...] when filtering
+    min_distance=[0.0, 50.0, 90.0, 0.0],
+    # bev maximum distance ranges for each range bucket, must be the same length as min_distance
+    max_distance=[50.0, 90.0, 121.0, 121.0],
+    min_point_numbers=0,
+    matching_class_agnostic_fps=False,
+)

autoware_ml/configs/detection3d/default_runtime.py

@@ -2,9 +2,17 @@
 
 default_hooks = dict(
     timer=dict(type="IterTimerHook"),
-    logger=dict(type="LoggerHook", interval=50),
+    logger=dict(
+        type="LoggerHook",
+        interval=50,
+        backend_args=dict(backend="local"),
+    ),
     param_scheduler=dict(type="ParamSchedulerHook"),
-    checkpoint=dict(type="CheckpointHook", interval=-1),
+    checkpoint=dict(
+        type="CheckpointHook",
+        interval=-1,
+        backend_args=dict(backend="local"),
+    ),
     sampler_seed=dict(type="DistSamplerSeedHook"),
     visualization=dict(type="Det3DVisualizationHook"),
 )

autoware_ml/detection3d/datasets/t4dataset.py

@@ -2,6 +2,7 @@
 from typing import List
 
 import numpy as np
+import tqdm
 from mmdet3d.datasets import NuScenesDataset
 from mmengine.logging import print_log
 from mmengine.registry import DATASETS
@@ -51,21 +52,27 @@ def filter_data(self) -> List[dict]:
         if not self.filter_cfg:
             return self.data_list
 
-        filter_frames_with_camera_order = self.filter_cfg.get("filter_frames_with_camera_order", None)
-        if filter_frames_with_camera_order is None:
+        filter_frames_with_camera_orders = self.filter_cfg.get("filter_frames_with_camera_orders", None)
+        if filter_frames_with_camera_orders is None:
             return self.data_list
 
         filtered_data_list = []
-        for entry in self.data_list:
+        for entry in tqdm.tqdm(self.data_list, desc="Filtering data"):
+            vehicle_type = entry.get("vehicle_type", None)
+            if vehicle_type is None:
+                raise KeyError(f"Missing 'vehicle_type' in entry: {entry}")
+
+            filter_frames_with_camera_order = filter_frames_with_camera_orders.get(vehicle_type, None)
+            if filter_frames_with_camera_order is None:
+                raise KeyError(f"Missing camera order for vehicle type '{vehicle_type}' in filter configuration.")
+
             filtered = False
             for camera_order in filter_frames_with_camera_order:
                 if camera_order not in entry["images"]:
                     filtered = True
                     break
 
-                if entry["images"][camera_order]["img_path"] is None or not osp.exists(
-                    entry["images"][camera_order]["img_path"]
-                ):
+                if entry["images"][camera_order]["img_path"] is None:
                     filtered = True
                     break
 
@@ -180,6 +187,7 @@ def parse_data_info(self, info: dict) -> dict:
                             cam_prefix,
                             img_info["img_path"],
                         )
+                    # print_log(f"Camera path: {img_info['img_path']}", logger="current")
 
             if self.default_cam_key is not None:
                 info["img_path"] = info["images"][self.default_cam_key]["img_path"]
@@ -192,4 +200,7 @@ def parse_data_info(self, info: dict) -> dict:
                 else:
                     info["lidar2img"] = info["cam2img"] @ info["lidar2cam"]
 
+        # Default difficulty to 0 if not present
+        if "difficulty" not in info:
+            info["difficulty"] = 0
         return info

autoware_ml/detection3d/evaluation/t4metric/t4metric.py

@@ -262,8 +262,7 @@ def _parse_ground_truth_from_sample(self, data_sample: Dict[str, Any]) -> dict:
             "num_lidar_pts": num_lidar_pts,
         }
 
-    @staticmethod
-    def _get_scene_info(data_infos: List[dict]) -> Tuple[List[str], List[str]]:
+    def _get_scene_info(self, data_infos: List[dict]) -> Tuple[List[str], List[str]]:
         """Get scene tokens and directory names from data infos.
 
         Args:
@@ -284,6 +283,7 @@ def _get_scene_info(data_infos: List[dict]) -> Tuple[List[str], List[str]]:
             if directory not in directories:
                 scene_tokens.append(scene_token)
                 directories.append(directory)
+
         return scene_tokens, directories
 
     @staticmethod

projects/BEVFusion/bevfusion/__init__.py

@@ -3,10 +3,17 @@
 from .bevfusion_necks import GeneralizedLSSFPN
 from .bevfusion_voxel_encoder import HardSimpleVoxelSinCosEncoder
 from .depth_lss import DepthLSSTransform, LSSTransform
-from .loading import BEVLoadMultiViewImageFromFiles
+from .depth_lss_v2 import LSSTransformV2, LSSTransformV2DepthAware
+from .loading import BEVLoadMultiViewImageFromFiles, PointsToMultiViewImageDepths
 from .sparse_encoder import BEVFusionSparseEncoder
 from .transformer import TransformerDecoderLayer
-from .transforms_3d import BEVFusionGlobalRotScaleTrans, BEVFusionRandomFlip3D, GridMask, ImageAug3D
+from .transforms_3d import (
+    BEVFusionGlobalRotScaleTrans,
+    BEVFusionRandomFlip3D,
+    BEVFusionRemoveLiDARPoints,
+    GridMask,
+    ImageAug3D,
+)
 from .utils import BBoxBEVL1Cost, HeuristicAssigner3D, HungarianAssigner3D, IoU3DCost, TransFusionBBoxCoder
 
 __all__ = [
@@ -27,6 +34,10 @@
     "TransformerDecoderLayer",
     "BEVFusionRandomFlip3D",
     "BEVFusionGlobalRotScaleTrans",
+    "BEVFusionRemoveLiDARPoints",
     "TransFusionBBoxCoder",
     "HardSimpleVoxelSinCosEncoder",
+    "LSSTransformV2",
+    "PointsToMultiViewImageDepths",
+    "LSSTransformV2DepthAware",
 ]