diff --git a/apps/dino-tracking/backend/requirements.txt b/apps/dino-tracking/backend/requirements.txt index 6aa897b80..7ad4908ff 100644 --- a/apps/dino-tracking/backend/requirements.txt +++ b/apps/dino-tracking/backend/requirements.txt @@ -1,5 +1,5 @@ -depthai==3.7.1 -depthai-nodes==0.5.0 +depthai==3.8.0 +depthai-nodes==0.6.0 opencv-python-headless~=4.10.0 python-dotenv python-box diff --git a/apps/dino-tracking/backend/src/annotations/outlines_overlay_node.py b/apps/dino-tracking/backend/src/annotations/outlines_overlay_node.py index 43f58f297..f8b07b276 100644 --- a/apps/dino-tracking/backend/src/annotations/outlines_overlay_node.py +++ b/apps/dino-tracking/backend/src/annotations/outlines_overlay_node.py @@ -1,7 +1,6 @@ import cv2 import depthai as dai import numpy as np -from depthai_nodes.message import SegmentationMask from depthai_nodes.node.base_host_node import BaseHostNode @@ -31,12 +30,8 @@ def process(self, frame_msg: dai.ImgFrame, segmentation: dai.Buffer): self.out.send(frame_msg) return - assert isinstance(segmentation, SegmentationMask) - mask = getattr(segmentation, "mask", None) - - if mask is None: - self.out.send(frame_msg) - return + assert isinstance(segmentation, dai.SegmentationMask) + mask = segmentation.getCvMask() frame = frame_msg.getCvFrame() H, W = frame.shape[:2] diff --git a/apps/dino-tracking/backend/src/object_selection/mask_selection_node.py b/apps/dino-tracking/backend/src/object_selection/mask_selection_node.py index b59c72b00..d9000fb43 100644 --- a/apps/dino-tracking/backend/src/object_selection/mask_selection_node.py +++ b/apps/dino-tracking/backend/src/object_selection/mask_selection_node.py @@ -1,6 +1,5 @@ import depthai as dai import numpy as np -from depthai_nodes.message import SegmentationMask from depthai_nodes.node import BaseHostNode @@ -31,10 +30,8 @@ def clear_selection(self) -> None: self._pending_click = None self._selected_mask = None - def process(self, segmentation: dai.Buffer): - assert isinstance(segmentation, SegmentationMask) - - segmentation_mask = segmentation.mask.astype(np.int32) + def process(self, segmentation: dai.SegmentationMask): + segmentation_mask = segmentation.getCvMask().astype(np.int32) if self._pending_click: segment_id = self._map_click_to_segment( @@ -78,10 +75,17 @@ def _map_click_to_segment( if patch.size == 0: return None - values, counts = np.unique(patch, return_counts=True) + # Native dai.SegmentationMask uses 255 as background/unassigned. + # Do not select the background as an object, while keeping class/instance + # id 0 valid. + foreground_patch = patch[patch != 255] + if foreground_patch.size == 0: + return None + + values, counts = np.unique(foreground_patch, return_counts=True) return int(values[np.argmax(counts)]) - def _send_mask(self, segmentation: SegmentationMask, mask: np.ndarray): + def _send_mask(self, segmentation: dai.SegmentationMask, mask: np.ndarray): mask_u8 = mask.astype(np.uint8) * 255 out = dai.ImgFrame() diff --git a/neural-networks/generic-example/AGENTS.md b/neural-networks/generic-example/AGENTS.md index dfac644eb..f15f0afaf 100644 --- a/neural-networks/generic-example/AGENTS.md +++ b/neural-networks/generic-example/AGENTS.md @@ -86,7 +86,7 @@ Reusable single-model inference scaffold. It runs one Model Zoo model with one i ## Validation - `Run:` `python3 main.py` -- `Alternative run:` `python3 main.py --model luxonis/mediapipe-selfie-segmentation:256x144 --overlay_mode` +- `Alternative run:` `python3 main.py --model luxonis/mediapipe-selfie-segmentation:256x144` - `Archive run:` `python3 main.py --model /path/to/custom-model.tar.xz` - `Success looks like:` Visualizer exposes `Video` and `Detections`, and the pipeline runs until `q` is pressed - `Common failure meaning:` model identifier unavailable for platform, private model auth missing, or selected model violates the single-input/single-output assumptions diff --git a/neural-networks/generic-example/README.md b/neural-networks/generic-example/README.md index 1d64a3fd4..dc8f6d068 100644 --- a/neural-networks/generic-example/README.md +++ b/neural-networks/generic-example/README.md @@ -23,7 +23,7 @@ Here is a list of all available parameters: -api API_KEY, --api_key API_KEY HubAI API key for private HubAI access. Can also use 'DEPTHAI_HUB_API_KEY' environment variable instead. (default: ) -overlay OVERLAY_MODE, --overlay_mode - If passed, overlays model output on the input image when the output is an array (e.g., depth maps, segmentation maps). Otherwise, displays outputs separately. + If passed, overlays model output on the input image when the output is an array (e.g., depth maps). Otherwise, displays outputs separately. ``` ## Peripheral Mode @@ -53,20 +53,26 @@ This will run a simple YOLOv6 object detection model (`luxonis/yolov6-nano:r2-co ```bash python3 main.py \ - --model luxonis/mediapipe-selfie-segmentation:256x144 \ - --overlay_mode + --model luxonis/mediapipe-selfie-segmentation:256x144 ``` This will run a selfie segmentation model. ```bash python3 main.py \ - --model luxonis/yolov8-instance-segmentation-nano:coco-512x288 \ - --overlay_mode + --model luxonis/yolov8-instance-segmentation-nano:coco-512x288 ``` And this will run an instance segmentation model. +```bash +python3 main.py \ + --model luxonis/midas-v2-1:small-512x288 \ + --overlay_mode +``` + +This will run a MiDaS depth estimation model and overlay the depth map on the input image. + ```bash python3 main.py \ --model /path/to/custom-model.tar.xz diff --git a/neural-networks/generic-example/requirements.txt b/neural-networks/generic-example/requirements.txt index 55e174d94..2d794695a 100644 --- a/neural-networks/generic-example/requirements.txt +++ b/neural-networks/generic-example/requirements.txt @@ -1,5 +1,5 @@ -depthai==3.7.1 -depthai-nodes==0.5.0 +depthai==3.8.0 +depthai-nodes==0.6.0 opencv-python-headless~=4.10.0 numpy>=1.22 python-dotenv diff --git a/neural-networks/generic-example/utils/arguments.py b/neural-networks/generic-example/utils/arguments.py index cd412a9a9..6895d1080 100644 --- a/neural-networks/generic-example/utils/arguments.py +++ b/neural-networks/generic-example/utils/arguments.py @@ -60,7 +60,7 @@ def initialize_argparser(): parser.add_argument( "-overlay", "--overlay_mode", - help="If passed, overlays model output on the input image when the output is an array (e.g., depth maps, segmentation maps). Otherwise, displays outputs separately.", + help="If passed, overlays model output on the input image when the output is an array (e.g., depth maps). Otherwise, displays outputs separately.", required=False, action="store_true", ) diff --git a/neural-networks/object-detection/yolo-p/AGENTS.md b/neural-networks/object-detection/yolo-p/AGENTS.md index 85a2aa2d9..5beb535dd 100644 --- a/neural-networks/object-detection/yolo-p/AGENTS.md +++ b/neural-networks/object-detection/yolo-p/AGENTS.md @@ -27,7 +27,7 @@ This is the repository reference for ADAS-style YOLO-P output: detections plus r - `Runs on:` RVC2 peripheral, RVC4 peripheral, and RVC4 standalone packaging - `Requires:` YOLO-P model and input resized to model dimensions - `Input:` camera frames by default or `ReplayVideo` via `--media_path` -- `Output:` `Road Segmentation` and `Detections` +- `Output:` `Video`, `Detections`, `Road Segmentation`, and `Lane Segmentation` - `Models:` YOLO-P YAMLs in [depthai_models/](depthai_models/) - `Visualizer / UI:` DepthAI Visualizer via `dai.RemoteConnection` @@ -35,17 +35,16 @@ This is the repository reference for ADAS-style YOLO-P output: detections plus r - [README.md](README.md) - [main.py](main.py) -- [utils/annotation_node.py](utils/annotation_node.py) - [utils/arguments.py](utils/arguments.py) ## Architecture - `ParsingNeuralNetwork` runs the YOLO-P multi-head model on camera or replay input. -- [utils/annotation_node.py](utils/annotation_node.py) consumes three outputs: +- [main.py](main.py) sends the model outputs to Visualizer topics: - detections - road segmentation - lane segmentation -- The Visualizer exposes the segmentation composite and detection overlay separately. +- The Visualizer renders the input video, detections, road segmentation, and lane segmentation topics. ## Constraints @@ -62,5 +61,5 @@ This is the repository reference for ADAS-style YOLO-P output: detections plus r ## Validation - `Run:` `python3 main.py` -- `Success looks like:` the Visualizer shows `Road Segmentation` and `Detections`, with lane/road overlays and detected vehicles +- `Success looks like:` the Visualizer shows `Video`, `Detections`, `Road Segmentation`, and `Lane Segmentation`, with road/lane segmentation and detected vehicles - `Common failure meaning:` the model output ordering changed, the replay input was resized incorrectly, or the operator expected a generic single-output detector diff --git a/neural-networks/object-detection/yolo-p/main.py b/neural-networks/object-detection/yolo-p/main.py index 67ec94364..3739e2dbc 100644 --- a/neural-networks/object-detection/yolo-p/main.py +++ b/neural-networks/object-detection/yolo-p/main.py @@ -4,7 +4,6 @@ from depthai_nodes.node import ParsingNeuralNetwork from utils.arguments import initialize_argparser -from utils.annotation_node import AnnotationNode _, args = initialize_argparser() @@ -54,19 +53,11 @@ input_node_out, nn_archive ) - # annotation - annotation_node = pipeline.create(AnnotationNode).build( - frame=input_node_out, - detections=nn.getOutput(0), - road_segmentations=nn.getOutput(1), - lane_segmentations=nn.getOutput(2), - ) - # visualization - visualizer.addTopic( - "Road Segmentation", annotation_node.out_segmentations, "images" - ) - visualizer.addTopic("Detections", annotation_node.out_detections, "images") + visualizer.addTopic("Video", nn.passthrough, "images") + visualizer.addTopic("Detections", nn.getOutput(0), "images") + visualizer.addTopic("Road Segmentation", nn.getOutput(1), "images") + visualizer.addTopic("Lane Segmentation", nn.getOutput(2), "images") print("Pipeline created.") diff --git a/neural-networks/object-detection/yolo-p/requirements.txt b/neural-networks/object-detection/yolo-p/requirements.txt index 5b2c84208..d4a6f982f 100644 --- a/neural-networks/object-detection/yolo-p/requirements.txt +++ b/neural-networks/object-detection/yolo-p/requirements.txt @@ -1,4 +1,4 @@ -depthai==3.7.1 -depthai-nodes==0.5.0 +depthai==3.8.0 +depthai-nodes==0.6.0 opencv-python-headless~=4.10.0 numpy>=1.22 diff --git a/neural-networks/object-detection/yolo-p/utils/annotation_node.py b/neural-networks/object-detection/yolo-p/utils/annotation_node.py deleted file mode 100644 index 7189d58d0..000000000 --- a/neural-networks/object-detection/yolo-p/utils/annotation_node.py +++ /dev/null @@ -1,79 +0,0 @@ -import depthai as dai -from depthai_nodes import SegmentationMask - -import cv2 -import numpy as np - - -class AnnotationNode(dai.node.HostNode): - def __init__( - self, - ) -> None: - super().__init__() - self.out_segmentations = self.createOutput() - self.out_detections = self.createOutput() - - def build( - self, - frame: dai.Node.Output, - detections: dai.Node.Output, - road_segmentations: dai.Node.Output, - lane_segmentations: dai.Node.Output, - ) -> "AnnotationNode": - self.link_args(frame, detections, road_segmentations, lane_segmentations) - return self - - def process( - self, - frame: dai.Buffer, - detections_message: dai.Buffer, - road_segmentations_message: dai.Buffer, - lane_segmentations_message: dai.Buffer, - ) -> None: - assert isinstance(frame, dai.ImgFrame) - assert isinstance(detections_message, dai.ImgDetections) - assert isinstance(road_segmentations_message, SegmentationMask) - assert isinstance(lane_segmentations_message, SegmentationMask) - - detections_message.setTransformation(frame.getTransformation()) - - frame = frame.getCvFrame() - output_frame = dai.ImgFrame() - - mask = road_segmentations_message.mask - - lane_segmentation_mask = lane_segmentations_message.mask > 0 - mask[lane_segmentation_mask] = 2 - - unique_values = np.unique(mask[mask >= 0]) - scaled_mask = np.zeros_like(mask, dtype=np.uint8) - - if unique_values.size != 0: - min_val, max_val = unique_values.min(), unique_values.max() - - if min_val == max_val: - scaled_mask = np.ones_like(mask, dtype=np.uint8) * 255 - else: - scaled_mask = ((mask - min_val) / (max_val - min_val) * 255).astype( - np.uint8 - ) - scaled_mask[mask == -1] = 0 - colored_mask = cv2.applyColorMap(scaled_mask, cv2.COLORMAP_RAINBOW) - colored_mask[mask == 0] = [0, 0, 0] - colored_mask[mask == -1] = [0, 0, 0] - - frame_height, frame_width, _ = frame.shape - colored_mask = cv2.resize( - colored_mask, (frame_width, frame_height), interpolation=cv2.INTER_AREA - ) - - colored_frame = cv2.addWeighted(frame, 0.8, colored_mask, 0.5, 0) - - output_frame.setTimestamp(detections_message.getTimestamp()) - output_frame.setSequenceNum(detections_message.getSequenceNum()) - - self.out_detections.send(detections_message) - - self.out_segmentations.send( - output_frame.setCvFrame(colored_frame, dai.ImgFrame.Type.BGR888i) - ) diff --git a/neural-networks/segmentation/blur-background/main.py b/neural-networks/segmentation/blur-background/main.py index b2426516e..f9870cdc7 100644 --- a/neural-networks/segmentation/blur-background/main.py +++ b/neural-networks/segmentation/blur-background/main.py @@ -44,6 +44,7 @@ seg_nn: ParsingNeuralNetwork = pipeline.create(ParsingNeuralNetwork).build( input_node, seg_model_description, fps=args.fps_limit ) + seg_nn.getParser(dai.node.SegmentationParser).setBackgroundClass(False) blur_background = pipeline.create(BlurBackground).build( seg_nn.passthrough, seg_nn.out diff --git a/neural-networks/segmentation/blur-background/requirements.txt b/neural-networks/segmentation/blur-background/requirements.txt index 5b2c84208..d4a6f982f 100644 --- a/neural-networks/segmentation/blur-background/requirements.txt +++ b/neural-networks/segmentation/blur-background/requirements.txt @@ -1,4 +1,4 @@ -depthai==3.7.1 -depthai-nodes==0.5.0 +depthai==3.8.0 +depthai-nodes==0.6.0 opencv-python-headless~=4.10.0 numpy>=1.22 diff --git a/neural-networks/segmentation/blur-background/utils/blur_detections.py b/neural-networks/segmentation/blur-background/utils/blur_detections.py index ca2070a74..0185670d4 100644 --- a/neural-networks/segmentation/blur-background/utils/blur_detections.py +++ b/neural-networks/segmentation/blur-background/utils/blur_detections.py @@ -1,6 +1,5 @@ import cv2 import depthai as dai -from depthai_nodes.message import SegmentationMask class BlurBackground(dai.node.HostNode): @@ -34,12 +33,11 @@ def process( """Blurs the background of the input frame based on the segmentation mask.""" assert isinstance(frame_msg, dai.ImgFrame) - assert isinstance(mask_msg, SegmentationMask) + assert isinstance(mask_msg, dai.SegmentationMask) frame = frame_msg.getCvFrame() - person_mask = ( - mask_msg.mask == 15 - ) # person is class 15 in the output of the model + mask = mask_msg.getCvMask() + person_mask = mask == 15 # person is class 15 in the output of the model bg = frame.copy() blurred_bg = cv2.blur(bg, (10, 10)) diff --git a/neural-networks/segmentation/depth-crop/main.py b/neural-networks/segmentation/depth-crop/main.py index 97251240b..3a3a1f7c4 100755 --- a/neural-networks/segmentation/depth-crop/main.py +++ b/neural-networks/segmentation/depth-crop/main.py @@ -64,6 +64,7 @@ nn = pipeline.create(ParsingNeuralNetwork).build( nnSource=nn_archive, input=manip.out ) + nn.getParser(dai.node.SegmentationParser).setBackgroundClass(False) # annotation annotation_node = pipeline.create(AnnotationNode).build( diff --git a/neural-networks/segmentation/depth-crop/requirements.txt b/neural-networks/segmentation/depth-crop/requirements.txt index 42a19107e..1fbf5f6a1 100644 --- a/neural-networks/segmentation/depth-crop/requirements.txt +++ b/neural-networks/segmentation/depth-crop/requirements.txt @@ -1,2 +1,2 @@ -depthai==3.7.1 -depthai-nodes==0.5.0 +depthai==3.8.0 +depthai-nodes==0.6.0 diff --git a/neural-networks/segmentation/depth-crop/utils/annotation_node.py b/neural-networks/segmentation/depth-crop/utils/annotation_node.py index 10516e8dc..77f4d3118 100644 --- a/neural-networks/segmentation/depth-crop/utils/annotation_node.py +++ b/neural-networks/segmentation/depth-crop/utils/annotation_node.py @@ -1,7 +1,7 @@ import cv2 import depthai as dai import numpy as np -from depthai_nodes import SegmentationMask, PRIMARY_COLOR +from depthai_nodes import PRIMARY_COLOR # Custom colormap with 0 mapped to black - better disparity visualization JET_CUSTOM = cv2.applyColorMap(np.arange(256, dtype=np.uint8), cv2.COLORMAP_JET) @@ -46,10 +46,14 @@ def process( ) -> None: frame = preview.getCvFrame() - assert isinstance(mask, SegmentationMask) + assert isinstance(mask, dai.SegmentationMask) - mask_data = mask.mask - mask_data = cv2.resize(mask_data, (frame.shape[1], frame.shape[0])) + mask_data = mask.getCvMask() + mask_data = cv2.resize( + mask_data, + (frame.shape[1], frame.shape[0]), + interpolation=cv2.INTER_NEAREST, + ) mask = np.zeros_like(frame) color = [