diff --git a/README.md b/README.md index 0c6a756..6bac455 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,16 @@ Triptych360 Banner -# Triptych360 - 360 Video Wall Controller +# Triptych360 - 360 Media + 3D Video Wall Controller -Triptych360 is a Python application built with PySide6 and ModernGL that takes a 360-degree equirectangular image or video and projects it in real-time onto three separate windows representing Left, Front, and Right views. This is designed for video wall installations mapped in a U-shape. +Triptych360 is a Python application built with PySide6 and ModernGL that takes a 360-degree equirectangular image/video or a 3D point-based model and projects it in real-time onto three separate windows representing Left, Front, and Right views. This is designed for video wall installations mapped in a U-shape. The projection mapping is entirely GPU-accelerated using GLSL fragment shaders, making it highly performant and capable of decoding high-framerate videos smoothly. ## Features - **GPU-Accelerated**: Fast real-time Equirectangular to Rectilinear conversion. - **Video Playback**: Native support for auto-looping 360 videos (`.mp4`, `.mkv`, `.avi`, `.mov`) via OpenCV. +- **3D Model Support**: Load LiDAR point clouds from `.las` / `.laz` and Gaussian splats from `.ply` / `.splat`. - **Multi-Window Display & Kiosk Mode**: Automatically opens up 3 separate views. Check the "Kiosk Mode" box to make them frameless and smoothly lock onto your 3 monitors. The projection math cleanly locks the edges to precisely 90 degrees horizontally, ensuring a seamless U-shape layout. - **Interactive Unified Controller**: Use your mouse to rotate and zoom across all display outputs instantly. - **3Dconnexion SpaceMouse Support**: Move around your panoramas smoothly using native SpaceMouse integration. @@ -36,6 +37,15 @@ The projection mapping is entirely GPU-accelerated using GLSL fragment shaders, pip install -r requirements.txt ``` +## Supported File Types + +- **360 Media** + - Images: `.png`, `.jpg`, `.jpeg` + - Videos: `.mp4`, `.mkv`, `.avi`, `.mov` +- **3D Data** + - LiDAR point clouds: `.las`, `.laz` + - Gaussian splats: `.ply`, `.splat` + ## Usage 1. **Launch the application:** @@ -44,8 +54,8 @@ The projection mapping is entirely GPU-accelerated using GLSL fragment shaders, ``` 2. **Load Media:** - * A controller window will appear. Click "Load Equirectangular Media". - * You can test with the provided `sample_equirectangular.jpg` or load any supported video/image file. + * A controller window will appear. Click **"Load Media / 3D Model"**. + * You can test with the provided `sample_equirectangular.jpg` or load any supported video/image/3D file (`.las`, `.laz`, `.ply`, `.splat`). * Three new windows will open showing the Left, Front, and Right segments of the media. 3. **Navigate & Control:** diff --git a/__pycache__/triptych360.cpython-312.pyc b/__pycache__/triptych360.cpython-312.pyc new file mode 100644 index 0000000..93e3736 Binary files /dev/null and b/__pycache__/triptych360.cpython-312.pyc differ diff --git a/requirements.txt b/requirements.txt index feeced1..803cb82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ numpy Pillow opencv-python pyspacemouse +laspy[lazrs] +plyfile diff --git a/triptych360.py b/triptych360.py index a385664..da05e4b 100644 --- a/triptych360.py +++ b/triptych360.py @@ -5,6 +5,8 @@ import json import os from PIL import Image +import laspy +from plyfile import PlyData from PySide6.QtWidgets import QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QFileDialog, QCheckBox from PySide6.QtOpenGLWidgets import QOpenGLWidget @@ -67,6 +69,35 @@ } """ +POINT_VERTEX_SHADER = """ +#version 330 core +in vec3 in_pos; +in vec4 in_color; +in float in_size; +uniform mat4 mvp; +out vec4 v_color; +void main() { + gl_Position = mvp * vec4(in_pos, 1.0); + gl_PointSize = max(1.0, in_size); + v_color = in_color; +} +""" + +POINT_FRAGMENT_SHADER = """ +#version 330 core +in vec4 v_color; +out vec4 f_color; +void main() { + vec2 uv = gl_PointCoord * 2.0 - 1.0; + float r2 = dot(uv, uv); + if (r2 > 1.0) { + discard; + } + float falloff = exp(-r2 * 4.0); + f_color = vec4(v_color.rgb, v_color.a * falloff); +} +""" + # ========================================== # OpenGL ViewPort Widget # ========================================== @@ -83,6 +114,17 @@ def __init__(self, yaw_offset_deg=0.0, frameless=False): self.vbo = None self.vao = None self.texture = None + self.render_mode = "pano" + + self.point_prog = None + self.point_vao = None + self.point_pos_vbo = None + self.point_color_vbo = None + self.point_size_vbo = None + self.point_count = 0 + self.point_positions = None + self.point_colors = None + self.point_sizes = None self.raw_image_data = None self.raw_width = 0 @@ -96,11 +138,14 @@ def __init__(self, yaw_offset_deg=0.0, frameless=False): def initializeGL(self): self.ctx = moderngl.create_context() self.prog = self.ctx.program(vertex_shader=VERTEX_SHADER, fragment_shader=FRAGMENT_SHADER) + self.point_prog = self.ctx.program(vertex_shader=POINT_VERTEX_SHADER, fragment_shader=POINT_FRAGMENT_SHADER) vertices = np.array([-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0], dtype='f4') self.vbo = self.ctx.buffer(vertices) self.vao = self.ctx.simple_vertex_array(self.prog, self.vbo, 'in_vert') if self.raw_image_data is not None: self._create_gl_texture() + if self.point_positions is not None: + self._create_point_buffers() def _create_gl_texture(self): if self.texture: @@ -120,7 +165,21 @@ def paintGL(self): fbo = self.ctx.detect_framebuffer() fbo.use() fbo.clear(0.1, 0.1, 0.1) - if self.texture is None: return + + if self.render_mode == "points": + if self.point_vao is None or self.point_count == 0: + return + self.ctx.enable(moderngl.DEPTH_TEST | moderngl.BLEND) + self.ctx.blend_func = moderngl.SRC_ALPHA, moderngl.ONE_MINUS_SRC_ALPHA + final_yaw = self.yaw_deg + self.yaw_offset_deg + mvp = self._compute_mvp_matrix(self.pitch_deg, final_yaw, self.fov_deg) + self.point_prog['mvp'].write(mvp.astype('f4').T.tobytes()) + self.point_vao.render(moderngl.POINTS) + return + + self.ctx.disable(moderngl.DEPTH_TEST) + if self.texture is None: + return self.texture.use(0) if 'tex' in self.prog: self.prog['tex'].value = 0 if 'aspect_ratio' in self.prog: self.prog['aspect_ratio'].value = self.aspect_ratio @@ -139,6 +198,7 @@ def update_texture_data(self, image_data): self.update() def set_texture_data(self, image_data, width, height): + self.render_mode = "pano" self.raw_image_data = image_data self.raw_width = width self.raw_height = height @@ -146,6 +206,71 @@ def set_texture_data(self, image_data, width, height): self.makeCurrent() self._create_gl_texture() self.update() + + def set_point_data(self, points, colors, sizes): + self.render_mode = "points" + self.point_positions = np.ascontiguousarray(points.astype('f4')) + self.point_colors = np.ascontiguousarray(colors.astype('f4')) + self.point_sizes = np.ascontiguousarray(sizes.astype('f4')) + self.point_count = self.point_positions.shape[0] + if self.ctx is not None: + self.makeCurrent() + self._create_point_buffers() + self.update() + + def _create_point_buffers(self): + if self.point_vao is not None: + self.point_vao.release() + self.point_vao = None + for attr in ("point_pos_vbo", "point_color_vbo", "point_size_vbo"): + buf = getattr(self, attr) + if buf is not None: + buf.release() + setattr(self, attr, None) + + self.point_pos_vbo = self.ctx.buffer(self.point_positions.tobytes()) + self.point_color_vbo = self.ctx.buffer(self.point_colors.tobytes()) + self.point_size_vbo = self.ctx.buffer(self.point_sizes.tobytes()) + self.point_vao = self.ctx.vertex_array(self.point_prog, [ + (self.point_pos_vbo, '3f', 'in_pos'), + (self.point_color_vbo, '4f', 'in_color'), + (self.point_size_vbo, '1f', 'in_size') + ]) + + def _compute_mvp_matrix(self, pitch_deg, yaw_deg, fov_deg): + fov = math.radians(max(20.0, min(150.0, fov_deg))) + near, far = 0.01, 100.0 + tan_half = math.tan(fov / 2.0) + proj = np.array([ + [1.0 / (self.aspect_ratio * tan_half), 0.0, 0.0, 0.0], + [0.0, 1.0 / tan_half, 0.0, 0.0], + [0.0, 0.0, -(far + near) / (far - near), -(2.0 * far * near) / (far - near)], + [0.0, 0.0, -1.0, 0.0] + ], dtype='f4') + + pitch = math.radians(pitch_deg) + yaw = math.radians(yaw_deg) + cp, sp = math.cos(pitch), math.sin(pitch) + cy, sy = math.cos(yaw), math.sin(yaw) + rx = np.array([ + [1.0, 0.0, 0.0, 0.0], + [0.0, cp, -sp, 0.0], + [0.0, sp, cp, 0.0], + [0.0, 0.0, 0.0, 1.0] + ], dtype='f4') + ry = np.array([ + [cy, 0.0, -sy, 0.0], + [0.0, 1.0, 0.0, 0.0], + [sy, 0.0, cy, 0.0], + [0.0, 0.0, 0.0, 1.0] + ], dtype='f4') + model = np.array([ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, -2.5], + [0.0, 0.0, 0.0, 1.0] + ], dtype='f4') + return proj @ (ry @ rx @ model) def update_view(self, pitch, yaw, fov): self.pitch_deg = pitch @@ -185,6 +310,7 @@ def __init__(self): self.image_data = None self.image_w = 0 self.image_h = 0 + self.scene_loaded = False self.video_cap = None self.video_timer = QTimer(self) @@ -240,13 +366,13 @@ def __init__(self): layout.setContentsMargins(40, 40, 40, 40) layout.setSpacing(20) - self.info_lbl = QLabel("No media loaded.") + self.info_lbl = QLabel("No media or 3D model loaded.") self.info_lbl.setObjectName("TitleLabel") self.info_lbl.setAlignment(Qt.AlignCenter) self.info_lbl.setWordWrap(True) layout.addWidget(self.info_lbl) - btn_load = QPushButton("Load Equirectangular Media") + btn_load = QPushButton("Load Media / 3D Model") btn_load.setCursor(Qt.PointingHandCursor) btn_load.clicked.connect(self.load_media_prompt) layout.addWidget(btn_load) @@ -284,7 +410,12 @@ def reset_idle(self): self.idle_time = 0.0 def load_media_prompt(self): - path, _ = QFileDialog.getOpenFileName(self, "Open 360 Media", "", "Media (*.png *.jpg *.jpeg *.mp4 *.mkv *.avi *.mov)") + path, _ = QFileDialog.getOpenFileName( + self, + "Open 360 Media / 3D Data", + "", + "Media & 3D (*.png *.jpg *.jpeg *.mp4 *.mkv *.avi *.mov *.las *.laz *.ply *.splat)" + ) if path: self.load_media_file(path) @@ -295,8 +426,9 @@ def load_media_file(self, path): self.video_cap = None sm_status = " | SpaceMouse ✅" if self.sm_device else " | UI Idle Pan ✅" + path_lower = str(path).lower() - if str(path).lower().endswith(('.mp4', '.mkv', '.avi', '.mov')): + if path_lower.endswith(('.mp4', '.mkv', '.avi', '.mov')): self.video_cap = cv2.VideoCapture(path) if not self.video_cap.isOpened(): self.info_lbl.setText("Error: Could not open video file.") @@ -311,11 +443,26 @@ def load_media_file(self, path): frame = cv2.flip(frame, 0) self.image_data = frame.tobytes() self.image_path = path + self.scene_loaded = True self.info_lbl.setText(f"Loaded Video: {os.path.basename(path)} @ {fps}fps{sm_status}") for view in self.view_windows: view.set_texture_data(self.image_data, self.image_w, self.image_h) self.position_windows() self.video_timer.start(int(1000 / fps)) + elif path_lower.endswith(('.las', '.laz')): + try: + points, colors, sizes = self._load_las_laz(path) + self._apply_point_scene(points, colors, sizes, f"Loaded Point Cloud: {os.path.basename(path)}{sm_status}") + except Exception as e: + self.info_lbl.setText(f"Error loading LAS/LAZ: {str(e)}") + self.scene_loaded = False + elif path_lower.endswith(('.ply', '.splat')): + try: + points, colors, sizes = self._load_gaussian_splats(path) + self._apply_point_scene(points, colors, sizes, f"Loaded Gaussian Splats: {os.path.basename(path)}{sm_status}") + except Exception as e: + self.info_lbl.setText(f"Error loading splats: {str(e)}") + self.scene_loaded = False else: try: img = Image.open(path).convert('RGB') @@ -323,14 +470,125 @@ def load_media_file(self, path): self.image_w, self.image_h = img.size self.image_data = img.tobytes() self.image_path = path + self.scene_loaded = True self.info_lbl.setText(f"Loaded Image: {os.path.basename(path)}{sm_status}") for view in self.view_windows: view.set_texture_data(self.image_data, self.image_w, self.image_h) self.position_windows() except Exception as e: self.info_lbl.setText(f"Error loading image: {str(e)}") + self.scene_loaded = False self.camera_updated.emit(self.pitch, self.yaw, self.fov) + + def _apply_point_scene(self, points, colors, sizes, status_text): + self.image_data = None + self.image_w = 0 + self.image_h = 0 + self.scene_loaded = True + self.info_lbl.setText(status_text) + for view in self.view_windows: + view.set_point_data(points, colors, sizes) + self.position_windows() + + def _normalize_points(self, points): + points = points.astype('f4') + center = points.mean(axis=0) + points = points - center + max_extent = np.max(np.linalg.norm(points, axis=1)) + if max_extent > 0: + points /= max_extent + return points + + def _load_las_laz(self, path): + las = laspy.read(path) + points = np.column_stack((las.x, las.y, las.z)).astype('f4') + if points.size == 0: + raise ValueError("File has no points.") + points = self._normalize_points(points) + + colors = np.ones((points.shape[0], 4), dtype='f4') + if all(hasattr(las, attr) for attr in ('red', 'green', 'blue')): + rgb = np.column_stack((las.red, las.green, las.blue)).astype('f4') + max_val = np.max(rgb) + if max_val > 0: + rgb /= max_val + colors[:, :3] = np.clip(rgb, 0.0, 1.0) + + sizes = np.full((points.shape[0],), 2.0, dtype='f4') + self.image_path = path + return points, colors, sizes + + def _load_gaussian_splats(self, path): + if path.lower().endswith('.ply'): + return self._load_gaussian_splat_ply(path) + return self._load_splat_binary(path) + + def _load_gaussian_splat_ply(self, path): + ply = PlyData.read(path) + vertex = ply['vertex'].data + names = vertex.dtype.names or () + required = ('x', 'y', 'z') + if not all(name in names for name in required): + raise ValueError("PLY missing x/y/z vertex fields.") + + points = np.column_stack((vertex['x'], vertex['y'], vertex['z'])).astype('f4') + if points.size == 0: + raise ValueError("PLY has no vertices.") + points = self._normalize_points(points) + + colors = np.ones((points.shape[0], 4), dtype='f4') + if all(name in names for name in ('red', 'green', 'blue')): + rgb = np.column_stack((vertex['red'], vertex['green'], vertex['blue'])).astype('f4') + max_val = np.max(rgb) + if max_val > 0: + rgb /= max_val + colors[:, :3] = np.clip(rgb, 0.0, 1.0) + elif all(name in names for name in ('f_dc_0', 'f_dc_1', 'f_dc_2')): + dc = np.column_stack((vertex['f_dc_0'], vertex['f_dc_1'], vertex['f_dc_2'])).astype('f4') + colors[:, :3] = np.clip(0.5 + dc, 0.0, 1.0) + + if 'opacity' in names: + opacity = vertex['opacity'].astype('f4') + colors[:, 3] = 1.0 / (1.0 + np.exp(-opacity)) + + sizes = np.full((points.shape[0],), 6.0, dtype='f4') + scale_fields = [f for f in ('scale_0', 'scale_1', 'scale_2') if f in names] + if scale_fields: + scales = np.column_stack([vertex[f] for f in scale_fields]).astype('f4') + sizes = np.clip(np.exp(scales.mean(axis=1)) * 3.0, 1.0, 24.0) + + self.image_path = path + return points, colors, sizes + + def _load_splat_binary(self, path): + dtype = np.dtype([ + ('x', ' 5000.0 and self.image_data: + if self.idle_time > 5000.0 and self.scene_loaded: self.yaw += 0.03 self.yaw %= 360.0 self.camera_updated.emit(self.pitch, self.yaw, self.fov) - if self.sm_device and self.image_data: + if self.sm_device and self.scene_loaded: try: state = self.sm_device.read() if state: @@ -387,7 +645,7 @@ def mouseReleaseEvent(self, event): self.dragging = False def mouseMoveEvent(self, event): - if self.dragging and self.image_data: + if self.dragging and self.scene_loaded: self.reset_idle() delta = event.position() - self.last_mouse_pos self.last_mouse_pos = event.position()