From dab2a5ca6fdd9087cc517058efddd652624ce5ec Mon Sep 17 00:00:00 2001 From: OpenNerdz Date: Fri, 31 Jul 2026 07:19:29 -0700 Subject: [PATCH] Fix pupil detection overflow and cross-platform tracker issues Address several correctness and portability bugs that made tracking unstable or prevented the project from running outside Windows. - Cast gray-pixel sums and thresholds to int so uint8 overflow no longer corrupts darkest-area search and binary thresholding - Fix contour length check operator precedence (len(arr) > 5) - Restore stuck-ellipse overlay drawing in the stereo tracker - Open cameras with CAP_ANY on macOS/Linux instead of CAP_MSMF-only - Replace keyboard package hotkeys with OpenCV waitKey (Head/Webcam trackers) - Write screen_position.txt next to the script instead of a hardcoded C: path - Prefer local eye_test.mp4 and local output paths in pupil detectors - Pin MediaPipe to versions that still expose solutions.face_mesh Verified with compile checks and headless smoke tests on eye_test.mp4. --- 3DTracker/Orlosky3DEyeTracker.py | 67 +++++++++++++++---- 3DTracker/Orlosky3DEyeTrackerStereo.py | 33 ++++++--- 3DTracker/readme.md | 10 ++- .../Orlosky3DEyeTrackerFrontCamera.py | 61 +++++++++++++---- HeadTracker/MonitorTracking.py | 12 ++-- HeadTracker/readme.md | 10 +-- HeadTracker/requirements.txt | 7 ++ OrloskyPupilDetector.py | 44 +++++++----- OrloskyPupilDetectorLite.py | 23 +++++-- OrloskyPupilDetectorRaspberryPi.py | 6 +- Webcam3DTracker/MonitorTracking.py | 49 +++++++------- Webcam3DTracker/readme.md | 14 +++- Webcam3DTracker/requirements.txt | 5 +- 13 files changed, 243 insertions(+), 98 deletions(-) create mode 100644 HeadTracker/requirements.txt diff --git a/3DTracker/Orlosky3DEyeTracker.py b/3DTracker/Orlosky3DEyeTracker.py index 89addc4..ecbc3a6 100644 --- a/3DTracker/Orlosky3DEyeTracker.py +++ b/3DTracker/Orlosky3DEyeTracker.py @@ -2,6 +2,7 @@ import random import math import numpy as np +import sys import tkinter as tk from tkinter import ttk, filedialog @@ -32,13 +33,38 @@ ellipse_capture_interval = 5 # save one ellipse every 10 frames max_stuck_ellipses = 20 # safety cap +CAMERA_CAPTURE_MODES = ( + ( + ("MSMF", cv2.CAP_MSMF, None), + ("DirectShow + XVID", cv2.CAP_DSHOW, "XVID"), + ) + if sys.platform == "win32" + else (("Auto", cv2.CAP_ANY, None),) +) + +def open_camera_capture(cam_index, start_mode=0): + for mode_index in range(start_mode, len(CAMERA_CAPTURE_MODES)): + mode_name, backend, fourcc = CAMERA_CAPTURE_MODES[mode_index] + cap = cv2.VideoCapture(cam_index, backend) + if fourcc is not None: + cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*fourcc)) + cap.set(cv2.CAP_PROP_FPS, 30) + ret, frame = cap.read() if cap.isOpened() else (False, None) + + if ret: + return cap, frame, mode_index + + cap.release() + print(f"Could not grab a frame from camera {cam_index} with {mode_name}.") + + return None, None, None + # Function to detect available cameras def detect_cameras(max_cams=10): available_cameras = [] for i in range(max_cams): - cap = cv2.VideoCapture(i, cv2.CAP_MSMF) - cap.set(cv2.CAP_PROP_FPS, 30) - if cap.isOpened(): + cap, _, _ = open_camera_capture(i) + if cap is not None: available_cameras.append(i) cap.release() return available_cameras @@ -64,7 +90,9 @@ def crop_to_aspect_ratio(image, width=640, height=480): # Apply thresholding to an image def apply_binary_threshold(image, darkestPixelValue, addedThreshold): - threshold = darkestPixelValue + addedThreshold + # Cast to int so uint8 pixel values do not wrap when adding the offset. + threshold = int(darkestPixelValue) + int(addedThreshold) + threshold = max(0, min(255, threshold)) _, thresholded_image = cv2.threshold(image, threshold, 255, cv2.THRESH_BINARY_INV) return thresholded_image @@ -89,7 +117,7 @@ def get_darkest_area(image): for dx in range(0, searchArea, internalSkipSize): if x + dx >= gray.shape[1]: break - current_sum += gray[y + dy][x + dx] + current_sum += int(gray[y + dy, x + dx]) num_pixels += 1 if current_sum < min_sum and num_pixels > 0: @@ -402,7 +430,7 @@ def process_frames(thresholded_image_strict, thresholded_image_medium, threshold final_rotated_rect = None - if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0] > 5): + if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0]) > 5: ellipse = cv2.fitEllipse(final_contours[0]) final_rotated_rect = ellipse @@ -475,7 +503,7 @@ def process_frames(thresholded_image_strict, thresholded_image_medium, threshold stuck_ellipses = stuck_ellipses[-max_stuck_ellipses:] # Draw previously saved ellipses so they stay "stuck" on screen - draw_stuck_ellipses(frame) + draw_stuck_ellipses(frame) # Calculate the extended endpoint of gaze line if final_rotated_rect is not None and center_x is not None and center_y is not None: @@ -874,10 +902,12 @@ def process_frame(frame): #find the darkest point darkest_point = get_darkest_area(frame) + if darkest_point is None: + return None # Convert to grayscale to handle pixel value operations gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - darkest_pixel_value = gray_frame[darkest_point[1], darkest_point[0]] + darkest_pixel_value = int(gray_frame[darkest_point[1], darkest_point[0]]) # apply thresholding operations at different levels # at least one should give us a good ellipse segment @@ -906,18 +936,31 @@ def process_camera(): cam_index = int(selected_camera.get()) reset_tracking_state() - cap = cv2.VideoCapture(cam_index, cv2.CAP_MSMF) + cap, _, mode_index = open_camera_capture(cam_index) - if not cap.isOpened(): + if cap is None: print("Error: Could not open camera.") return + print(f"Opened camera {cam_index} with {CAMERA_CAPTURE_MODES[mode_index][0]}.") + while True: ret, frame = cap.read() + if not ret: + # Retry remaining backends if the current capture stream stalls. + retry_mode = mode_index + 1 + cap.release() + cap, frame, mode_index = open_camera_capture(cam_index, retry_mode) + if cap is None: + print("Error: Could not read from camera.") + break + print(f"Switched camera {cam_index} to {CAMERA_CAPTURE_MODES[mode_index][0]}.") + ret = True + if not ret: break - cv2.imshow("ORiginal Frame", frame) + cv2.imshow("Original Frame", frame) frame = cv2.flip(frame, 0) process_frame(frame) @@ -942,7 +985,7 @@ def process_camera(): # Process a selected video file def process_video(): - video_path = filedialog.askopenfilename(filetypes=[("Video Files", "*.mp4;*.avi")]) + video_path = filedialog.askopenfilename(filetypes=[("Video Files", "*.mp4 *.avi"), ("All files", "*.*")]) if not video_path: return # User canceled selection diff --git a/3DTracker/Orlosky3DEyeTrackerStereo.py b/3DTracker/Orlosky3DEyeTrackerStereo.py index fe9ef27..734ab17 100644 --- a/3DTracker/Orlosky3DEyeTrackerStereo.py +++ b/3DTracker/Orlosky3DEyeTrackerStereo.py @@ -162,7 +162,9 @@ def crop_to_aspect_ratio(image, width=640, height=480): # Apply thresholding to an image def apply_binary_threshold(image, darkestPixelValue, addedThreshold): - threshold = darkestPixelValue + addedThreshold + # Cast to int so uint8 pixel values do not wrap when adding the offset. + threshold = int(darkestPixelValue) + int(addedThreshold) + threshold = max(0, min(255, threshold)) _, thresholded_image = cv2.threshold(image, threshold, 255, cv2.THRESH_BINARY_INV) return thresholded_image @@ -187,7 +189,7 @@ def get_darkest_area(image): for dx in range(0, searchArea, internalSkipSize): if x + dx >= gray.shape[1]: break - current_sum += gray[y + dy][x + dx] + current_sum += int(gray[y + dy, x + dx]) num_pixels += 1 if current_sum < min_sum and num_pixels > 0: @@ -556,7 +558,7 @@ def process_frames(thresholded_image_strict, thresholded_image_medium, threshold final_rotated_rect = None - if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0] > 5): + if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0]) > 5: ellipse = cv2.fitEllipse(final_contours[0]) final_rotated_rect = ellipse @@ -630,7 +632,10 @@ def process_frames(thresholded_image_strict, thresholded_image_medium, threshold if len(stuck_ellipses) > max_stuck_ellipses: stuck_ellipses = stuck_ellipses[-max_stuck_ellipses:] - # Calculate the extended endpoint of gaze line + # Draw previously saved ellipses so they stay "stuck" on screen + draw_stuck_ellipses(frame) + + # Calculate the extended endpoint of gaze line if final_rotated_rect is not None and center_x is not None and center_y is not None: # Compute the vector from model_center_average to center_x, center_y dx = center_x - model_center_average[0] @@ -1075,10 +1080,16 @@ def process_frame(frame, return_rendered_frame=False, eye_id=None): #find the darkest point darkest_point = get_darkest_area(frame) + if darkest_point is None: + draw_persistent_overlays(frame) + cv2.imshow(eye_window_name("Tracking"), frame) + if return_rendered_frame: + return None, frame + return None # Convert to grayscale to handle pixel value operations gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - darkest_pixel_value = gray_frame[darkest_point[1], darkest_point[0]] + darkest_pixel_value = int(gray_frame[darkest_point[1], darkest_point[0]]) # apply thresholding operations at different levels # at least one should give us a good ellipse segment @@ -1094,9 +1105,9 @@ def process_frame(frame, return_rendered_frame=False, eye_id=None): #take the three images thresholded at different levels and process them final_rotated_rect = process_frames(thresholded_image_strict, thresholded_image_medium, thresholded_image_relaxed, frame, gray_frame, darkest_point, False, False) - if return_rendered_frame: - return final_rotated_rect, frame - + if return_rendered_frame: + return final_rotated_rect, frame + return final_rotated_rect @@ -1105,14 +1116,16 @@ def process_video(parent=None): while True: video_path = filedialog.askopenfilename( parent=parent, - filetypes=[("Video Files", "*.mp4;*.avi")], + filetypes=[("Video Files", "*.mp4 *.avi"), ("All files", "*.*")], ) if not video_path: cv2.destroyAllWindows() return # User canceled selection - reset_tracking_state() + # Keep per-eye state and stereo gaze output in sync for video mode. + reset_eye_tracking_state(current_eye_id) + load_eye_tracking_state(current_eye_id) cap = cv2.VideoCapture(video_path) if not cap.isOpened(): diff --git a/3DTracker/readme.md b/3DTracker/readme.md index 7258929..e9fef42 100644 --- a/3DTracker/readme.md +++ b/3DTracker/readme.md @@ -12,12 +12,18 @@ Requirements - Python 3 or above - OpenCV (opencv-python) - NumPy +- Tkinter (usually included with Python; on Linux install `python3-tk` if needed) - (Optional) PyOpenGL and gl_sphere.py for 3D visualization To install dependencies via terminal: ```bash -pip install opencv-python numpy tkinter -``` +pip install opencv-python numpy +``` + +Camera backends +- On Windows, the trackers try MSMF then DirectShow. +- On macOS/Linux, they use OpenCV's default (`CAP_ANY`) backend. +- Stereo tracking supports independent left/right camera selection and flip. Stereo Tracking diff --git a/FrontCameraTracker/Orlosky3DEyeTrackerFrontCamera.py b/FrontCameraTracker/Orlosky3DEyeTrackerFrontCamera.py index 5623925..b8f38be 100644 --- a/FrontCameraTracker/Orlosky3DEyeTrackerFrontCamera.py +++ b/FrontCameraTracker/Orlosky3DEyeTrackerFrontCamera.py @@ -51,13 +51,38 @@ circle_y = EXT_CY +CAMERA_CAPTURE_MODES = ( + ( + ("MSMF", cv2.CAP_MSMF, None), + ("DirectShow + XVID", cv2.CAP_DSHOW, "XVID"), + ) + if sys.platform == "win32" + else (("Auto", cv2.CAP_ANY, None),) +) + +def open_camera_capture(cam_index, start_mode=0): + for mode_index in range(start_mode, len(CAMERA_CAPTURE_MODES)): + mode_name, backend, fourcc = CAMERA_CAPTURE_MODES[mode_index] + cap = cv2.VideoCapture(cam_index, backend) + if fourcc is not None: + cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*fourcc)) + cap.set(cv2.CAP_PROP_FPS, 30) + ret, _ = cap.read() if cap.isOpened() else (False, None) + + if ret: + return cap, mode_index + + cap.release() + print(f"Could not grab a frame from camera {cam_index} with {mode_name}.") + + return None, None + # Function to detect available cameras def detect_cameras(max_cams=10): available_cameras = [] for i in range(max_cams): - cap = cv2.VideoCapture(i, cv2.CAP_MSMF) - cap.set(cv2.CAP_PROP_FPS, 30) - if cap.isOpened(): + cap, _ = open_camera_capture(i) + if cap is not None: available_cameras.append(i) cap.release() return available_cameras @@ -83,7 +108,9 @@ def crop_to_aspect_ratio(image, width=640, height=480): # Apply thresholding to an image def apply_binary_threshold(image, darkestPixelValue, addedThreshold): - threshold = darkestPixelValue + addedThreshold + # Cast to int so uint8 pixel values do not wrap when adding the offset. + threshold = int(darkestPixelValue) + int(addedThreshold) + threshold = max(0, min(255, threshold)) _, thresholded_image = cv2.threshold(image, threshold, 255, cv2.THRESH_BINARY_INV) return thresholded_image @@ -108,7 +135,7 @@ def get_darkest_area(image): for dx in range(0, searchArea, internalSkipSize): if x + dx >= gray.shape[1]: break - current_sum += gray[y + dy][x + dx] + current_sum += int(gray[y + dy, x + dx]) num_pixels += 1 if current_sum < min_sum and num_pixels > 0: @@ -364,7 +391,7 @@ def process_frames(thresholded_image_strict, thresholded_image_medium, threshold final_rotated_rect = None - if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0] > 5): + if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0]) > 5: ellipse = cv2.fitEllipse(final_contours[0]) final_rotated_rect = ellipse @@ -951,10 +978,12 @@ def process_frame(frame): #find the darkest point darkest_point = get_darkest_area(frame) + if darkest_point is None: + return None # Convert to grayscale to handle pixel value operations gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - darkest_pixel_value = gray_frame[darkest_point[1], darkest_point[0]] + darkest_pixel_value = int(gray_frame[darkest_point[1], darkest_point[0]]) # apply thresholding operations at different levels # at least one should give us a good ellipse segment @@ -980,19 +1009,23 @@ def process_camera(): cam_index = int(selected_camera.get()) # ---- Eye camera (existing) ---- - eye_cap = cv2.VideoCapture(0, cv2.CAP_MSMF) - if not eye_cap.isOpened(): - print(f"Error: Could not open eye camera at index {cam_index}.") + eye_cap, eye_mode = open_camera_capture(0) + if eye_cap is None: + print("Error: Could not open eye camera at index 0.") return + print(f"Eye camera opened with {CAMERA_CAPTURE_MODES[eye_mode][0]}.") # ---- External camera (new) ---- external_index = cam_index # adjust if needed - external_cap = cv2.VideoCapture(1, cv2.CAP_MSMF) + external_cap, external_mode = open_camera_capture(external_index) - if external_cap.isOpened(): + if external_cap is not None: external_cap.set(cv2.CAP_PROP_FRAME_WIDTH, EXT_WIDTH) external_cap.set(cv2.CAP_PROP_FRAME_HEIGHT, EXT_HEIGHT) - print(f"External camera opened at index {external_index} ({EXT_WIDTH}x{EXT_HEIGHT}).") + print( + f"External camera opened at index {external_index} " + f"({EXT_WIDTH}x{EXT_HEIGHT}) with {CAMERA_CAPTURE_MODES[external_mode][0]}." + ) else: print(f"Warning: Could not open external camera at index {external_index}.") external_cap = None @@ -1055,7 +1088,7 @@ def process_camera(): # Process a selected video file def process_video(): - video_path = filedialog.askopenfilename(filetypes=[("Video Files", "*.mp4;*.avi")]) + video_path = filedialog.askopenfilename(filetypes=[("Video Files", "*.mp4 *.avi"), ("All files", "*.*")]) if not video_path: return # User canceled selection diff --git a/HeadTracker/MonitorTracking.py b/HeadTracker/MonitorTracking.py index b29164f..db15450 100644 --- a/HeadTracker/MonitorTracking.py +++ b/HeadTracker/MonitorTracking.py @@ -7,7 +7,6 @@ import threading import time import subprocess -import keyboard MONITOR_WIDTH, MONITOR_HEIGHT = pyautogui.size() CENTER_X = MONITOR_WIDTH // 2 @@ -257,14 +256,13 @@ def project(pt3d): cv2.imshow("Head-Aligned Cube", frame) cv2.imshow("Facial Landmarks", landmarks_frame) - if keyboard.is_pressed('f7'): + # Use OpenCV key events instead of the `keyboard` package (root/permissions issues on Linux/macOS). + key = cv2.waitKey(1) & 0xFF + if key in (ord('m'), ord('M'), ord('f'), ord('F'), ord('7')): mouse_control_enabled = not mouse_control_enabled print(f"[Mouse Control] {'Enabled' if mouse_control_enabled else 'Disabled'}") - time.sleep(0.3) # debounce to prevent rapid toggling - - - key = cv2.waitKey(1) & 0xFF - if key == ord('q'): + time.sleep(0.2) # debounce to prevent rapid toggling + elif key == ord('q'): break elif key == ord('c'): calibration_offset_yaw = 180 - raw_yaw_deg diff --git a/HeadTracker/readme.md b/HeadTracker/readme.md index 0ac1a11..779b589 100644 --- a/HeadTracker/readme.md +++ b/HeadTracker/readme.md @@ -13,14 +13,14 @@ REQUIREMENTS ------------ - Python 3.x - OpenCV -- MediaPipe +- MediaPipe (`>=0.10.5,<0.10.15` — needed for the `solutions.face_mesh` API used here) - NumPy - PyAutoGUI -- Keyboard +- PyQt5 (optional, for CursorCircle.py) To install the required packages: ```bash - pip install opencv-python mediapipe numpy pyautogui keyboard + pip install -r requirements.txt ``` USAGE @@ -43,10 +43,12 @@ FEATURES - Real-time head pose estimation. - Virtual wireframe cube aligned to your head orientation. - Mouse control mapped to yaw and pitch angle from your face. -- Press F7 to toggle mouse control on/off. +- Press **m** (or **f** / **7**) to toggle mouse control on/off. - Press 'c' to calibrate when looking straight at your monitor center. - Press 'q' to quit the application. +Note: Hotkeys use OpenCV window focus (`cv2.waitKey`), so no root/`keyboard` package is required on Linux or macOS. + NOTES ----- - Calibration centers the screen mapping to your current head pose. diff --git a/HeadTracker/requirements.txt b/HeadTracker/requirements.txt new file mode 100644 index 0000000..8d64b0e --- /dev/null +++ b/HeadTracker/requirements.txt @@ -0,0 +1,7 @@ +opencv-python +numpy +# MediaPipe solutions.face_mesh is used by MonitorTracking.py. +# Pin below the Tasks-API-only transition so installs stay compatible. +mediapipe>=0.10.5,<0.10.15 +pyautogui +PyQt5 diff --git a/OrloskyPupilDetector.py b/OrloskyPupilDetector.py index 66fd489..f96e384 100644 --- a/OrloskyPupilDetector.py +++ b/OrloskyPupilDetector.py @@ -2,10 +2,14 @@ import numpy as np import random import math +import sys import tkinter as tk import os from tkinter import filedialog -import matplotlib.pyplot as plt +try: + import matplotlib.pyplot as plt +except ImportError: + plt = None # Crop the image to maintain a specific aspect ratio (width:height) before resizing. def crop_to_aspect_ratio(image, width=640, height=480): @@ -30,8 +34,9 @@ def crop_to_aspect_ratio(image, width=640, height=480): #apply thresholding to an image def apply_binary_threshold(image, darkestPixelValue, addedThreshold): - # Calculate the threshold as the sum of the two input values - threshold = darkestPixelValue + addedThreshold + # Cast to int so uint8 pixel values do not wrap when adding the offset. + threshold = int(darkestPixelValue) + int(addedThreshold) + threshold = max(0, min(255, threshold)) # Apply the binary threshold _, thresholded_image = cv2.threshold(image, threshold, 255, cv2.THRESH_BINARY_INV) @@ -57,7 +62,7 @@ def get_darkest_area(image): for y in range(ignoreBounds, gray.shape[0] - ignoreBounds, imageSkipSize): for x in range(ignoreBounds, gray.shape[1] - ignoreBounds, imageSkipSize): # Calculate sum of pixel values in the search area, skipping pixels based on internalSkipSize - current_sum = np.int64(0) + current_sum = 0 num_pixels = 0 for dy in range(0, searchArea, internalSkipSize): if y + dy >= gray.shape[0]: @@ -65,7 +70,7 @@ def get_darkest_area(image): for dx in range(0, searchArea, internalSkipSize): if x + dx >= gray.shape[1]: break - current_sum += gray[y + dy][x + dx] + current_sum += int(gray[y + dy, x + dx]) num_pixels += 1 # Update the darkest point if the current block is darker @@ -346,7 +351,7 @@ def process_frames(thresholded_image_strict, thresholded_image_medium, threshold final_contours = [optimize_contours_by_angle(final_contours, gray_frame)] - if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0] > 5): + if final_contours and not isinstance(final_contours[0], list) and len(final_contours[0]) > 5: #cv2.drawContours(test_frame, final_contours, -1, (255, 255, 255), 1) ellipse = cv2.fitEllipse(final_contours[0]) final_rotated_rect = ellipse @@ -406,12 +411,15 @@ def process_frame(frame): def process_video(video_path, input_method): fourcc = cv2.VideoWriter_fourcc(*'mp4v') # Codec for MP4 format - out = cv2.VideoWriter('C:/Storage/Source Videos/output_video.mp4', fourcc, 30.0, (640, 480)) # Output video filename, codec, frame rate, and frame size + output_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "output_video.mp4") + out = cv2.VideoWriter(output_path, fourcc, 30.0, (640, 480)) # Output next to this script if input_method == 1: cap = cv2.VideoCapture(video_path) elif input_method == 2: - cap = cv2.VideoCapture(00, cv2.CAP_DSHOW) # Camera input + # Prefer a cross-platform backend; CAP_DSHOW is Windows-only. + backend = cv2.CAP_DSHOW if sys.platform == "win32" else cv2.CAP_ANY + cap = cv2.VideoCapture(0, backend) # Camera input cap.set(cv2.CAP_PROP_EXPOSURE, -5) else: print("Invalid video source.") @@ -435,6 +443,8 @@ def process_video(video_path, input_method): #find the darkest point darkest_point = get_darkest_area(frame) + if darkest_point is None: + continue if debug_mode_on: darkest_image = frame.copy() @@ -443,7 +453,7 @@ def process_video(video_path, input_method): # Convert to grayscale to handle pixel value operations gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - darkest_pixel_value = gray_frame[darkest_point[1], darkest_point[0]] + darkest_pixel_value = int(gray_frame[darkest_point[1], darkest_point[0]]) # apply thresholding operations at different levels # at least one should give us a good ellipse segment @@ -481,15 +491,19 @@ def process_video(video_path, input_method): out.release() cv2.destroyAllWindows() -#Prompts the user to select a video file if the hardcoded path is not found -#This is just for my debugging convenience :) +#Prompts the user to select a video file if no local sample is found def select_video(): root = tk.Tk() root.withdraw() # Hide the main window - video_path = 'C:/Google Drive/Eye Tracking/fulleyetest.mp4' - if not os.path.exists(video_path): - print("No file found at hardcoded path. Please select a video file.") - video_path = filedialog.askopenfilename(title="Select Video File", filetypes=[("Video Files", "*.mp4;*.avi")]) + # Prefer the included sample clip, then fall back to a file picker. + candidate_paths = [ + os.path.join(os.path.dirname(os.path.abspath(__file__)), "eye_test.mp4"), + "eye_test.mp4", + ] + video_path = next((path for path in candidate_paths if os.path.exists(path)), None) + if video_path is None: + print("No local eye_test.mp4 found. Please select a video file.") + video_path = filedialog.askopenfilename(title="Select Video File", filetypes=[("Video Files", "*.mp4 *.avi"), ("All files", "*.*")]) if not video_path: print("No file selected. Exiting.") return diff --git a/OrloskyPupilDetectorLite.py b/OrloskyPupilDetectorLite.py index 29fc633..15649ef 100644 --- a/OrloskyPupilDetectorLite.py +++ b/OrloskyPupilDetectorLite.py @@ -25,7 +25,9 @@ def crop_to_aspect_ratio(image, width=640, height=480): # Apply thresholding to an image def apply_binary_threshold(image, darkestPixelValue, addedThreshold): - threshold = darkestPixelValue + addedThreshold + # Cast to int so uint8 pixel values do not wrap when adding the offset. + threshold = int(darkestPixelValue) + int(addedThreshold) + threshold = max(0, min(255, threshold)) _, thresholded_image = cv2.threshold(image, threshold, 255, cv2.THRESH_BINARY_INV) return thresholded_image @@ -50,7 +52,7 @@ def get_darkest_area(image): for dx in range(0, searchArea, internalSkipSize): if x + dx >= gray.shape[1]: break - current_sum += gray[y + dy][x + dx] + current_sum += int(gray[y + dy, x + dx]) num_pixels += 1 if current_sum < min_sum and num_pixels > 0: @@ -128,7 +130,12 @@ def process_video(video_path, input_method): fourcc = cv2.VideoWriter_fourcc(*'mp4v') out = cv2.VideoWriter('output_video.mp4', fourcc, 30.0, (640, 480)) - cap = cv2.VideoCapture(video_path) if input_method == 1 else cv2.VideoCapture(0, cv2.CAP_DSHOW) + if input_method == 1: + cap = cv2.VideoCapture(video_path) + else: + import sys + backend = cv2.CAP_DSHOW if sys.platform == "win32" else cv2.CAP_ANY + cap = cv2.VideoCapture(0, backend) if not cap.isOpened(): print("Error: Could not open video.") @@ -155,9 +162,13 @@ def process_video(video_path, input_method): def select_video(): root = tk.Tk() root.withdraw() - video_path = 'C:/Storage/Google Drive/Eye Tracking/fulleyetest3.mp4' - if not os.path.exists(video_path): - video_path = filedialog.askopenfilename(filetypes=[("Video Files", "*.mp4;*.avi")]) + candidate_paths = [ + os.path.join(os.path.dirname(os.path.abspath(__file__)), "eye_test.mp4"), + "eye_test.mp4", + ] + video_path = next((path for path in candidate_paths if os.path.exists(path)), None) + if video_path is None: + video_path = filedialog.askopenfilename(filetypes=[("Video Files", "*.mp4 *.avi"), ("All files", "*.*")]) if video_path: process_video(video_path, 1) diff --git a/OrloskyPupilDetectorRaspberryPi.py b/OrloskyPupilDetectorRaspberryPi.py index 4d2920c..fed6ee2 100644 --- a/OrloskyPupilDetectorRaspberryPi.py +++ b/OrloskyPupilDetectorRaspberryPi.py @@ -23,7 +23,9 @@ def crop_to_aspect_ratio(image, width=640, height=480): # Apply thresholding to an image def apply_binary_threshold(image, darkestPixelValue, addedThreshold): - threshold = darkestPixelValue + addedThreshold + # Cast to int so uint8 pixel values do not wrap when adding the offset. + threshold = int(darkestPixelValue) + int(addedThreshold) + threshold = max(0, min(255, threshold)) _, thresholded_image = cv2.threshold(image, threshold, 255, cv2.THRESH_BINARY_INV) return thresholded_image @@ -48,7 +50,7 @@ def get_darkest_area(image): for dx in range(0, searchArea, internalSkipSize): if x + dx >= gray.shape[1]: break - current_sum += gray[y + dy][x + dx] + current_sum += int(gray[y + dy, x + dx]) num_pixels += 1 if current_sum < min_sum and num_pixels > 0: diff --git a/Webcam3DTracker/MonitorTracking.py b/Webcam3DTracker/MonitorTracking.py index 532260a..7eaaf1b 100644 --- a/Webcam3DTracker/MonitorTracking.py +++ b/Webcam3DTracker/MonitorTracking.py @@ -8,7 +8,6 @@ from collections import deque import pyautogui import threading -import keyboard # Screen and mouse control setup (from old script) MONITOR_WIDTH, MONITOR_HEIGHT = pyautogui.size() @@ -80,13 +79,17 @@ 461, 125, 354, 218, 438, 195, 167, 393, 165, 391, 3, 248] -# ===== NEW: File writing for screen position ===== -screen_position_file = "C:/Storage/Google Drive/Software/EyeTracker3DPython/screen_position.txt" +# ===== File writing for screen position ===== +# Use a project-local path so this works on Windows, macOS, and Linux. +screen_position_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "screen_position.txt") def write_screen_position(x, y): """Write screen position to file, overwriting the same line""" - with open(screen_position_file, 'w') as f: - f.write(f"{x},{y}\n") + try: + with open(screen_position_file, 'w') as f: + f.write(f"{x},{y}\n") + except OSError as e: + print(f"Could not write screen position to {screen_position_file}: {e}") def _rot_x(a): ca, sa = math.cos(a), math.sin(a) @@ -178,8 +181,8 @@ def create_monitor_plane(head_center, R_final, face_landmarks, w, h, -def update_orbit_from_keys(): - """Keyboard orbit controls that PRINT every frame while a key is held.""" +def update_orbit_from_keys(key): + """Orbit controls from OpenCV key codes (cross-platform; no `keyboard` package).""" global orbit_yaw, orbit_pitch, orbit_radius yaw_step = math.radians(1.5) pitch_step = math.radians(1.5) @@ -188,23 +191,23 @@ def update_orbit_from_keys(): changed = False # Rotate - if keyboard.is_pressed('j'): # yaw left + if key == ord('j'): # yaw left orbit_yaw -= yaw_step; changed = True - if keyboard.is_pressed('l'): # yaw right + if key == ord('l'): # yaw right orbit_yaw += yaw_step; changed = True - if keyboard.is_pressed('i'): # pitch up + if key == ord('i'): # pitch up orbit_pitch += pitch_step; changed = True - if keyboard.is_pressed('k'): # pitch down + if key == ord('k'): # pitch down orbit_pitch -= pitch_step; changed = True # Zoom - if keyboard.is_pressed('['): # zoom out + if key == ord('['): # zoom out orbit_radius += zoom_step; changed = True - if keyboard.is_pressed(']'): # zoom in + if key == ord(']'): # zoom in orbit_radius = max(80.0, orbit_radius - zoom_step); changed = True - # Reset (prints every frame while held) - if keyboard.is_pressed('r'): + # Reset + if key == ord('r'): orbit_yaw = 0.0 orbit_pitch = 0.0 orbit_radius = 600.0 @@ -676,7 +679,7 @@ def draw_poly(points, color, thickness): "R = reset view", "X = add marker", "q = quit", - "F7 = toggle mouse control" + "m = toggle mouse control" ] font = cv2.FONT_HERSHEY_SIMPLEX @@ -720,6 +723,8 @@ def mouse_mover(): while cap.isOpened(): ret, frame = cap.read() + # Read keys early so orbit controls and toggles work without the `keyboard` package. + key = cv2.waitKey(1) & 0xFF if not ret: break @@ -858,7 +863,7 @@ def mouse_mover(): cv2.circle(frame, (x, y), 0, (255, 255, 255), -1) # Smooth orbit controls each frame - update_orbit_from_keys() + update_orbit_from_keys(key) # Build 3D landmarks in your existing scale (x*w, y*h, z*w) landmarks3d = None @@ -889,14 +894,12 @@ def mouse_mover(): cv2.imshow("Integrated Eye Tracking", frame) - # Handle keyboard input - if keyboard.is_pressed('f7'): + # Handle keyboard input (OpenCV keys work on Windows/macOS/Linux without root) + if key in (ord('m'), ord('M'), ord('7')): # 'm' preferred; '7' kept for prior Mac PR habit mouse_control_enabled = not mouse_control_enabled print(f"[Mouse Control] {'Enabled' if mouse_control_enabled else 'Disabled'}") - time.sleep(0.3) # debounce to prevent rapid toggling - - key = cv2.waitKey(1) & 0xFF - if key == ord('q'): + time.sleep(0.2) # debounce to prevent rapid toggling + elif key == ord('q'): break elif key == ord('c') and not (left_sphere_locked and right_sphere_locked): current_nose_scale = compute_scale(nose_points_3d) diff --git a/Webcam3DTracker/readme.md b/Webcam3DTracker/readme.md index af829f3..44c755c 100644 --- a/Webcam3DTracker/readme.md +++ b/Webcam3DTracker/readme.md @@ -30,7 +30,7 @@ Windows will open showing: Interactive controls: ----- - c = calibrate (screen center) -- F7 = toggle mouse control (disabled by default) +- m (or 7) = toggle mouse control (disabled by default) - j/l = orbit yaw left/right - i/k = orbit pitch up/down - [ / ] = zoom orbit view out/in @@ -38,6 +38,17 @@ Interactive controls: - x = stamp a green marker on the monitor where your gaze hits - q = quit +Hotkeys use OpenCV window focus (`cv2.waitKey`), so the `keyboard` package is not required +(and no root privileges are needed on Linux/macOS). + +Install +----- +```bash +pip install -r requirements.txt +``` + +`screen_position.txt` is written next to `MonitorTracking.py` (portable across OSes). + Notes ----- Make sure you look at screen center when pressing c. The debug view won't render until you do this. @@ -47,4 +58,5 @@ Troubleshooting ----- - If gaze appears jittery, increase filter_length. - If the wrong camera opens, change cv2.VideoCapture(0) to another index. +- If MediaPipe fails after install, pin an older build: `pip install "mediapipe>=0.10.5,<0.10.15"`. - For better accuracy, use consistent lighting and this webcam: https://amzn.to/43of401. diff --git a/Webcam3DTracker/requirements.txt b/Webcam3DTracker/requirements.txt index 7792b95..c76dc6c 100644 --- a/Webcam3DTracker/requirements.txt +++ b/Webcam3DTracker/requirements.txt @@ -1,6 +1,7 @@ opencv-python numpy -mediapipe +# MediaPipe solutions.face_mesh is used by MonitorTracking.py. +# Pin below the Tasks-API-only transition so installs stay compatible. +mediapipe>=0.10.5,<0.10.15 scipy pyautogui -keyboard