diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index a3f663f21..bf9ebe2b9 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -64,6 +64,51 @@ interface UpdateStatusSummary { detail?: string; } +type RendererPhoneRemoteSessionStatus = + | "waiting" + | "phone-connected" + | "preview-live" + | "mic-active" + | "reconnecting" + | "disconnected" + | "camera-permission-denied" + | "microphone-permission-denied" + | "no-audio-track" + | "phone-backgrounded" + | "phone-sleeping" + | "error"; + +type RendererPhoneRemoteSignalMessage = + | { + type: "offer" | "answer"; + description: RTCSessionDescriptionInit; + } + | { + type: "ice-candidate"; + candidate: RTCIceCandidateInit | null; + }; + +interface RendererPhoneRemoteStatusMessage { + status: RendererPhoneRemoteSessionStatus; + detail?: string; + hasAudio?: boolean; + hasVideo?: boolean; + facingMode?: "user" | "environment"; +} + +interface RendererPhoneRemoteSession { + id: string; + code: string; + joinUrl: string; + localJoinUrl: string; + lanJoinUrl: string; + tunnelJoinUrl?: string; + urlMode: "secure-tunnel" | "lan"; + tunnelError?: string; + expiresAt: number; + status: RendererPhoneRemoteSessionStatus; +} + type RendererExtensionInfo = import("./extensions/extensionTypes").ExtensionInfo; type RendererExtensionReview = import("./extensions/extensionTypes").ExtensionReview; type RendererMarketplaceExtension = import("./extensions/extensionTypes").MarketplaceExtension; @@ -863,6 +908,28 @@ interface Window { cancelCountdown: () => Promise<{ success: boolean }>; getActiveCountdown: () => Promise<{ success: boolean; seconds: number | null }>; onCountdownTick: (callback: (seconds: number) => void) => () => void; + createPhoneRemoteSession: () => Promise<{ + success: boolean; + session: RendererPhoneRemoteSession; + error?: string; + }>; + endPhoneRemoteSession: (sessionId: string) => Promise<{ success: boolean }>; + sendPhoneRemoteSignal: ( + sessionId: string, + message: RendererPhoneRemoteSignalMessage, + ) => Promise<{ success: boolean; index?: number; error?: string }>; + onPhoneRemoteSignal: ( + callback: (payload: { + sessionId: string; + message: RendererPhoneRemoteSignalMessage; + }) => void, + ) => () => void; + onPhoneRemoteStatus: ( + callback: (payload: { + sessionId: string; + status: RendererPhoneRemoteStatusMessage; + }) => void, + ) => () => void; extensionsDiscover: () => Promise; extensionsList: () => Promise; extensionsGet: (id: string) => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index f6e4dc029..e6ba62f55 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -3,6 +3,7 @@ import { registerAssetHandlers } from "./register/assets"; import { registerCaptionHandlers } from "./register/captions"; import { registerExportHandlers } from "./register/export"; import { registerPermissionHandlers } from "./register/permissions"; +import { registerPhoneRemoteHandlers } from "./register/phoneRemote"; import { registerProjectHandlers } from "./register/project"; import { registerRecordingHandlers } from "./register/recording"; import { registerSettingsHandlers } from "./register/settings"; @@ -69,4 +70,5 @@ export function registerIpcHandlers( registerCaptionHandlers(); registerProjectHandlers(); registerSettingsHandlers(); + registerPhoneRemoteHandlers(); } diff --git a/electron/ipc/monitorResolver.ts b/electron/ipc/monitorResolver.ts index e71f36c11..f14bbd38c 100644 --- a/electron/ipc/monitorResolver.ts +++ b/electron/ipc/monitorResolver.ts @@ -13,7 +13,7 @@ export interface WinMonitorHandle { /** * Retrieves raw HMONITOR handles from the Windows OS using a PowerShell bridge. - * This is necessary because Electron's display IDs are often internal hashes that + * This is necessary because Electron's display IDs are often internal hashes that * cannot be used directly with native Windows APIs like Graphics Capture (WGC). */ export function getMonitorHandles(): WinMonitorHandle[] { @@ -53,10 +53,14 @@ public class MonitorHelper { [MonitorHelper]::GetMonitors() `.trim(); - const result = spawnSync("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", psScript], { - encoding: "utf-8", - timeout: 5000, - }); + const result = spawnSync( + "powershell.exe", + ["-NoProfile", "-NonInteractive", "-Command", psScript], + { + encoding: "utf-8", + timeout: 5000, + }, + ); if (result.error || result.status !== 0) { // Silent failure is preferred; the caller will fall back to coordinate-based matching. diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index 9f7ddeb08..22cc968fa 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -407,4 +407,24 @@ describe("getCompanionAudioFallbackPaths", () => { "Recorded output is too small to contain playable video", ); }); + + it("keeps a non-empty recording usable when FFmpeg is missing in a dev install", async () => { + vi.doMock("../ffmpeg/binary", () => ({ + getFfmpegBinaryPath: () => { + throw new Error( + "FFmpeg binary is unavailable. Install ffmpeg-static for this platform or make ffmpeg available on PATH.", + ); + }, + })); + + const videoPath = path.join(tempRoot, "recording-456.mp4"); + await fs.writeFile(videoPath, Buffer.alloc(4096)); + + const { validateRecordedVideo } = await import("./diagnostics"); + + await expect(validateRecordedVideo(videoPath)).resolves.toEqual({ + fileSizeBytes: 4096, + durationSeconds: null, + }); + }); }); diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index 1f0003a9a..1154ddbc4 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -201,9 +201,7 @@ export async function probeMediaDurationSeconds(filePath: string): Promise(await fs.readFile(projectPath, "utf-8")); } catch (error) { - console.warn("[prune] Aborting recording prune because a saved project is unreadable", { - projectPath, - error, - }); + console.warn( + "[prune] Aborting recording prune because a saved project is unreadable", + { + projectPath, + error, + }, + ); throw error; } const candidatePaths = [ diff --git a/electron/ipc/register/phoneRemote.ts b/electron/ipc/register/phoneRemote.ts new file mode 100644 index 000000000..712cf1925 --- /dev/null +++ b/electron/ipc/register/phoneRemote.ts @@ -0,0 +1,150 @@ +import { ipcMain, webContents } from "electron"; +import { getPhoneRemoteJoinUrls } from "../../phoneRemote/server"; +import { + addLaptopSignal, + createPhoneRemoteSession, + endPhoneRemoteSession, + subscribePhoneRemoteStore, +} from "../../phoneRemote/sessionStore"; +import type { PhoneRemoteSignalMessage } from "../../phoneRemote/types"; + +let subscribed = false; + +function sendToOwner(ownerWebContentsId: number, channel: string, payload: unknown) { + const target = webContents.fromId(ownerWebContentsId); + if (!target || target.isDestroyed()) { + return; + } + target.send(channel, payload); +} + +function parseSignalMessage(value: unknown): PhoneRemoteSignalMessage | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + + const record = value as Record; + if (record.type === "offer" || record.type === "answer") { + const description = record.description as Record | null; + if ( + !description || + typeof description.type !== "string" || + typeof description.sdp !== "string" + ) { + return null; + } + if (description.type !== "offer" && description.type !== "answer") { + return null; + } + if (description.type !== record.type) { + return null; + } + return { + type: record.type, + description: { + type: description.type, + sdp: description.sdp, + }, + }; + } + + if (record.type === "ice-candidate") { + if (record.candidate === null) { + return { + type: "ice-candidate", + candidate: null, + }; + } + if ( + !record.candidate || + typeof record.candidate !== "object" || + Array.isArray(record.candidate) + ) { + return null; + } + const candidate = record.candidate as Record; + if (typeof candidate.candidate !== "string") { + return null; + } + return { + type: "ice-candidate", + candidate: { + candidate: candidate.candidate, + sdpMid: typeof candidate.sdpMid === "string" ? candidate.sdpMid : null, + sdpMLineIndex: + typeof candidate.sdpMLineIndex === "number" ? candidate.sdpMLineIndex : null, + usernameFragment: + typeof candidate.usernameFragment === "string" + ? candidate.usernameFragment + : undefined, + }, + }; + } + + return null; +} + +export function registerPhoneRemoteHandlers() { + if (subscribed) { + return; + } + + subscribed = true; + subscribePhoneRemoteStore((event) => { + if (event.type === "signal") { + sendToOwner(event.ownerWebContentsId, "phone-remote-signal", { + sessionId: event.sessionId, + message: event.message, + }); + return; + } + + sendToOwner(event.ownerWebContentsId, "phone-remote-status", { + sessionId: event.sessionId, + status: event.status, + }); + }); + + ipcMain.handle("phone-remote:create-session", async (event) => { + const urls = await getPhoneRemoteJoinUrls(); + const session = createPhoneRemoteSession(event.sender.id, { + ...urls, + joinUrl: `${urls.joinUrl}?code=`, + localJoinUrl: `${urls.localJoinUrl}?code=`, + lanJoinUrl: `${urls.lanJoinUrl}?code=`, + tunnelJoinUrl: urls.tunnelJoinUrl ? `${urls.tunnelJoinUrl}?code=` : undefined, + }); + + return { + success: true, + session: { + ...session, + joinUrl: `${session.joinUrl}${encodeURIComponent(session.code)}`, + localJoinUrl: `${session.localJoinUrl}${encodeURIComponent(session.code)}`, + lanJoinUrl: `${session.lanJoinUrl}${encodeURIComponent(session.code)}`, + tunnelJoinUrl: session.tunnelJoinUrl + ? `${session.tunnelJoinUrl}${encodeURIComponent(session.code)}` + : undefined, + }, + }; + }); + + ipcMain.handle("phone-remote:end-session", (_event, sessionId: string) => { + return { success: endPhoneRemoteSession(sessionId) }; + }); + + ipcMain.handle("phone-remote:send-signal", (event, sessionId: string, value: unknown) => { + const message = parseSignalMessage(value); + if (!message) { + return { + success: false, + error: "Invalid phone remote signal payload.", + }; + } + + const envelope = addLaptopSignal(sessionId, event.sender.id, message); + return envelope + ? { success: true, index: envelope.index } + : { success: false, error: "Phone remote session was not found." }; + }); +} diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index b13453c74..7e04b4ef2 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -13,7 +13,6 @@ import { systemPreferences, } from "electron"; import { showCursor } from "../../cursorHider"; -import { getMonitorHandles } from "../monitorResolver"; import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bounds"; import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; @@ -31,6 +30,7 @@ import { writeCursorTelemetry, } from "../cursor/telemetry"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; +import { getMonitorHandles } from "../monitorResolver"; import { ensureNativeCaptureHelperBinary, ensureSwiftHelperBinary, @@ -433,13 +433,16 @@ export function registerRecordingHandlers( const recordingsDir = await getRecordingsDir(); const timestamp = Date.now(); const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); - tempVideoPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mp4`); - + tempVideoPath = path.join( + app.getPath("temp"), + `recordly-native-${timestamp}.mp4`, + ); + let captureOutput = ""; let systemAudioPath: string | null = null; let microphonePath: string | null = null; let orphanedMicAudioPath: string | null = null; - + const browserMicFallbackRequested = shouldStartWindowsBrowserMicrophoneFallback(options); const captureTarget = resolveWindowsCaptureTarget( @@ -482,7 +485,6 @@ export function registerRecordingHandlers( // Fallback to coordinate-based matching if handle resolution fails config.displayId = captureTarget.displayId; } - config.displayX = Math.round(captureTarget.bounds.x); config.displayY = Math.round(captureTarget.bounds.y); config.displayW = Math.round(captureTarget.bounds.width); @@ -507,7 +509,10 @@ export function registerRecordingHandlers( if (options?.capturesMicrophone && !browserMicFallbackRequested) { microphonePath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`); - tempMicPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mic.wav`); + tempMicPath = path.join( + app.getPath("temp"), + `recordly-native-${timestamp}.mic.wav`, + ); config.captureMic = true; config.micOutputPath = tempMicPath; if (options.microphoneLabel) { @@ -905,333 +910,337 @@ export function registerRecordingHandlers( const start = Date.now(); console.log("[PERF:MAIN] Handler: stop-native-screen-recording: STARTED"); try { - // Windows native capture stop path - if (process.platform === "win32" && windowsNativeCaptureActive) { - let stagedTempVideoPath: string | null = null; - let stagedTempSystemAudioPath: string | null = null; - let stagedTempMicAudioPath: string | null = null; - try { - if (!windowsCaptureProcess) { - throw new Error("Native Windows capture process is not running"); - } - - const proc = windowsCaptureProcess; - const preferredVideoPath = windowsCaptureTargetPath; - const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = windowsSystemAudioPath; - const diagnosticsMicAudioPath = windowsMicAudioPath; - setWindowsCaptureStopRequested(true); - proc.stdin.write("stop\n"); - const tempVideoPath = await waitForWindowsCaptureStop(proc); - stagedTempVideoPath = tempVideoPath; - const finalVideoPath = preferredVideoPath ?? tempVideoPath; + // Windows native capture stop path + if (process.platform === "win32" && windowsNativeCaptureActive) { + let stagedTempVideoPath: string | null = null; + let stagedTempSystemAudioPath: string | null = null; + let stagedTempMicAudioPath: string | null = null; + try { + if (!windowsCaptureProcess) { + throw new Error("Native Windows capture process is not running"); + } - // Native Windows capture results are initially written to a safe temporary path - // (to avoid encoding failures with non-ASCII characters). We move them to the final - // destination now using Node.js, which handles Unicode paths correctly. - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } + const proc = windowsCaptureProcess; + const preferredVideoPath = windowsCaptureTargetPath; + const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + const diagnosticsSystemAudioPath = windowsSystemAudioPath; + const diagnosticsMicAudioPath = windowsMicAudioPath; + setWindowsCaptureStopRequested(true); + proc.stdin.write("stop\n"); + const tempVideoPath = await waitForWindowsCaptureStop(proc); + stagedTempVideoPath = tempVideoPath; + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + + // Native Windows capture results are initially written to a safe temporary path + // (to avoid encoding failures with non-ASCII characters). We move them to the final + // destination now using Node.js, which handles Unicode paths correctly. + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } - if (windowsSystemAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempAudioPath = tempVideoPath.replace(".mp4", ".system.wav"); - stagedTempSystemAudioPath = tempAudioPath; - const finalAudioPath = windowsSystemAudioPath; - if (await pathExists(tempAudioPath)) { - await moveFileWithOverwrite(tempAudioPath, finalAudioPath); - const tempJson = tempAudioPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalAudioPath + ".json"); + if (windowsSystemAudioPath && tempVideoPath.endsWith(".mp4")) { + const tempAudioPath = tempVideoPath.replace(".mp4", ".system.wav"); + stagedTempSystemAudioPath = tempAudioPath; + const finalAudioPath = windowsSystemAudioPath; + if (await pathExists(tempAudioPath)) { + await moveFileWithOverwrite(tempAudioPath, finalAudioPath); + const tempJson = tempAudioPath + ".json"; + if (await pathExists(tempJson)) { + await moveFileWithOverwrite(tempJson, finalAudioPath + ".json"); + } } } - } - if (windowsMicAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempMicPath = tempVideoPath.replace(".mp4", ".mic.wav"); - stagedTempMicAudioPath = tempMicPath; - const finalMicPath = windowsMicAudioPath; - if (await pathExists(tempMicPath)) { - await moveFileWithOverwrite(tempMicPath, finalMicPath); - const tempJson = tempMicPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalMicPath + ".json"); + if (windowsMicAudioPath && tempVideoPath.endsWith(".mp4")) { + const tempMicPath = tempVideoPath.replace(".mp4", ".mic.wav"); + stagedTempMicAudioPath = tempMicPath; + const finalMicPath = windowsMicAudioPath; + if (await pathExists(tempMicPath)) { + await moveFileWithOverwrite(tempMicPath, finalMicPath); + const tempJson = tempMicPath + ".json"; + if (await pathExists(tempJson)) { + await moveFileWithOverwrite(tempJson, finalMicPath + ".json"); + } } } - } - const validation = await validateRecordedVideo(finalVideoPath); + const validation = await validateRecordedVideo(finalVideoPath); - setWindowsCaptureProcess(null); - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); - await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); - setWindowsPendingVideoPath(finalVideoPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - }); - await writeWindowsRecordingDiagnostics(finalVideoPath, { - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - details: { + setWindowsCaptureProcess(null); + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsOrphanedMicAudioPath(null); + await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); + setWindowsPendingVideoPath(finalVideoPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: finalVideoPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - }, - }); + }); + await writeWindowsRecordingDiagnostics(finalVideoPath, { + phase: "stop", + outputPath: finalVideoPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + details: { + fileSizeBytes: validation.fileSizeBytes, + durationSeconds: validation.durationSeconds, + }, + }); - // Persist cursor telemetry before returning so the editor can find it immediately - snapshotCursorTelemetryForPersistence(); - try { - await persistPendingCursorTelemetry(finalVideoPath); - } catch (error) { - console.warn("Failed to persist cursor telemetry during native stop:", error); - } + // Persist cursor telemetry before returning so the editor can find it immediately + snapshotCursorTelemetryForPersistence(); + try { + await persistPendingCursorTelemetry(finalVideoPath); + } catch (error) { + console.warn( + "Failed to persist cursor telemetry during native stop:", + error, + ); + } - return { success: true, path: finalVideoPath }; - } catch (error) { - console.error("Failed to stop native Windows capture:", error); - const fallbackPath = await resolveExistingPath( - windowsCaptureTargetPath, - stagedTempVideoPath, - ); - const recoveredSystemAudioPath = await resolveExistingPath( - windowsSystemAudioPath, - stagedTempSystemAudioPath, - ); - const recoveredMicAudioPath = await resolveExistingPath( - windowsMicAudioPath, - stagedTempMicAudioPath, - ); - const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = recoveredSystemAudioPath ?? windowsSystemAudioPath; - const diagnosticsMicAudioPath = recoveredMicAudioPath ?? windowsMicAudioPath; - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureProcess(null); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); + return { success: true, path: finalVideoPath }; + } catch (error) { + console.error("Failed to stop native Windows capture:", error); + const fallbackPath = await resolveExistingPath( + windowsCaptureTargetPath, + stagedTempVideoPath, + ); + const recoveredSystemAudioPath = await resolveExistingPath( + windowsSystemAudioPath, + stagedTempSystemAudioPath, + ); + const recoveredMicAudioPath = await resolveExistingPath( + windowsMicAudioPath, + stagedTempMicAudioPath, + ); + const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + const diagnosticsSystemAudioPath = + recoveredSystemAudioPath ?? windowsSystemAudioPath; + const diagnosticsMicAudioPath = recoveredMicAudioPath ?? windowsMicAudioPath; + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureProcess(null); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsOrphanedMicAudioPath(null); - if (fallbackPath) { - try { - const validation = await validateRecordedVideo(fallbackPath); - setWindowsPendingVideoPath(fallbackPath); - setWindowsSystemAudioPath(recoveredSystemAudioPath); - setWindowsMicAudioPath(recoveredMicAudioPath); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { + if (fallbackPath) { + try { + const validation = await validateRecordedVideo(fallbackPath); + setWindowsPendingVideoPath(fallbackPath); + setWindowsSystemAudioPath(recoveredSystemAudioPath); + setWindowsMicAudioPath(recoveredMicAudioPath); + await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - recoveredAfterStopFailure: true, - }, - }); - return { success: true, path: fallbackPath }; - } catch { - // File is absent or failed validation. + error: String(error), + }); + await writeWindowsRecordingDiagnostics(fallbackPath, { + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + details: { + fileSizeBytes: validation.fileSizeBytes, + durationSeconds: validation.durationSeconds, + recoveredAfterStopFailure: true, + }, + }); + return { success: true, path: fallbackPath }; + } catch { + // File is absent or failed validation. + } } - } - setWindowsSystemAudioPath(null); - setWindowsMicAudioPath(null); - setWindowsPendingVideoPath(null); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); + setWindowsSystemAudioPath(null); + setWindowsMicAudioPath(null); + setWindowsPendingVideoPath(null); + await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - }, - }); + error: String(error), + }); + await writeWindowsRecordingDiagnostics(fallbackPath, { + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + details: { + fileSizeBytes: await getFileSizeIfPresent(fallbackPath), + }, + }); + return { + success: false, + message: "Failed to stop native Windows capture", + error: String(error), + }; + } + } + + if (process.platform !== "darwin") { return { success: false, - message: "Failed to stop native Windows capture", - error: String(error), + message: "Native screen recording is only available on macOS.", }; } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive) { - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - return { success: false, message: "No native screen recording is active." }; - } + if (!nativeScreenRecordingActive) { + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } - try { - if (!nativeCaptureProcess) { - throw new Error("Native capture helper process is not running"); + return { success: false, message: "No native screen recording is active." }; } - const process = nativeCaptureProcess; - const preferredVideoPath = nativeCaptureTargetPath; - const preferredSystemAudioPath = nativeCaptureSystemAudioPath; - const preferredMicrophonePath = nativeCaptureMicrophonePath; - console.log( - "[stop-native] Audio paths — system:", - preferredSystemAudioPath, - "mic:", - preferredMicrophonePath, - ); - setNativeCaptureStopRequested(true); - process.stdin.write("stop\n"); - const tempVideoPath = await waitForNativeCaptureStop(process); - console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); - setNativeCaptureProcess(null); - setNativeScreenRecordingActive(false); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - - const finalVideoPath = preferredVideoPath ?? tempVideoPath; - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } + try { + if (!nativeCaptureProcess) { + throw new Error("Native capture helper process is not running"); + } - if (preferredSystemAudioPath || preferredMicrophonePath) { + const process = nativeCaptureProcess; + const preferredVideoPath = nativeCaptureTargetPath; + const preferredSystemAudioPath = nativeCaptureSystemAudioPath; + const preferredMicrophonePath = nativeCaptureMicrophonePath; console.log( - "[stop-native] Attempting audio mux (merging separate tracks) into:", - finalVideoPath, + "[stop-native] Audio paths — system:", + preferredSystemAudioPath, + "mic:", + preferredMicrophonePath, ); - try { - await muxNativeMacRecordingWithAudio( + setNativeCaptureStopRequested(true); + process.stdin.write("stop\n"); + const tempVideoPath = await waitForNativeCaptureStop(process); + console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); + setNativeCaptureProcess(null); + setNativeScreenRecordingActive(false); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } + + if (preferredSystemAudioPath || preferredMicrophonePath) { + console.log( + "[stop-native] Attempting audio mux (merging separate tracks) into:", finalVideoPath, - preferredSystemAudioPath, - preferredMicrophonePath, - ); - console.log("[stop-native] Audio mux completed successfully"); - } catch (error) { - console.warn( - "[stop-native] Audio mux failed (video still has inline audio):", - error, ); + try { + await muxNativeMacRecordingWithAudio( + finalVideoPath, + preferredSystemAudioPath, + preferredMicrophonePath, + ); + console.log("[stop-native] Audio mux completed successfully"); + } catch (error) { + console.warn( + "[stop-native] Audio mux failed (video still has inline audio):", + error, + ); + } + } else { + console.log("[stop-native] No separate audio tracks to mux"); } - } else { - console.log("[stop-native] No separate audio tracks to mux"); - } - return await finalizeStoredVideo(finalVideoPath); - } catch (error) { - console.error("Failed to stop native ScreenCaptureKit recording:", error); - const fallbackPath = nativeCaptureTargetPath; - const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; - const fallbackMicrophonePath = nativeCaptureMicrophonePath; - const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); - setNativeScreenRecordingActive(false); - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); + return await finalizeStoredVideo(finalVideoPath); + } catch (error) { + console.error("Failed to stop native ScreenCaptureKit recording:", error); + const fallbackPath = nativeCaptureTargetPath; + const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; + const fallbackMicrophonePath = nativeCaptureMicrophonePath; + const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); + setNativeScreenRecordingActive(false); + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "stop", - sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, - sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", - displayId: lastNativeCaptureDiagnostics?.displayId ?? null, - displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, - windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, - helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, - outputPath: fallbackPath, - systemAudioPath: fallbackSystemAudioPath, - microphonePath: fallbackMicrophonePath, - osRelease: lastNativeCaptureDiagnostics?.osRelease, - supported: lastNativeCaptureDiagnostics?.supported, - helperExists: lastNativeCaptureDiagnostics?.helperExists, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: fallbackFileSizeBytes, - error: String(error), - }); + recordNativeCaptureDiagnostics({ + backend: "mac-screencapturekit", + phase: "stop", + sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, + sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", + displayId: lastNativeCaptureDiagnostics?.displayId ?? null, + displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, + windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, + helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, + outputPath: fallbackPath, + systemAudioPath: fallbackSystemAudioPath, + microphonePath: fallbackMicrophonePath, + osRelease: lastNativeCaptureDiagnostics?.osRelease, + supported: lastNativeCaptureDiagnostics?.supported, + helperExists: lastNativeCaptureDiagnostics?.helperExists, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: fallbackFileSizeBytes, + error: String(error), + }); - // Try to recover: if the target file exists on disk, finalize with it - if (fallbackPath) { - try { - await fs.access(fallbackPath); - console.log( - "[stop-native-screen-recording] Recovering with fallback path:", - fallbackPath, - ); - if (fallbackSystemAudioPath || fallbackMicrophonePath) { - try { - await muxNativeMacRecordingWithAudio( - fallbackPath, - fallbackSystemAudioPath, - fallbackMicrophonePath, - ); - } catch (muxError) { - console.warn( - "Failed to mux recovered native macOS audio into capture:", - muxError, - ); + // Try to recover: if the target file exists on disk, finalize with it + if (fallbackPath) { + try { + await fs.access(fallbackPath); + console.log( + "[stop-native-screen-recording] Recovering with fallback path:", + fallbackPath, + ); + if (fallbackSystemAudioPath || fallbackMicrophonePath) { + try { + await muxNativeMacRecordingWithAudio( + fallbackPath, + fallbackSystemAudioPath, + fallbackMicrophonePath, + ); + } catch (muxError) { + console.warn( + "Failed to mux recovered native macOS audio into capture:", + muxError, + ); + } } + return await finalizeStoredVideo(fallbackPath); + } catch { + // File doesn't exist or isn't accessible } - return await finalizeStoredVideo(fallbackPath); - } catch { - // File doesn't exist or isn't accessible } - } - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } return { success: false, diff --git a/electron/main.ts b/electron/main.ts index ed05ffeb6..3855581b3 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -26,6 +26,7 @@ import { registerIpcHandlers, } from "./ipc/handlers"; import { ensureMediaServer } from "./mediaServer"; +import { cleanupPhoneRemoteServer } from "./phoneRemote/server"; import { ensurePackagedRendererServer } from "./rendererServer"; import type { UpdateToastPayload } from "./updater"; import { @@ -853,6 +854,7 @@ function createSourceSelectorWindowWrapper() { app.on("before-quit", () => { killWindowsCaptureProcess(); showCursor(); + cleanupPhoneRemoteServer(); cleanupNativeVideoExportSessions(); void cleanupAllExportStreams(); }); diff --git a/electron/phoneRemote/mobilePage.ts b/electron/phoneRemote/mobilePage.ts new file mode 100644 index 000000000..424ac2eb5 --- /dev/null +++ b/electron/phoneRemote/mobilePage.ts @@ -0,0 +1,535 @@ +export function getPhoneRemoteMobilePage(): string { + return ` + + + + + Recordly Phone Camera + + + +
+
+
+

Recordly Camera

+

Send this phone camera and mic to your laptop recording.

+
+
Waiting
+
+ +
+ +
Camera preview will appear here after permission is granted.
+
+ +
+ + + +
+ +
+ + +
+
+ + + +`; +} diff --git a/electron/phoneRemote/server.ts b/electron/phoneRemote/server.ts new file mode 100644 index 000000000..cbe4ff075 --- /dev/null +++ b/electron/phoneRemote/server.ts @@ -0,0 +1,477 @@ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import { getPhoneRemoteMobilePage } from "./mobilePage"; +import { + addPhoneSignal, + getLaptopSignalsSince, + getPhoneRemoteSessionByCode, + pruneExpiredPhoneRemoteSessions, + toPublicSession, + updatePhoneRemoteStatus, +} from "./sessionStore"; +import type { + PhoneRemoteJoinUrls, + PhoneRemoteSignalMessage, + PhoneRemoteStatusMessage, +} from "./types"; + +const JSON_LIMIT_BYTES = 256 * 1024; +const TUNNEL_READY_TIMEOUT_MS = 8000; +const TRY_CLOUDFLARE_URL_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com/i; +const PHONE_REMOTE_STATUSES = new Set([ + "waiting", + "phone-connected", + "preview-live", + "mic-active", + "reconnecting", + "disconnected", + "camera-permission-denied", + "microphone-permission-denied", + "no-audio-track", + "phone-backgrounded", + "phone-sleeping", + "error", +]); + +let serverBasePort: number | null = null; +let phoneRemoteServerStartPromise: Promise | null = null; +let tunnelProcess: ChildProcessWithoutNullStreams | null = null; +let tunnelBaseUrl: string | null = null; +let tunnelError: string | null = null; +let tunnelStartPromise: Promise | null = null; +let pruneTimer: NodeJS.Timeout | null = null; + +function getLanHost(): string { + const interfaces = os.networkInterfaces(); + + for (const addresses of Object.values(interfaces)) { + for (const address of addresses ?? []) { + if (address.family === "IPv4" && !address.internal) { + return address.address; + } + } + } + + return "127.0.0.1"; +} + +function writeJson(response: ServerResponse, statusCode: number, payload: unknown) { + response.writeHead(statusCode, { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + }); + response.end(JSON.stringify(payload)); +} + +function writeText(response: ServerResponse, statusCode: number, message: string) { + response.writeHead(statusCode, { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + "Cache-Control": "no-store", + "Content-Type": "text/plain; charset=utf-8", + }); + response.end(message); +} + +function readJsonBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let totalBytes = 0; + const chunks: Buffer[] = []; + + request.on("data", (chunk: Buffer) => { + totalBytes += chunk.byteLength; + if (totalBytes > JSON_LIMIT_BYTES) { + reject(new Error("Request body is too large")); + request.destroy(); + return; + } + chunks.push(chunk); + }); + + request.on("end", () => { + if (chunks.length === 0) { + resolve({}); + return; + } + + try { + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + } catch { + reject(new Error("Request body must be valid JSON")); + } + }); + + request.on("error", reject); + }); +} + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function parsePhoneRemoteSignal(value: unknown): PhoneRemoteSignalMessage | null { + const record = asRecord(value); + if (!record || typeof record.type !== "string") { + return null; + } + + if (record.type === "offer" || record.type === "answer") { + const description = asRecord(record.description); + if ( + !description || + typeof description.type !== "string" || + typeof description.sdp !== "string" + ) { + return null; + } + if (description.type !== "offer" && description.type !== "answer") { + return null; + } + if (description.type !== record.type) { + return null; + } + return { + type: record.type, + description: { + type: description.type, + sdp: description.sdp, + }, + }; + } + + if (record.type === "ice-candidate") { + if (record.candidate === null) { + return { + type: "ice-candidate", + candidate: null, + }; + } + + const candidate = asRecord(record.candidate); + if (!candidate || typeof candidate.candidate !== "string") { + return null; + } + + return { + type: "ice-candidate", + candidate: { + candidate: candidate.candidate, + sdpMid: typeof candidate.sdpMid === "string" ? candidate.sdpMid : null, + sdpMLineIndex: + typeof candidate.sdpMLineIndex === "number" ? candidate.sdpMLineIndex : null, + usernameFragment: + typeof candidate.usernameFragment === "string" + ? candidate.usernameFragment + : undefined, + }, + }; + } + + return null; +} + +function parsePhoneRemoteStatus(value: unknown): PhoneRemoteStatusMessage | null { + const record = asRecord(value); + if (!record || typeof record.status !== "string") { + return null; + } + + if (!PHONE_REMOTE_STATUSES.has(record.status as PhoneRemoteStatusMessage["status"])) { + return null; + } + + return { + status: record.status as PhoneRemoteStatusMessage["status"], + detail: typeof record.detail === "string" ? record.detail : undefined, + hasAudio: typeof record.hasAudio === "boolean" ? record.hasAudio : undefined, + hasVideo: typeof record.hasVideo === "boolean" ? record.hasVideo : undefined, + facingMode: + record.facingMode === "user" || record.facingMode === "environment" + ? record.facingMode + : undefined, + }; +} + +function getCodeFromUrl(url: URL): string | null { + const code = url.searchParams.get("code"); + return code && code.trim().length > 0 ? code.trim().toUpperCase() : null; +} + +async function resolveSessionFromRequest( + request: IncomingMessage, + url: URL, +): Promise | null> { + let code = getCodeFromUrl(url); + + if (!code && request.method === "POST") { + const body = asRecord(await readJsonBody(request)); + code = typeof body?.code === "string" ? body.code.trim().toUpperCase() : null; + (request as IncomingMessage & { parsedJsonBody?: Record }).parsedJsonBody = + body ?? {}; + } + + return code ? getPhoneRemoteSessionByCode(code) : null; +} + +function getParsedJsonBody(request: IncomingMessage): Record { + return ( + (request as IncomingMessage & { parsedJsonBody?: Record }) + .parsedJsonBody ?? {} + ); +} + +async function handlePhoneRemoteRequest(request: IncomingMessage, response: ServerResponse) { + try { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + + if (request.method === "OPTIONS") { + writeText(response, 204, ""); + return; + } + + if (request.method === "GET" && (url.pathname === "/" || url.pathname === "/phone")) { + response.writeHead(url.pathname === "/" ? 302 : 200, { + ...(url.pathname === "/" ? { Location: "/phone" } : {}), + "Cache-Control": "no-store", + "Content-Type": "text/html; charset=utf-8", + }); + response.end(url.pathname === "/" ? "" : getPhoneRemoteMobilePage()); + return; + } + + if (request.method === "GET" && url.pathname === "/api/phone-remote/session") { + const session = getCodeFromUrl(url) + ? getPhoneRemoteSessionByCode(getCodeFromUrl(url) ?? "") + : null; + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + writeJson(response, 200, { + session: toPublicSession(session), + }); + return; + } + + if (request.method === "GET" && url.pathname === "/api/phone-remote/signals") { + const session = await resolveSessionFromRequest(request, url); + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + const after = Number.parseInt(url.searchParams.get("after") ?? "0", 10); + const afterIndex = Number.isFinite(after) ? Math.max(0, after) : 0; + const signals = getLaptopSignalsSince(session, afterIndex); + const lastSignal = signals.length > 0 ? signals[signals.length - 1] : null; + writeJson(response, 200, { + signals, + nextIndex: lastSignal?.index ?? afterIndex, + }); + return; + } + + if (request.method === "POST" && url.pathname === "/api/phone-remote/signal") { + const session = await resolveSessionFromRequest(request, url); + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + const body = getParsedJsonBody(request); + const message = parsePhoneRemoteSignal(body.message); + if (!message) { + writeText(response, 400, "Invalid phone signal payload."); + return; + } + + addPhoneSignal(session, message); + writeJson(response, 200, { success: true }); + return; + } + + if (request.method === "POST" && url.pathname === "/api/phone-remote/status") { + const session = await resolveSessionFromRequest(request, url); + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + const status = parsePhoneRemoteStatus(getParsedJsonBody(request)); + if (!status) { + writeText(response, 400, "Invalid phone status payload."); + return; + } + + updatePhoneRemoteStatus(session, status); + writeJson(response, 200, { success: true }); + return; + } + + writeText(response, 404, "Not Found"); + } catch (error) { + writeText(response, 500, error instanceof Error ? error.message : String(error)); + } +} + +async function ensurePhoneRemoteServerPort(): Promise { + if (serverBasePort) { + return serverBasePort; + } + + if (phoneRemoteServerStartPromise) { + return phoneRemoteServerStartPromise; + } + + phoneRemoteServerStartPromise = new Promise((resolve, reject) => { + const server = createServer((request, response) => { + void handlePhoneRemoteRequest(request, response); + }); + + server.once("error", reject); + server.listen(0, "0.0.0.0", () => { + const address = server.address() as AddressInfo | null; + if (!address) { + reject(new Error("Phone remote server did not expose a TCP address")); + return; + } + + serverBasePort = address.port; + console.log(`[phone-remote] Listening on port ${serverBasePort}`); + if (!pruneTimer) { + pruneTimer = setInterval(() => { + pruneExpiredPhoneRemoteSessions(); + }, 60_000); + pruneTimer.unref?.(); + } + resolve(serverBasePort); + }); + }); + + return phoneRemoteServerStartPromise; +} + +async function startQuickTunnel(localBaseUrl: string): Promise { + if (tunnelBaseUrl || tunnelError) { + return tunnelBaseUrl; + } + + if (tunnelStartPromise) { + return tunnelStartPromise; + } + + tunnelStartPromise = new Promise((resolve) => { + let settled = false; + const settle = (value: string | null) => { + if (settled) { + return; + } + settled = true; + resolve(value); + }; + + try { + tunnelProcess = spawn( + "cloudflared", + ["tunnel", "--url", localBaseUrl, "--no-autoupdate"], + { windowsHide: true }, + ); + } catch (error) { + tunnelError = error instanceof Error ? error.message : String(error); + settle(null); + return; + } + + const spawnedProcess = tunnelProcess; + const timeout = setTimeout(() => { + tunnelError = "Secure tunnel did not become ready in time."; + if (!spawnedProcess.killed) { + spawnedProcess.kill(); + } + if (tunnelProcess === spawnedProcess) { + tunnelProcess = null; + } + settle(null); + }, TUNNEL_READY_TIMEOUT_MS); + + const inspectOutput = (chunk: Buffer) => { + const output = chunk.toString("utf8"); + const match = output.match(TRY_CLOUDFLARE_URL_PATTERN); + if (match) { + tunnelBaseUrl = match[0].replace(/\/$/, ""); + clearTimeout(timeout); + settle(tunnelBaseUrl); + } + }; + + spawnedProcess.stdout.on("data", inspectOutput); + spawnedProcess.stderr.on("data", inspectOutput); + spawnedProcess.once("error", (error) => { + tunnelError = error.message; + clearTimeout(timeout); + settle(null); + }); + spawnedProcess.once("exit", (code) => { + if (!settled) { + tunnelError = + typeof code === "number" + ? `Secure tunnel exited with code ${code}.` + : "Secure tunnel exited before it became ready."; + clearTimeout(timeout); + settle(null); + } + if (tunnelProcess === spawnedProcess) { + tunnelProcess = null; + } + }); + }); + + return tunnelStartPromise; +} + +export async function getPhoneRemoteJoinUrls(): Promise { + const port = await ensurePhoneRemoteServerPort(); + const lanBaseUrl = `http://${getLanHost()}:${port}`; + const localBaseUrl = `http://127.0.0.1:${port}`; + const secureTunnelBaseUrl = await startQuickTunnel(localBaseUrl); + const buildJoinUrl = (baseUrl: string) => `${baseUrl}/phone`; + + if (secureTunnelBaseUrl) { + return { + joinUrl: buildJoinUrl(secureTunnelBaseUrl), + localJoinUrl: buildJoinUrl(localBaseUrl), + lanJoinUrl: buildJoinUrl(lanBaseUrl), + tunnelJoinUrl: buildJoinUrl(secureTunnelBaseUrl), + urlMode: "secure-tunnel", + }; + } + + return { + joinUrl: buildJoinUrl(lanBaseUrl), + localJoinUrl: buildJoinUrl(localBaseUrl), + lanJoinUrl: buildJoinUrl(lanBaseUrl), + urlMode: "lan", + tunnelError: tunnelError ?? "Secure tunnel is unavailable.", + }; +} + +export function cleanupPhoneRemoteServer() { + if (tunnelProcess) { + tunnelProcess.kill(); + tunnelProcess = null; + } + + if (pruneTimer) { + clearInterval(pruneTimer); + pruneTimer = null; + } +} diff --git a/electron/phoneRemote/sessionStore.test.ts b/electron/phoneRemote/sessionStore.test.ts new file mode 100644 index 000000000..2867f1ece --- /dev/null +++ b/electron/phoneRemote/sessionStore.test.ts @@ -0,0 +1,113 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + addLaptopSignal, + addPhoneSignal, + clearPhoneRemoteSessionsForTests, + createPhoneRemoteSession, + getLaptopSignalsSince, + getPhoneRemoteSession, + subscribePhoneRemoteStore, + updatePhoneRemoteStatus, +} from "./sessionStore"; + +const NOW = new Date("2026-01-01T00:00:00.000Z"); +const urls = { + joinUrl: "https://example.test/phone", + localJoinUrl: "http://127.0.0.1:1234/phone", + lanJoinUrl: "http://192.168.1.2:1234/phone", + urlMode: "secure-tunnel" as const, +}; + +afterEach(() => { + vi.useRealTimers(); + clearPhoneRemoteSessionsForTests(); +}); + +describe("phone remote session store", () => { + it("creates an expiring pairing session with a code", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const now = NOW.getTime(); + const session = createPhoneRemoteSession(42, urls, now); + + expect(session.id).toBeTruthy(); + expect(session.code).toMatch(/^[A-Z0-9]+$/); + expect(session.code).toHaveLength(8); + expect(session.status).toBe("waiting"); + expect(session.expiresAt).toBe(now + 600_000); + }); + + it("accepts laptop signals only from the owning webContents", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const session = createPhoneRemoteSession(42, urls, NOW.getTime()); + const offer = { + type: "offer" as const, + description: { type: "offer" as const, sdp: "v=0" }, + }; + + expect(addLaptopSignal(session.id, 7, offer)).toBeNull(); + + const envelope = addLaptopSignal(session.id, 42, offer); + const storedSession = getPhoneRemoteSession(session.id); + + expect(envelope?.index).toBe(1); + expect(storedSession ? getLaptopSignalsSince(storedSession, 0) : []).toHaveLength(1); + }); + + it("emits phone signal and status updates to the session owner", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const listener = vi.fn(); + subscribePhoneRemoteStore(listener); + const publicSession = createPhoneRemoteSession(42, urls, NOW.getTime()); + const session = getPhoneRemoteSession(publicSession.id); + + expect(session).not.toBeNull(); + if (!session) { + return; + } + + addPhoneSignal(session, { + type: "answer", + description: { type: "answer", sdp: "v=0" }, + }); + updatePhoneRemoteStatus(session, { + status: "mic-active", + hasAudio: true, + hasVideo: true, + }); + + expect(listener).toHaveBeenCalledWith( + expect.objectContaining({ + type: "signal", + sessionId: publicSession.id, + ownerWebContentsId: 42, + }), + ); + expect(listener).toHaveBeenCalledWith( + expect.objectContaining({ + type: "status", + sessionId: publicSession.id, + ownerWebContentsId: 42, + status: expect.objectContaining({ status: "mic-active" }), + }), + ); + }); + + it("refreshes the session expiry on phone status heartbeats", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const createdAt = NOW.getTime(); + const session = createPhoneRemoteSession(42, urls, createdAt); + + vi.setSystemTime(createdAt + 300_000); + updatePhoneRemoteStatus(session, { + status: "preview-live", + hasAudio: true, + hasVideo: true, + }); + + expect(session.expiresAt).toBe(createdAt + 900_000); + }); +}); diff --git a/electron/phoneRemote/sessionStore.ts b/electron/phoneRemote/sessionStore.ts new file mode 100644 index 000000000..2049f9e8c --- /dev/null +++ b/electron/phoneRemote/sessionStore.ts @@ -0,0 +1,204 @@ +import crypto from "node:crypto"; +import type { + PhoneRemoteJoinUrls, + PhoneRemotePublicSession, + PhoneRemoteSession, + PhoneRemoteSignalEnvelope, + PhoneRemoteSignalMessage, + PhoneRemoteStatusMessage, + PhoneRemoteStoreEvent, +} from "./types"; + +const SESSION_TTL_MS = 10 * 60 * 1000; +const MAX_SIGNAL_HISTORY = 200; + +const sessions = new Map(); +const listeners = new Set<(event: PhoneRemoteStoreEvent) => void>(); + +function createPairingCode(): string { + let code = ""; + while (code.length < 8) { + code += crypto + .randomBytes(6) + .toString("base64url") + .replace(/[^A-Z0-9]/gi, "") + .toUpperCase(); + } + return code.slice(0, 8); +} + +function appendSignal( + signals: PhoneRemoteSignalEnvelope[], + message: PhoneRemoteSignalMessage, +): PhoneRemoteSignalEnvelope { + const previousIndex = signals.length > 0 ? signals[signals.length - 1].index : 0; + const envelope = { + index: previousIndex + 1, + sentAt: Date.now(), + message, + }; + + signals.push(envelope); + if (signals.length > MAX_SIGNAL_HISTORY) { + signals.splice(0, signals.length - MAX_SIGNAL_HISTORY); + } + + return envelope; +} + +function emit(event: PhoneRemoteStoreEvent) { + for (const listener of listeners) { + listener(event); + } +} + +export function subscribePhoneRemoteStore(listener: (event: PhoneRemoteStoreEvent) => void) { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function pruneExpiredPhoneRemoteSessions(now = Date.now()) { + for (const [sessionId, session] of sessions.entries()) { + if (session.expiresAt <= now) { + sessions.delete(sessionId); + emit({ + type: "status", + sessionId, + ownerWebContentsId: session.ownerWebContentsId, + status: { + status: "disconnected", + detail: "Phone session expired.", + }, + }); + } + } +} + +export function createPhoneRemoteSession( + ownerWebContentsId: number, + urls: PhoneRemoteJoinUrls, + now = Date.now(), +): PhoneRemotePublicSession { + pruneExpiredPhoneRemoteSessions(now); + + const session: PhoneRemoteSession = { + ...urls, + id: crypto.randomUUID(), + code: createPairingCode(), + createdAt: now, + expiresAt: now + SESSION_TTL_MS, + ownerWebContentsId, + status: "waiting", + laptopSignals: [], + phoneSignals: [], + }; + + sessions.set(session.id, session); + return toPublicSession(session); +} + +export function getPhoneRemoteSession(sessionId: string): PhoneRemoteSession | null { + pruneExpiredPhoneRemoteSessions(); + return sessions.get(sessionId) ?? null; +} + +export function getPhoneRemoteSessionByCode(code: string): PhoneRemoteSession | null { + pruneExpiredPhoneRemoteSessions(); + const normalizedCode = code.trim().toUpperCase(); + + for (const session of sessions.values()) { + if (session.code === normalizedCode) { + return session; + } + } + + return null; +} + +export function endPhoneRemoteSession(sessionId: string): boolean { + const session = sessions.get(sessionId); + if (!session) { + return false; + } + + sessions.delete(sessionId); + emit({ + type: "status", + sessionId, + ownerWebContentsId: session.ownerWebContentsId, + status: { + status: "disconnected", + detail: "Phone session ended.", + }, + }); + return true; +} + +export function addLaptopSignal( + sessionId: string, + ownerWebContentsId: number, + message: PhoneRemoteSignalMessage, +): PhoneRemoteSignalEnvelope | null { + const session = getPhoneRemoteSession(sessionId); + if (!session || session.ownerWebContentsId !== ownerWebContentsId) { + return null; + } + + return appendSignal(session.laptopSignals, message); +} + +export function getLaptopSignalsSince( + session: PhoneRemoteSession, + afterIndex: number, +): PhoneRemoteSignalEnvelope[] { + return session.laptopSignals.filter((signal) => signal.index > afterIndex); +} + +export function addPhoneSignal( + session: PhoneRemoteSession, + message: PhoneRemoteSignalMessage, +): PhoneRemoteSignalEnvelope { + const envelope = appendSignal(session.phoneSignals, message); + emit({ + type: "signal", + sessionId: session.id, + ownerWebContentsId: session.ownerWebContentsId, + message, + }); + return envelope; +} + +export function updatePhoneRemoteStatus( + session: PhoneRemoteSession, + status: PhoneRemoteStatusMessage, +) { + session.expiresAt = Date.now() + SESSION_TTL_MS; + session.status = status.status; + session.lastStatusDetail = status.detail; + emit({ + type: "status", + sessionId: session.id, + ownerWebContentsId: session.ownerWebContentsId, + status, + }); +} + +export function toPublicSession(session: PhoneRemoteSession): PhoneRemotePublicSession { + return { + id: session.id, + code: session.code, + expiresAt: session.expiresAt, + status: session.status, + joinUrl: session.joinUrl, + localJoinUrl: session.localJoinUrl, + lanJoinUrl: session.lanJoinUrl, + tunnelJoinUrl: session.tunnelJoinUrl, + urlMode: session.urlMode, + tunnelError: session.tunnelError, + }; +} + +export function clearPhoneRemoteSessionsForTests() { + sessions.clear(); + listeners.clear(); +} diff --git a/electron/phoneRemote/types.ts b/electron/phoneRemote/types.ts new file mode 100644 index 000000000..41abc1491 --- /dev/null +++ b/electron/phoneRemote/types.ts @@ -0,0 +1,75 @@ +export type PhoneRemoteSessionStatus = + | "waiting" + | "phone-connected" + | "preview-live" + | "mic-active" + | "reconnecting" + | "disconnected" + | "camera-permission-denied" + | "microphone-permission-denied" + | "no-audio-track" + | "phone-backgrounded" + | "phone-sleeping" + | "error"; + +export type PhoneRemoteSignalMessage = + | { + type: "offer" | "answer"; + description: RTCSessionDescriptionInit; + } + | { + type: "ice-candidate"; + candidate: RTCIceCandidateInit | null; + }; + +export type PhoneRemoteStatusMessage = { + status: PhoneRemoteSessionStatus; + detail?: string; + hasAudio?: boolean; + hasVideo?: boolean; + facingMode?: "user" | "environment"; +}; + +export type PhoneRemoteSignalEnvelope = { + index: number; + sentAt: number; + message: PhoneRemoteSignalMessage; +}; + +export type PhoneRemoteJoinUrls = { + joinUrl: string; + localJoinUrl: string; + lanJoinUrl: string; + tunnelJoinUrl?: string; + urlMode: "secure-tunnel" | "lan"; + tunnelError?: string; +}; + +export type PhoneRemotePublicSession = PhoneRemoteJoinUrls & { + id: string; + code: string; + expiresAt: number; + status: PhoneRemoteSessionStatus; +}; + +export type PhoneRemoteSession = PhoneRemotePublicSession & { + ownerWebContentsId: number; + createdAt: number; + laptopSignals: PhoneRemoteSignalEnvelope[]; + phoneSignals: PhoneRemoteSignalEnvelope[]; + lastStatusDetail?: string; +}; + +export type PhoneRemoteStoreEvent = + | { + type: "status"; + sessionId: string; + ownerWebContentsId: number; + status: PhoneRemoteStatusMessage; + } + | { + type: "signal"; + sessionId: string; + ownerWebContentsId: number; + message: PhoneRemoteSignalMessage; + }; diff --git a/electron/preload.ts b/electron/preload.ts index 04695d6be..92ea85215 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -938,6 +938,27 @@ contextBridge.exposeInMainWorld("electronAPI", { startCountdown: (seconds: number) => ipcRenderer.invoke("start-countdown", seconds), cancelCountdown: () => ipcRenderer.invoke("cancel-countdown"), getActiveCountdown: () => ipcRenderer.invoke("get-active-countdown"), + createPhoneRemoteSession: () => ipcRenderer.invoke("phone-remote:create-session"), + endPhoneRemoteSession: (sessionId: string) => + ipcRenderer.invoke("phone-remote:end-session", sessionId), + sendPhoneRemoteSignal: (sessionId: string, message: unknown) => + ipcRenderer.invoke("phone-remote:send-signal", sessionId, message), + onPhoneRemoteSignal: (callback: (payload: { sessionId: string; message: unknown }) => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: { sessionId: string; message: unknown }, + ) => callback(payload); + ipcRenderer.on("phone-remote-signal", listener); + return () => ipcRenderer.removeListener("phone-remote-signal", listener); + }, + onPhoneRemoteStatus: (callback: (payload: { sessionId: string; status: unknown }) => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: { sessionId: string; status: unknown }, + ) => callback(payload); + ipcRenderer.on("phone-remote-status", listener); + return () => ipcRenderer.removeListener("phone-remote-status", listener); + }, onCountdownTick: (callback: (seconds: number) => void) => { const listener = (_event: Electron.IpcRendererEvent, seconds: number) => callback(seconds); ipcRenderer.on("countdown-tick", listener); diff --git a/src/components/launch/LaunchWindow.module.css b/src/components/launch/LaunchWindow.module.css index 3638e7563..c5aed7b83 100644 --- a/src/components/launch/LaunchWindow.module.css +++ b/src/components/launch/LaunchWindow.module.css @@ -175,6 +175,115 @@ color: var(--launch-accent); } +.phoneStatusPill { + margin-left: auto; + max-width: 92px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + border-radius: 999px; + background: rgba(255, 255, 255, 0.07); + padding: 2px 7px; + color: #a5a5b3; + font-size: 10px; + font-weight: 700; + text-transform: capitalize; +} + +.phoneRemotePanel { + display: flex; + flex-direction: column; + gap: 8px; + margin: 6px 2px 8px; + border-radius: 12px; + border: 1px solid rgba(61, 139, 255, 0.18); + background: rgba(61, 139, 255, 0.08); + padding: 10px; + color: #d8d8df; + font-size: 12px; +} + +.phoneRemoteHeader { + display: flex; + justify-content: space-between; + gap: 8px; + color: #9ea8bd; + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.phoneRemoteCode { + border-radius: 10px; + background: rgba(255, 255, 255, 0.08); + padding: 10px; + text-align: center; + color: #eeeef2; + font-size: 22px; + font-weight: 800; + letter-spacing: 0.12em; + user-select: text; +} + +.phoneRemoteQr { + display: flex; + justify-content: center; + border-radius: 12px; + background: rgba(255, 255, 255, 0.08); + padding: 10px; +} + +.phoneRemoteQr svg { + display: block; + width: min(100%, 176px); + height: auto; + border-radius: 8px; + box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.12); +} + +.phoneRemoteLink { + overflow-wrap: anywhere; + border-radius: 8px; + background: rgba(0, 0, 0, 0.18); + padding: 7px 8px; + color: #9cbfff; + font-size: 11px; + line-height: 1.35; + user-select: text; +} + +.phoneRemoteButton { + width: 100%; + border: 0; + border-radius: 9px; + background: #3d8bff; + padding: 8px 10px; + color: #fff; + font-size: 12px; + font-weight: 800; + cursor: pointer; +} + +.phoneRemoteButton:hover { + background: #62a4ff; +} + +.phoneRemoteWarning { + border-radius: 9px; + background: rgba(245, 158, 11, 0.12); + padding: 8px 9px; + color: #f8d28a; + font-size: 11px; + line-height: 1.35; +} + +.phoneRemoteDetail { + color: #a5a5b3; + font-size: 11px; + line-height: 1.35; +} + .finalizingState { display: inline-flex; align-items: center; diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index d7dc51bc4..bb854be91 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -12,13 +12,16 @@ import { XIcon, } from "@phosphor-icons/react"; import { AnimatePresence, motion } from "motion/react"; -import { useEffect, useRef } from "react"; +import { useEffect, useMemo, useRef } from "react"; import { RxDragHandleDots2 } from "react-icons/rx"; +import { toast } from "sonner"; import { Separator } from "@/components/ui/separator"; import { useScopedT } from "../../contexts/I18nContext"; import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; +import { usePhoneRemoteMedia } from "../../hooks/usePhoneRemoteMedia"; import { useScreenRecorder } from "../../hooks/useScreenRecorder"; import { useVideoDevices } from "../../hooks/useVideoDevices"; +import { createQrSvgPath } from "../../lib/simpleQr"; import { Button } from "../ui/button"; import { HudInteractionContext } from "./contexts/HudInteractionContext"; import { canToggleFloatingWebcamPreview } from "./floatingWebcamPreview"; @@ -55,6 +58,7 @@ export function LaunchWindow() { function LaunchWindowContent() { const t = useScopedT("launch"); const { openId, requestClose, requestOpen } = useLaunchPopoverCoordinator(); + const phoneRemote = usePhoneRemoteMedia(); const { recording, @@ -73,16 +77,40 @@ function LaunchWindowContent() { setSystemAudioEnabled, webcamEnabled, setWebcamEnabled, + webcamSource, + setWebcamSource, webcamDeviceId, setWebcamDeviceId, + phoneMicrophoneEnabled, + setPhoneMicrophoneEnabled, countdownDelay, setCountdownDelay, preparePermissions, - } = useScreenRecorder(); + } = useScreenRecorder({ + remoteWebcamStream: phoneRemote.videoActive ? phoneRemote.videoCaptureStream : null, + remoteMicrophoneStream: phoneRemote.micActive ? phoneRemote.audioStream : null, + }); const { elapsed, formatTime } = useRecordingTimer(recording, paused); const hudContentRef = useRef(null); const hudBarRef = useRef(null); + const micPopoverOpen = openId === "mic"; + const webcamPopoverOpen = openId === "webcam"; + const isPhoneWebcamSelected = webcamEnabled && webcamSource === "phone"; + const isPhoneMicAvailable = phoneRemote.micActive && phoneMicrophoneEnabled; + const phoneJoinUrl = phoneRemote.session?.joinUrl ?? null; + const phoneJoinQr = useMemo( + () => (phoneJoinUrl ? createQrSvgPath(phoneJoinUrl) : null), + [phoneJoinUrl], + ); + const narratorMicrophoneActive = microphoneEnabled || isPhoneMicAvailable; + const narratorMicrophoneTitle = isPhoneMicAvailable + ? "Phone microphone active" + : phoneMicrophoneEnabled + ? "Phone microphone waiting" + : microphoneEnabled + ? t("recording.disableMicrophone") + : t("recording.enableMicrophone"); const { selectedSource, @@ -97,14 +125,14 @@ function LaunchWindowContent() { const showWebcamControls = webcamEnabled && !recording; const { devices, selectedDeviceId, setSelectedDeviceId } = useMicrophoneDevices( - microphoneEnabled || openId === "mic", + microphoneEnabled || micPopoverOpen, microphoneDeviceId, ); const { devices: videoDevices, selectedDeviceId: selectedVideoDeviceId, setSelectedDeviceId: setSelectedVideoDeviceId, - } = useVideoDevices(webcamEnabled || openId === "webcam"); + } = useVideoDevices(webcamSource === "local" && (webcamEnabled || webcamPopoverOpen)); const { hudOverlayMousePassthroughSupported, @@ -146,9 +174,11 @@ function LaunchWindowContent() { setRecordingWebcamPreviewNode, } = useWebcamPreviewOverlay({ webcamEnabled, + webcamSource, webcamDeviceId, + remotePreviewStream: phoneRemote.previewStream, showWebcamControls, - webcamPopoverOpen: openId === "webcam", + webcamPopoverOpen, hudOverlayMousePassthroughSupported, }); @@ -196,12 +226,118 @@ function LaunchWindowContent() { ease: [0.22, 1, 0.36, 1] as const, }; + const phoneRemoteStatusLabel = (() => { + switch (phoneRemote.status) { + case "idle": + return "Not connected"; + case "waiting": + return "Waiting for phone"; + case "phone-connected": + return "Phone connected"; + case "preview-live": + return "Camera live"; + case "mic-active": + return "Mic active"; + case "reconnecting": + return "Reconnecting"; + case "disconnected": + return "Disconnected"; + case "error": + return "Needs attention"; + } + })(); + + const showRecordingMicrophoneStatus = () => { + if (isPhoneMicAvailable) { + toast.info("Phone microphone is recording."); + return; + } + + if (phoneMicrophoneEnabled) { + toast.info( + "Phone microphone is waiting or disconnected. Screen recording is continuing.", + ); + return; + } + + if (microphoneEnabled) { + toast.info("Laptop microphone is recording."); + return; + } + + toast.info("No narrator microphone is active."); + }; + + const togglePhoneMicrophone = () => { + if (!phoneMicrophoneEnabled && !phoneRemote.session) { + void selectPhoneAsCamera(); + return; + } + + const nextEnabled = !phoneMicrophoneEnabled; + setPhoneMicrophoneEnabled(nextEnabled); + if (nextEnabled) { + setMicrophoneEnabled(false); + } + }; + + const selectPhoneAsCamera = async () => { + if (recording) { + return; + } + + setWebcamSource("phone"); + setWebcamEnabled(true); + setPhoneMicrophoneEnabled(true); + setMicrophoneEnabled(false); + if (!phoneRemote.session) { + await phoneRemote.startSession(); + } + }; + + const selectLaptopWebcam = () => { + if (recording) { + return; + } + + setWebcamSource("local"); + setPhoneMicrophoneEnabled(false); + phoneRemote.stopSession(); + }; + + const handleRecordClick = () => { + if (!hasSelectedSource && platform !== "linux") { + beginInteractiveHudAction(); + requestOpen("sources"); + return; + } + + if ( + webcamSource === "phone" && + webcamEnabled && + !phoneRemote.videoActive && + !phoneRemote.micActive + ) { + const shouldContinue = window.confirm( + "Phone is not connected yet. Start recording the laptop screen without phone camera or phone mic?", + ); + if (!shouldContinue) { + beginInteractiveHudAction(); + requestOpen("webcam"); + return; + } + } + + toggleRecording(); + }; + const recordingControls = ( setMicrophoneEnabled(!microphoneEnabled)} + microphoneTitle={narratorMicrophoneTitle} + onToggleMicrophone={showRecordingMicrophoneStatus} onPauseResume={paused ? resumeRecording : pauseRecording} onStopRecording={toggleRecording} onHideHud={() => window.electronAPI?.hudOverlayHide?.()} @@ -248,6 +384,10 @@ function LaunchWindowContent() { systemAudioEnabled={systemAudioEnabled} onToggleSystemAudio={() => setSystemAudioEnabled(!systemAudioEnabled)} microphoneEnabled={microphoneEnabled} + phoneMicrophoneEnabled={phoneMicrophoneEnabled} + isPhoneMicAvailable={isPhoneMicAvailable} + phoneRemoteStatusLabel={phoneRemoteStatusLabel} + onTogglePhoneMicrophone={togglePhoneMicrophone} onDisableMicrophone={() => setMicrophoneEnabled(false)} devices={devices} microphoneDeviceId={microphoneDeviceId} @@ -262,14 +402,10 @@ function LaunchWindowContent() { variant="ghost" size="icon" iconSize="lg" - title={ - microphoneEnabled - ? t("recording.disableMicrophone") - : t("recording.enableMicrophone") - } - className={microphoneEnabled ? styles.ibActive : ""} + title={narratorMicrophoneTitle} + className={narratorMicrophoneActive ? styles.ibActive : ""} > - {microphoneEnabled ? ( + {narratorMicrophoneActive ? ( ) : ( @@ -281,7 +417,14 @@ function LaunchWindowContent() { setWebcamEnabled(false)} + webcamSource={webcamSource} + onDisableWebcam={() => { + setWebcamEnabled(false); + if (webcamSource === "phone") { + setPhoneMicrophoneEnabled(false); + phoneRemote.stopSession(); + } + }} canToggleFloatingPreview={canToggleFloatingWebcamPreview( hudOverlayMousePassthroughSupported, )} @@ -289,10 +432,22 @@ function LaunchWindowContent() { onToggleFloatingPreview={() => setShowFloatingWebcamPreview((current) => !current)} showWebcamControls={showWebcamControls} setWebcamPreviewNode={setWebcamPreviewNode} + isPhoneWebcamSelected={isPhoneWebcamSelected} + phoneRemoteStatusLabel={phoneRemoteStatusLabel} + phoneRemoteSession={phoneRemote.session} + phoneJoinQr={phoneJoinQr} + phoneRemoteMicActive={phoneRemote.micActive} + phoneRemoteSecureJoinReady={phoneRemote.secureJoinReady} + phoneRemoteError={phoneRemote.error} + phoneRemoteStatusDetail={phoneRemote.statusDetail} + onSelectPhoneAsCamera={() => void selectPhoneAsCamera()} + onSelectLaptopWebcam={selectLaptopWebcam} + onCopyPhoneJoinLink={() => void phoneRemote.copyJoinLink()} videoDevices={videoDevices} webcamDeviceId={webcamDeviceId} selectedVideoDeviceId={selectedVideoDeviceId} onSelectVideoDevice={(deviceId) => { + selectLaptopWebcam(); setWebcamEnabled(true); setSelectedVideoDeviceId(deviceId); setWebcamDeviceId(deviceId); @@ -337,14 +492,7 @@ function LaunchWindowContent() { + {!phoneRemoteSecureJoinReady ? ( +
+ Phone browsers usually require HTTPS for camera and mic access. + A secure tunnel was unavailable, so use the LAN link only if + your browser allows it. +
+ ) : null} + {phoneRemoteError || phoneRemoteStatusDetail ? ( +
+ {phoneRemoteError ?? phoneRemoteStatusDetail} +
+ ) : null} + + ) : ( + + )} + + ) : null} + {webcamSource === "phone" ? ( + } onClick={onSelectLaptopWebcam}> + Use laptop webcam instead + + ) : null} {webcamEnabled && ( <> - } onClick={() => { - onDisableWebcam(); - requestClose(POPOVER_ID); - }}> + } + onClick={() => { + onDisableWebcam(); + requestClose(POPOVER_ID); + }} + > {t("recording.turnOffWebcam")} {canToggleFloatingPreview ? ( : } + icon={ + showFloatingWebcamPreview ? : + } selected={showFloatingWebcamPreview} onClick={onToggleFloatingPreview} > @@ -86,7 +196,9 @@ export function WebcamPopover({ )} {!webcamEnabled && ( -
{t("recording.selectWebcamToEnable")}
+
+ {t("recording.selectWebcamToEnable")} +
)} {showWebcamControls && (
@@ -96,33 +208,41 @@ export function WebcamPopover({ className="h-full w-full object-cover" muted playsInline - style={{ transform: "scaleX(-1)" }} + style={{ + transform: webcamSource === "local" ? "scaleX(-1)" : undefined, + }} />
)} - {videoDevices.map((device) => ( - - ) : ( - - ) - } - selected={ - webcamEnabled && - (webcamDeviceId === device.deviceId || selectedVideoDeviceId === device.deviceId) - } - onClick={() => onSelectVideoDevice(device.deviceId)} - > - {device.label} - - ))} - {videoDevices.length === 0 && ( -
{t("recording.noWebcamsFound")}
+ {webcamSource === "local" + ? videoDevices.map((device) => ( + + ) : ( + + ) + } + selected={ + webcamEnabled && + (webcamDeviceId === device.deviceId || + selectedVideoDeviceId === device.deviceId) + } + onClick={() => onSelectVideoDevice(device.deviceId)} + > + {device.label} + + )) + : null} + {webcamSource === "local" && videoDevices.length === 0 && ( +
+ {t("recording.noWebcamsFound")} +
)} ); diff --git a/src/components/ui/button.tsx b/src/components/ui/button.tsx index 2ea10d35c..4ad994787 100644 --- a/src/components/ui/button.tsx +++ b/src/components/ui/button.tsx @@ -64,4 +64,3 @@ const Button = React.forwardRef( Button.displayName = "Button"; export { Button, buttonVariants }; - diff --git a/src/components/video-editor/editorPreferences.test.ts b/src/components/video-editor/editorPreferences.test.ts index 59566e62d..92c8fa079 100644 --- a/src/components/video-editor/editorPreferences.test.ts +++ b/src/components/video-editor/editorPreferences.test.ts @@ -176,7 +176,6 @@ describe("editorPreferences", () => { backgroundBlur: 3.5, showCursor: false, aspectRatio: "native", - zoomInOverlapMs: 200, exportFormat: "gif", gifFrameRate: 30, gifLoop: false, @@ -214,7 +213,6 @@ describe("editorPreferences", () => { expect(loadEditorPreferences()).toEqual({ ...DEFAULT_EDITOR_PREFERENCES, aspectRatio: "16:9", - zoomInOverlapMs: 200, customAspectWidth: "21", customAspectHeight: "9", }); @@ -289,7 +287,6 @@ describe("editorPreferences", () => { backgroundBlur: 1.5, zoomMotionBlur: 0.75, connectZooms: false, - zoomInOverlapMs: 200, showCursor: false, loopCursor: true, cursorStyle: "figma", @@ -303,6 +300,7 @@ describe("editorPreferences", () => { padding: { top: 30, right: 30, bottom: 30, left: 30, linked: true }, aspectRatio: "4:5", exportEncodingMode: "quality", + exportQuality: "source", exportFormat: "gif", gifFrameRate: 20, gifLoop: false, diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 704371a6b..9ebc54be9 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -429,7 +429,7 @@ export function normalizeProjectEditor(editor: Partial): Pro : DEFAULT_MOTION_PRESET.zoomInDurationMs; const normalizedZoomInOverlapMs = isFiniteNumber(editor.zoomInOverlapMs) ? clamp(editor.zoomInOverlapMs, 0, normalizedZoomInDurationMs) - : DEFAULT_ZOOM_IN_OVERLAP_MS; + : clamp(DEFAULT_ZOOM_IN_OVERLAP_MS, 0, normalizedZoomInDurationMs); const normalizedZoomOutDurationMs = isFiniteNumber(editor.zoomOutDurationMs) ? clamp(editor.zoomOutDurationMs, 60, 4000) : DEFAULT_MOTION_PRESET.zoomOutDurationMs; diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts index 8e1cc6bf5..ab1af945e 100644 --- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts +++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts @@ -6,7 +6,7 @@ import { ZOOM_OUT_EARLY_START_MS, } from "./constants"; import { clampFocusToScale } from "./focusUtils"; -import { clamp01, cubicBezier, easeOutZoom } from "./mathUtils"; +import { clamp01, easeOutZoom } from "./mathUtils"; const CHAINED_ZOOM_PAN_GAP_MS = 1350; const CONNECTED_ZOOM_PAN_DURATION_MS = 1000; @@ -34,14 +34,6 @@ type ConnectedPanTransition = { endScale: number; }; -function lerp(start: number, end: number, amount: number) { - return start + (end - start) * amount; -} - -function easeConnectedPan(value: number) { - return cubicBezier(0.1, 0.0, 0.2, 1.0, value); -} - export function computeRegionStrength( region: ZoomRegion, timeMs: number, @@ -79,13 +71,6 @@ export function computeRegionStrength( return 1 - easeOutZoom(progress); } -function getLinearFocus(start: ZoomFocus, end: ZoomFocus, amount: number): ZoomFocus { - return { - cx: lerp(start.cx, end.cx, amount), - cy: lerp(start.cy, end.cy, amount), - }; -} - function getResolvedFocus(region: ZoomRegion, zoomScale: number): ZoomFocus { return clampFocusToScale(region.focus, zoomScale); } @@ -199,44 +184,6 @@ function getConnectedRegionHold(timeMs: number, connectedPairs: ConnectedRegionP return null; } -function getConnectedRegionTransition(connectedPairs: ConnectedRegionPair[], timeMs: number) { - for (const pair of connectedPairs) { - const { currentRegion, nextRegion, transitionStart, transitionEnd } = pair; - - if (timeMs < transitionStart || timeMs > transitionEnd) { - continue; - } - - const transitionProgress = easeConnectedPan( - clamp01((timeMs - transitionStart) / Math.max(1, transitionEnd - transitionStart)), - ); - const currentScale = ZOOM_DEPTH_SCALES[currentRegion.depth]; - const nextScale = ZOOM_DEPTH_SCALES[nextRegion.depth]; - const transitionScale = lerp(currentScale, nextScale, transitionProgress); - const currentFocus = getResolvedFocus(currentRegion, currentScale); - const nextFocus = getResolvedFocus(nextRegion, nextScale); - const transitionFocus = getLinearFocus(currentFocus, nextFocus, transitionProgress); - - return { - region: { - ...nextRegion, - focus: transitionFocus, - }, - strength: 1, - blendedScale: transitionScale, - transition: { - progress: transitionProgress, - startFocus: currentFocus, - endFocus: nextFocus, - startScale: currentScale, - endScale: nextScale, - }, - }; - } - - return null; -} - export function findDominantRegion( regions: ZoomRegion[], timeMs: number, diff --git a/src/hooks/usePhoneRemoteMedia.ts b/src/hooks/usePhoneRemoteMedia.ts new file mode 100644 index 000000000..c1b1dfe95 --- /dev/null +++ b/src/hooks/usePhoneRemoteMedia.ts @@ -0,0 +1,595 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +const PHONE_VIDEO_WIDTH = 1280; +const PHONE_VIDEO_HEIGHT = 720; +const PHONE_VIDEO_FRAME_RATE = 30; +const REMOTE_STALE_TIMEOUT_MS = 5000; + +type PhoneRemoteConnectionState = + | "idle" + | "waiting" + | "phone-connected" + | "preview-live" + | "mic-active" + | "reconnecting" + | "disconnected" + | "error"; + +type PhoneRemoteSignalMessage = RendererPhoneRemoteSignalMessage; +type PhoneRemoteStatusMessage = RendererPhoneRemoteStatusMessage; + +type UsePhoneRemoteMediaReturn = { + session: RendererPhoneRemoteSession | null; + status: PhoneRemoteConnectionState; + statusDetail: string | null; + error: string | null; + previewStream: MediaStream | null; + videoCaptureStream: MediaStream | null; + audioStream: MediaStream | null; + micActive: boolean; + videoActive: boolean; + secureJoinReady: boolean; + startSession: () => Promise; + stopSession: () => void; + copyJoinLink: () => Promise; +}; + +function isPhoneSignalMessage(value: unknown): value is PhoneRemoteSignalMessage { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + + const record = value as Record; + if (record.type === "offer" || record.type === "answer") { + const description = record.description as Record | null; + return ( + Boolean(description) && + (description?.type === "offer" || description?.type === "answer") && + typeof description?.sdp === "string" + ); + } + + if (record.type === "ice-candidate") { + return record.candidate === null || typeof record.candidate === "object"; + } + + return false; +} + +function normalizeStatus(status: PhoneRemoteStatusMessage["status"]): PhoneRemoteConnectionState { + switch (status) { + case "waiting": + case "phone-connected": + case "preview-live": + case "mic-active": + case "reconnecting": + case "disconnected": + return status; + case "camera-permission-denied": + case "microphone-permission-denied": + case "no-audio-track": + case "phone-backgrounded": + case "phone-sleeping": + return "reconnecting"; + case "error": + return "error"; + } +} + +function drawCoverImage( + context: CanvasRenderingContext2D, + video: HTMLVideoElement, + width: number, + height: number, +) { + const sourceWidth = video.videoWidth; + const sourceHeight = video.videoHeight; + if (sourceWidth <= 0 || sourceHeight <= 0) { + return; + } + + const scale = Math.max(width / sourceWidth, height / sourceHeight); + const drawWidth = sourceWidth * scale; + const drawHeight = sourceHeight * scale; + const drawX = (width - drawWidth) / 2; + const drawY = (height - drawHeight) / 2; + context.drawImage(video, drawX, drawY, drawWidth, drawHeight); +} + +export function usePhoneRemoteMedia(): UsePhoneRemoteMediaReturn { + const [session, setSession] = useState(null); + const [status, setStatus] = useState("idle"); + const [statusDetail, setStatusDetail] = useState(null); + const [error, setError] = useState(null); + const [previewStream, setPreviewStream] = useState(null); + const [videoCaptureStream, setVideoCaptureStream] = useState(null); + const [audioStream, setAudioStream] = useState(null); + const [micActive, setMicActive] = useState(false); + const [videoActive, setVideoActive] = useState(false); + const sessionRef = useRef(null); + const peerRef = useRef(null); + const remoteStreamRef = useRef(null); + const videoCaptureStreamRef = useRef(null); + const canvasRef = useRef(null); + const canvasVideoRef = useRef(null); + const drawFrameRef = useRef(null); + const audioContextRef = useRef(null); + const audioSourceRef = useRef(null); + const audioDestinationRef = useRef(null); + const remoteAudioTrackRef = useRef(null); + const remoteVideoTrackRef = useRef(null); + const pendingRemoteIceCandidatesRef = useRef([]); + const pendingLocalIceSignalsRef = useRef([]); + const offerSentRef = useRef(false); + const remoteStaleTimerRef = useRef(null); + + const setCurrentSession = useCallback((nextSession: RendererPhoneRemoteSession | null) => { + sessionRef.current = nextSession; + setSession(nextSession); + }, []); + + const sendSignal = useCallback(async (message: PhoneRemoteSignalMessage) => { + const currentSession = sessionRef.current; + if (!currentSession) { + return; + } + + const result = await window.electronAPI.sendPhoneRemoteSignal(currentSession.id, message); + if (!result.success) { + throw new Error(result.error ?? "Failed to send phone signal."); + } + }, []); + + const clearRemoteStaleTimer = useCallback(() => { + if (remoteStaleTimerRef.current !== null) { + window.clearTimeout(remoteStaleTimerRef.current); + remoteStaleTimerRef.current = null; + } + }, []); + + const armRemoteStaleTimer = useCallback(() => { + clearRemoteStaleTimer(); + remoteStaleTimerRef.current = window.setTimeout(() => { + setStatus((current) => + current === "preview-live" || current === "mic-active" ? "reconnecting" : current, + ); + setStatusDetail("Phone media has stalled. Screen recording can continue safely."); + }, REMOTE_STALE_TIMEOUT_MS); + }, [clearRemoteStaleTimer]); + + const ensureVideoCaptureStream = useCallback(() => { + if (videoCaptureStreamRef.current) { + return videoCaptureStreamRef.current; + } + + const canvas = document.createElement("canvas"); + canvas.width = PHONE_VIDEO_WIDTH; + canvas.height = PHONE_VIDEO_HEIGHT; + const context = canvas.getContext("2d"); + if (!context) { + throw new Error("Unable to prepare the phone camera canvas."); + } + + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.autoplay = true; + canvasRef.current = canvas; + canvasVideoRef.current = video; + + const draw = () => { + context.fillStyle = "#0d0d12"; + context.fillRect(0, 0, PHONE_VIDEO_WIDTH, PHONE_VIDEO_HEIGHT); + const hasUsableVideo = + video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && + video.videoWidth > 0 && + video.videoHeight > 0; + + if (hasUsableVideo) { + drawCoverImage(context, video, PHONE_VIDEO_WIDTH, PHONE_VIDEO_HEIGHT); + } else { + context.fillStyle = "#777783"; + context.font = "600 34px sans-serif"; + context.textAlign = "center"; + context.fillText( + "Phone camera waiting", + PHONE_VIDEO_WIDTH / 2, + PHONE_VIDEO_HEIGHT / 2, + ); + } + + drawFrameRef.current = window.requestAnimationFrame(draw); + }; + + drawFrameRef.current = window.requestAnimationFrame(draw); + const stream = canvas.captureStream(PHONE_VIDEO_FRAME_RATE); + videoCaptureStreamRef.current = stream; + setVideoCaptureStream(stream); + return stream; + }, []); + + const ensureAudioDestination = useCallback(() => { + if (audioDestinationRef.current) { + return audioDestinationRef.current; + } + + const context = new AudioContext({ sampleRate: 48000 }); + const destination = context.createMediaStreamDestination(); + audioContextRef.current = context; + audioDestinationRef.current = destination; + setAudioStream(destination.stream); + return destination; + }, []); + + const attachRemoteAudioTrack = useCallback( + (track: MediaStreamTrack) => { + remoteAudioTrackRef.current = track; + const destination = ensureAudioDestination(); + audioSourceRef.current?.disconnect(); + const context = audioContextRef.current; + if (!context) { + return; + } + + const source = context.createMediaStreamSource(new MediaStream([track])); + source.connect(destination); + audioSourceRef.current = source; + context.resume().catch(() => undefined); + + const updateAudioState = () => { + const active = track.readyState === "live" && track.enabled; + setMicActive(active); + if (active) { + setStatus("mic-active"); + setStatusDetail("Phone microphone is active."); + } + }; + + track.onmute = updateAudioState; + track.onunmute = updateAudioState; + track.onended = () => { + if (remoteAudioTrackRef.current === track) { + remoteAudioTrackRef.current = null; + } + setMicActive(false); + setStatus("reconnecting"); + setStatusDetail("Phone microphone track ended."); + }; + updateAudioState(); + }, + [ensureAudioDestination], + ); + + const attachRemoteStream = useCallback( + (stream: MediaStream) => { + remoteStreamRef.current = stream; + setPreviewStream(stream); + ensureVideoCaptureStream(); + + const video = canvasVideoRef.current; + if (video && video.srcObject !== stream) { + video.srcObject = stream; + video.play().catch(() => undefined); + } + + const videoTrack = stream.getVideoTracks()[0]; + if (videoTrack) { + remoteVideoTrackRef.current = videoTrack; + setVideoActive(videoTrack.readyState === "live"); + videoTrack.onmute = () => { + setVideoActive(false); + setStatus("reconnecting"); + setStatusDetail("Phone camera track muted."); + armRemoteStaleTimer(); + }; + videoTrack.onunmute = () => { + setVideoActive(true); + setStatus("preview-live"); + setStatusDetail("Phone camera preview is live."); + armRemoteStaleTimer(); + }; + videoTrack.onended = () => { + if (remoteVideoTrackRef.current === videoTrack) { + remoteVideoTrackRef.current = null; + } + setVideoActive(false); + setStatus("reconnecting"); + setStatusDetail("Phone camera track ended."); + armRemoteStaleTimer(); + }; + } + + const audioTrack = stream.getAudioTracks()[0]; + if (audioTrack) { + attachRemoteAudioTrack(audioTrack); + } else { + setMicActive(false); + } + + if (videoTrack || audioTrack) { + setStatus(videoTrack ? "preview-live" : "mic-active"); + setStatusDetail( + videoTrack ? "Phone camera preview is live." : "Phone microphone is active.", + ); + armRemoteStaleTimer(); + } + }, + [armRemoteStaleTimer, attachRemoteAudioTrack, ensureVideoCaptureStream], + ); + + const closePeer = useCallback(() => { + const peer = peerRef.current; + peerRef.current = null; + if (peer) { + peer.onicecandidate = null; + peer.ontrack = null; + peer.onconnectionstatechange = null; + peer.close(); + } + pendingRemoteIceCandidatesRef.current = []; + pendingLocalIceSignalsRef.current = []; + offerSentRef.current = false; + }, []); + + const createPeer = useCallback(() => { + closePeer(); + const peer = new RTCPeerConnection({ + iceServers: [ + { urls: "stun:stun.l.google.com:19302" }, + { urls: "stun:stun1.l.google.com:19302" }, + { urls: "stun:stun2.l.google.com:19302" }, + ], + }); + + peer.addTransceiver("video", { direction: "recvonly" }); + peer.addTransceiver("audio", { direction: "recvonly" }); + peer.onicecandidate = (event) => { + const message: PhoneRemoteSignalMessage = { + type: "ice-candidate", + candidate: event.candidate ? event.candidate.toJSON() : null, + }; + + if (!offerSentRef.current) { + pendingLocalIceSignalsRef.current.push(message); + return; + } + + sendSignal(message).catch((signalError) => { + console.warn("Failed to send phone remote ICE candidate:", signalError); + }); + }; + peer.ontrack = (event) => { + const stream = event.streams[0] ?? remoteStreamRef.current ?? new MediaStream(); + if (!event.streams[0] && event.track) { + stream.addTrack(event.track); + } + attachRemoteStream(stream); + }; + peer.onconnectionstatechange = () => { + switch (peer.connectionState) { + case "connected": + setStatus("phone-connected"); + setStatusDetail("Phone is connected."); + break; + case "disconnected": + case "failed": + setStatus("reconnecting"); + setStatusDetail("Phone connection dropped. Screen recording can continue."); + break; + case "closed": + setStatus("disconnected"); + setStatusDetail("Phone disconnected."); + break; + default: + break; + } + }; + peerRef.current = peer; + return peer; + }, [attachRemoteStream, closePeer, sendSignal]); + + const stopSession = useCallback(() => { + const currentSession = sessionRef.current; + if (currentSession) { + void window.electronAPI.endPhoneRemoteSession(currentSession.id); + } + closePeer(); + clearRemoteStaleTimer(); + audioSourceRef.current?.disconnect(); + audioSourceRef.current = null; + audioContextRef.current?.close().catch(() => undefined); + audioContextRef.current = null; + audioDestinationRef.current = null; + remoteAudioTrackRef.current = null; + remoteVideoTrackRef.current = null; + pendingRemoteIceCandidatesRef.current = []; + pendingLocalIceSignalsRef.current = []; + offerSentRef.current = false; + if (drawFrameRef.current !== null) { + window.cancelAnimationFrame(drawFrameRef.current); + drawFrameRef.current = null; + } + videoCaptureStreamRef.current?.getTracks().forEach((track) => track.stop()); + videoCaptureStreamRef.current = null; + remoteStreamRef.current = null; + canvasVideoRef.current = null; + canvasRef.current = null; + setCurrentSession(null); + setStatus("idle"); + setStatusDetail(null); + setError(null); + setPreviewStream(null); + setVideoCaptureStream(null); + setAudioStream(null); + setMicActive(false); + setVideoActive(false); + }, [clearRemoteStaleTimer, closePeer, setCurrentSession]); + + const startSession = useCallback(async () => { + setStatus("waiting"); + setStatusDetail("Creating phone connection."); + setError(null); + + try { + ensureVideoCaptureStream(); + ensureAudioDestination(); + const result = await window.electronAPI.createPhoneRemoteSession(); + if (!result.success || !result.session) { + throw new Error(result.error ?? "Failed to create phone session."); + } + + setCurrentSession(result.session); + const peer = createPeer(); + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer); + await sendSignal({ type: "offer", description: offer }); + offerSentRef.current = true; + for (const candidateSignal of pendingLocalIceSignalsRef.current.splice(0)) { + await sendSignal(candidateSignal); + } + setStatus("waiting"); + setStatusDetail("Waiting for phone to connect."); + } catch (startError) { + const message = startError instanceof Error ? startError.message : String(startError); + setError(message); + setStatus("error"); + setStatusDetail(message); + } + }, [ + createPeer, + ensureAudioDestination, + ensureVideoCaptureStream, + sendSignal, + setCurrentSession, + ]); + + const copyJoinLink = useCallback(async () => { + const joinUrl = sessionRef.current?.joinUrl; + if (!joinUrl) { + return false; + } + + try { + await navigator.clipboard.writeText(joinUrl); + return true; + } catch { + return false; + } + }, []); + + useEffect(() => { + let active = true; + const removeSignalListener = window.electronAPI.onPhoneRemoteSignal(async (payload) => { + if (!active) { + return; + } + const currentSession = sessionRef.current; + if (!currentSession || payload.sessionId !== currentSession.id) { + return; + } + if (!isPhoneSignalMessage(payload.message)) { + return; + } + + const peer = peerRef.current; + if (!peer) { + return; + } + + try { + if (payload.message.type === "answer") { + if (peer.signalingState !== "have-local-offer") { + return; + } + await peer.setRemoteDescription(payload.message.description); + if (!active) { + return; + } + for (const candidate of pendingRemoteIceCandidatesRef.current.splice(0)) { + try { + await peer.addIceCandidate(candidate); + } catch (candidateError) { + console.warn( + "Failed to apply queued phone ICE candidate:", + candidateError, + ); + } + } + return; + } + if (payload.message.type === "ice-candidate" && payload.message.candidate) { + if (!peer.remoteDescription) { + pendingRemoteIceCandidatesRef.current.push(payload.message.candidate); + return; + } + await peer.addIceCandidate(payload.message.candidate); + } + } catch (signalError) { + if (!active) { + return; + } + console.warn("Failed to apply phone remote signal:", signalError); + setStatus("reconnecting"); + setStatusDetail("Phone signal failed. Waiting for reconnect."); + } + }); + + const removeStatusListener = window.electronAPI.onPhoneRemoteStatus((payload) => { + const currentSession = sessionRef.current; + if (!currentSession || payload.sessionId !== currentSession.id) { + return; + } + + setStatus(normalizeStatus(payload.status.status)); + setStatusDetail(payload.status.detail ?? null); + const actualAudioActive = + remoteAudioTrackRef.current?.readyState === "live" && + remoteAudioTrackRef.current.enabled; + const actualVideoActive = + remoteVideoTrackRef.current?.readyState === "live" && + !remoteVideoTrackRef.current.muted; + + if (typeof payload.status.hasAudio === "boolean") { + setMicActive(payload.status.hasAudio ? Boolean(actualAudioActive) : false); + } + if (typeof payload.status.hasVideo === "boolean") { + setVideoActive(payload.status.hasVideo ? Boolean(actualVideoActive) : false); + } + if ( + payload.status.status === "preview-live" || + payload.status.status === "mic-active" || + payload.status.status === "phone-connected" + ) { + armRemoteStaleTimer(); + } + }); + + return () => { + active = false; + removeSignalListener(); + removeStatusListener(); + }; + }, [armRemoteStaleTimer]); + + useEffect(() => { + return () => { + stopSession(); + }; + }, [stopSession]); + + return { + session, + status, + statusDetail, + error, + previewStream, + videoCaptureStream, + audioStream, + micActive, + videoActive, + secureJoinReady: session?.urlMode === "secure-tunnel", + startSession, + stopSession, + copyJoinLink, + }; +} diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5cd70056..e64eebdb4 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -37,8 +37,10 @@ const WEBCAM_WIDTH = 1280; const WEBCAM_HEIGHT = 720; const WEBCAM_FRAME_RATE = 30; const WEBCAM_SUFFIX = "-webcam"; +const SOURCE_AUDIO_MUX_TOAST_ID = "recording-audio-mux-warning"; const MICROPHONE_FALLBACK_ERROR_TOAST_ID = "recording-microphone-fallback-error"; const MICROPHONE_SIDECAR_ERROR_TOAST_ID = "recording-microphone-sidecar-error"; +const DUAL_MIC_WARNING_TOAST_ID = "recording-dual-mic-warning"; export type BrowserMicrophoneProfile = | "processed" | "no-agc" @@ -126,6 +128,13 @@ type DesktopCaptureMediaDevices = { getDisplayMedia: (constraints: unknown) => Promise; }; +type WebcamSource = "local" | "phone"; + +type UseScreenRecorderOptions = { + remoteWebcamStream?: MediaStream | null; + remoteMicrophoneStream?: MediaStream | null; +}; + type UseScreenRecorderReturn = { recording: boolean; paused: boolean; @@ -145,8 +154,12 @@ type UseScreenRecorderReturn = { setSystemAudioEnabled: (enabled: boolean) => void; webcamEnabled: boolean; setWebcamEnabled: (enabled: boolean) => void; + webcamSource: WebcamSource; + setWebcamSource: (source: WebcamSource) => void; webcamDeviceId: string | undefined; setWebcamDeviceId: (deviceId: string | undefined) => void; + phoneMicrophoneEnabled: boolean; + setPhoneMicrophoneEnabled: (enabled: boolean) => void; countdownDelay: number; setCountdownDelay: (delay: number) => void; }; @@ -319,7 +332,10 @@ async function createAudioInputDeviceSnapshot(): Promise< return audioInputs.length > 0 ? audioInputs : null; } -export function useScreenRecorder(): UseScreenRecorderReturn { +export function useScreenRecorder({ + remoteWebcamStream, + remoteMicrophoneStream, +}: UseScreenRecorderOptions = {}): UseScreenRecorderReturn { const [recording, setRecording] = useState(false); const [paused, setPaused] = useState(false); const [starting, setStarting] = useState(false); @@ -330,7 +346,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const [microphoneDeviceId, setMicrophoneDeviceId] = useState(undefined); const [systemAudioEnabled, setSystemAudioEnabled] = useState(false); const [webcamEnabled, setWebcamEnabled] = useState(false); + const [webcamSource, setWebcamSource] = useState("local"); const [webcamDeviceId, setWebcamDeviceId] = useState(undefined); + const [phoneMicrophoneEnabled, setPhoneMicrophoneEnabled] = useState(false); const [countdownDelay, setCountdownDelayState] = useState(3); const mediaRecorder = useRef(null); const webcamRecorder = useRef(null); @@ -338,6 +356,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const screenStream = useRef(null); const microphoneStream = useRef(null); const webcamStream = useRef(null); + const webcamStreamOwnedByRecorder = useRef(true); const mixingContext = useRef(null); const chunks = useRef([]); const webcamChunks = useRef([]); @@ -570,8 +589,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } if (webcamStream.current) { - webcamStream.current.getTracks().forEach((track) => track.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; + webcamStreamOwnedByRecorder.current = true; } if (mixingContext.current) { @@ -883,6 +905,67 @@ export function useScreenRecorder(): UseScreenRecorderReturn { [resetMicFallbackTimingDiagnostics], ); + const createMixedAudioStream = useCallback( + (inputs: Array<{ track: MediaStreamTrack; gain?: number }>) => { + const liveInputs = inputs.filter(({ track }) => track.readyState === "live"); + if (liveInputs.length === 0) { + return null; + } + + if ( + liveInputs.length === 1 && + (liveInputs[0].gain === undefined || liveInputs[0].gain === 1) + ) { + return new MediaStream([liveInputs[0].track]); + } + + const context = new AudioContext({ sampleRate: 48000 }); + mixingContext.current = context; + const destination = context.createMediaStreamDestination(); + for (const input of liveInputs) { + const source = context.createMediaStreamSource(new MediaStream([input.track])); + if (typeof input.gain === "number" && input.gain !== 1) { + const gain = context.createGain(); + gain.gain.value = input.gain; + source.connect(gain).connect(destination); + } else { + source.connect(destination); + } + } + + return destination.stream; + }, + [], + ); + + const startMicrophoneSidecarRecorder = useCallback( + (audioStreamForSidecar: MediaStream | null, mainStartedAt: number) => { + const audioTrack = audioStreamForSidecar?.getAudioTracks()[0]; + if (!audioTrack) { + return false; + } + + micFallbackChunks.current = []; + const recorder = new MediaRecorder(new MediaStream([audioTrack]), { + mimeType: "audio/webm;codecs=opus", + audioBitsPerSecond: AUDIO_BITRATE_VOICE, + }); + micFallbackRecorderMetadata.current = { + mimeType: recorder.mimeType, + audioBitsPerSecond: AUDIO_BITRATE_VOICE, + timesliceMs: RECORDER_TIMESLICE_MS, + }; + resetMicFallbackTimingDiagnostics(); + micFallbackRecorderStartedAt.current = performance.now(); + recorder.ondataavailable = appendMicFallbackChunk; + micFallbackStartDelayMs.current = Math.max(0, Date.now() - mainStartedAt); + recorder.start(RECORDER_TIMESLICE_MS); + micFallbackRecorder.current = recorder; + return true; + }, + [appendMicFallbackChunk, resetMicFallbackTimingDiagnostics], + ); + const stopWebcamRecorder = useCallback(async () => { const recorder = webcamRecorder.current; const pending = webcamStopPromise.current; @@ -927,6 +1010,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackBlobPromise ?? stopMicFallbackRecorder(); const webcamPath = await stopWebcamRecorder(); await storeMicrophoneSidecar(resolvedMicFallbackBlobPromise, result.path, startDelayMs); + cleanupCapturedMedia(); await finalizeRecordingSession(result.path, webcamPath); if (typeof window.electronAPI?.hudOverlayClose === "function") { @@ -936,6 +1020,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return result.path; }, [ + cleanupCapturedMedia, finalizeRecordingSession, stopMicFallbackRecorder, stopWebcamRecorder, @@ -958,21 +1043,41 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } try { - webcamStream.current = await navigator.mediaDevices.getUserMedia({ - video: webcamDeviceId - ? { - deviceId: { exact: webcamDeviceId }, - width: { ideal: WEBCAM_WIDTH }, - height: { ideal: WEBCAM_HEIGHT }, - frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, - } - : { - width: { ideal: WEBCAM_WIDTH }, - height: { ideal: WEBCAM_HEIGHT }, - frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, - }, - audio: false, - }); + if (webcamSource === "phone") { + const remoteVideoTrack = remoteWebcamStream?.getVideoTracks()[0]; + if (!remoteVideoTrack || remoteVideoTrack.readyState !== "live") { + console.warn( + "Phone webcam stream is not ready; continuing without webcam layer.", + ); + resolvedWebcamPath.current = null; + pendingWebcamPathPromise.current = Promise.resolve(null); + webcamStopPromise.current = Promise.resolve(null); + webcamRecorder.current = null; + webcamStartTime.current = null; + webcamTimeOffsetMs.current = 0; + return; + } + + webcamStream.current = new MediaStream([remoteVideoTrack]); + webcamStreamOwnedByRecorder.current = false; + } else { + webcamStream.current = await navigator.mediaDevices.getUserMedia({ + video: webcamDeviceId + ? { + deviceId: { exact: webcamDeviceId }, + width: { ideal: WEBCAM_WIDTH }, + height: { ideal: WEBCAM_HEIGHT }, + frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, + } + : { + width: { ideal: WEBCAM_WIDTH }, + height: { ideal: WEBCAM_HEIGHT }, + frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, + }, + audio: false, + }); + webcamStreamOwnedByRecorder.current = true; + } const mimeType = selectWebcamMimeType(); webcamChunks.current = []; @@ -1034,9 +1139,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current = null; webcamStartTime.current = null; if (webcamStream.current) { - webcamStream.current.getTracks().forEach((track) => track.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; } + webcamStreamOwnedByRecorder.current = true; } }; } catch (error) { @@ -1051,11 +1159,21 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamStartTime.current = null; webcamTimeOffsetMs.current = 0; if (webcamStream.current) { - webcamStream.current.getTracks().forEach((track) => track.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; } + webcamStreamOwnedByRecorder.current = true; } - }, [getRecordingDurationMs, selectWebcamMimeType, webcamDeviceId, webcamEnabled]); + }, [ + getRecordingDurationMs, + remoteWebcamStream, + selectWebcamMimeType, + webcamDeviceId, + webcamEnabled, + webcamSource, + ]); /** Start the prepared webcam MediaRecorder. Call after main recording begins. */ const beginWebcamCapture = useCallback(() => { @@ -1133,39 +1251,51 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // We pass null for webcamPath initially to avoid blocking on webcam disk writes/muxing. await finalizeRecordingSession(finalPath, null); - // 2. Perform background finalization (webcam, muxing, sidecars) - // We don't await this to keep the UI responsive + // 2. Perform background finalization (webcam, muxing, sidecars). + // We don't await this to keep the UI responsive. void (async () => { try { - // Await the webcam path in the background const webcamPath = await webcamPathPromise; + let finalizedPath = finalPath; console.log( "[useScreenRecorder] Background native processing: webcamPath is", webcamPath, ); - // Store sidecars + if (isNativeWindows) { + const muxResult = + await window.electronAPI.muxNativeWindowsRecording(expectedDurationMs); + if (!muxResult?.success || !muxResult.path) { + void logNativeCaptureDiagnostics("mux-native-windows-recording"); + const warningMessage = + muxResult?.error || + muxResult?.message || + "Failed to finish the native Windows audio mux"; + toast.warning( + `${warningMessage}. Recording was saved, but audio playback or export may be incomplete.`, + { id: SOURCE_AUDIO_MUX_TOAST_ID, duration: 10000 }, + ); + finalizedPath = muxResult?.path ?? finalizedPath; + } else { + finalizedPath = muxResult.path; + } + } + await storeMicrophoneSidecar( micFallbackBlobPromise, - finalPath, + finalizedPath, fallbackStartDelayMs, fallbackTrackSettings, ); - // Perform muxing/renaming if on Windows - if (isNativeWindows) { - await window.electronAPI.muxNativeWindowsRecording(expectedDurationMs); - } - - console.log( - "[useScreenRecorder] Emitting setCurrentRecordingSession with:", - { finalPath, webcamPath }, - ); + console.log("[useScreenRecorder] Emitting setCurrentRecordingSession with:", { + finalPath: finalizedPath, + webcamPath, + }); - // Update the session state to notify the editor that all background assets (webcam, mic, etc.) are now ready. - // This broadcasts a 'recording-session-changed' event that the open editor listens to for re-scanning assets. + // Notify the editor that background assets (webcam, mic, muxed video) are ready. await window.electronAPI.setCurrentRecordingSession({ - videoPath: finalPath, + videoPath: finalizedPath, webcamPath, timeOffsetMs: webcamTimeOffsetMs.current, hideOverlayCursorByDefault: hideEditorOverlayCursorByDefault.current, @@ -1177,12 +1307,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } catch (bgError) { console.error("Error in background finalization:", bgError); } finally { - // After all background tasks are done (webcam, mic sidecars, muxing), - // we can safely close the HUD window to release hardware and resources. + cleanupCapturedMedia(); if (typeof window.electronAPI?.hudOverlayClose === "function") { - console.log( - "[useScreenRecorder] All background tasks finished, closing HUD", - ); + console.log("[useScreenRecorder] All background tasks finished, closing HUD"); window.electronAPI.hudOverlayClose(); } } @@ -1403,6 +1530,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recordingSessionTimestamp.current = Date.now(); resetRecordingClock(recordingSessionTimestamp.current); await prepareWebcamRecorder(); + const phoneMicrophoneTrack = + phoneMicrophoneEnabled && remoteMicrophoneStream + ? remoteMicrophoneStream + .getAudioTracks() + .find((track) => track.readyState === "live") + : undefined; + if (phoneMicrophoneTrack && microphoneEnabled) { + toast.warning( + "Both phone mic and laptop mic are enabled. This can cause echo or doubled voice.", + { id: DUAL_MIC_WARNING_TOAST_ID, duration: 10000 }, + ); + } const useNativeMacScreenCapture = platform === "darwin" && (selectedSource.id?.startsWith("screen:") || @@ -1502,8 +1641,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ? 0 : webcamStartTime.current - mainStartedAt; + const sidecarAudioInputs: Array<{ track: MediaStreamTrack; gain?: number }> = + []; + if (phoneMicrophoneTrack) { + sidecarAudioInputs.push({ + track: phoneMicrophoneTrack, + gain: MIC_GAIN_BOOST, + }); + } + // When native mic capture is unavailable or explicitly bypassed, - // record mic via browser getUserMedia as a sidecar file. + // record mic via browser getUserMedia so it can be mixed into + // the microphone sidecar with phone audio when enabled. if (nativeResult.microphoneFallbackRequired && microphoneEnabled) { void logNativeCaptureDiagnostics("start-browser-microphone-fallback"); console.info("Using browser microphone processing for this recording."); @@ -1513,10 +1662,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { browserMicrophoneProfile.current, ); micFallbackRequestedConstraints.current = microphoneConstraints; - const micStream = + microphoneStream.current = await navigator.mediaDevices.getUserMedia(microphoneConstraints); micFallbackTrackSettings.current = - createMicrophoneTrackSettingsSnapshot(micStream); + createMicrophoneTrackSettingsSnapshot(microphoneStream.current); micFallbackAudioInputDevices.current = await createAudioInputDeviceSnapshot().catch(() => null); console.info( @@ -1527,25 +1676,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { "Browser microphone audio input devices:", micFallbackAudioInputDevices.current, ); - micFallbackChunks.current = []; - const recorder = new MediaRecorder(micStream, { - mimeType: "audio/webm;codecs=opus", - audioBitsPerSecond: AUDIO_BITRATE_VOICE, - }); - micFallbackRecorderMetadata.current = { - mimeType: recorder.mimeType, - audioBitsPerSecond: AUDIO_BITRATE_VOICE, - timesliceMs: RECORDER_TIMESLICE_MS, - }; - resetMicFallbackTimingDiagnostics(); - micFallbackRecorderStartedAt.current = performance.now(); - recorder.ondataavailable = appendMicFallbackChunk; - micFallbackStartDelayMs.current = Math.max( - 0, - Date.now() - mainStartedAt, - ); - recorder.start(RECORDER_TIMESLICE_MS); - micFallbackRecorder.current = recorder; + const localMicTrack = microphoneStream.current.getAudioTracks()[0]; + if (localMicTrack) { + sidecarAudioInputs.push({ + track: localMicTrack, + gain: MIC_GAIN_BOOST, + }); + } } catch (micError) { micFallbackStartDelayMs.current = null; micFallbackTrackSettings.current = null; @@ -1567,6 +1704,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } } + if (sidecarAudioInputs.length > 0) { + const sidecarStream = createMixedAudioStream(sidecarAudioInputs); + if (!startMicrophoneSidecarRecorder(sidecarStream, mainStartedAt)) { + micFallbackStartDelayMs.current = null; + } + } + setRecording(true); try { await window.electronAPI?.setRecordingState(true); @@ -1587,7 +1731,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { hideEditorOverlayCursorByDefault.current = browserCursorPolicy.hideEditorOverlayCursorByDefault; - const wantsAudioCapture = microphoneEnabled || systemAudioEnabled; + const wantsAudioCapture = + microphoneEnabled || systemAudioEnabled || Boolean(phoneMicrophoneTrack); const browserCaptureSource = await resolveBrowserCaptureSource(selectedSource); if ( @@ -1630,13 +1775,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { cursor: browserCursorPolicy.streamCursor, }; - if (wantsAudioCapture) { - let screenMediaStream: MediaStream; - const acquireLinuxPortalStream = (withAudio: boolean) => - mediaDevices.getDisplayMedia({ + const acquireLinuxPortalStream = async (withAudio: boolean): Promise => { + try { + return await mediaDevices.getDisplayMedia({ audio: withAudio, video: { - displaySurface: "monitor", + displaySurface: selectedSource.id?.startsWith("window:") + ? "window" + : "monitor", width: { ideal: TARGET_WIDTH, max: TARGET_WIDTH }, height: { ideal: TARGET_HEIGHT, max: TARGET_HEIGHT }, frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE }, @@ -1645,6 +1791,43 @@ export function useScreenRecorder(): UseScreenRecorderReturn { selfBrowserSurface: "exclude", surfaceSwitching: "exclude", }); + } catch (err) { + console.warn( + "Linux portal failed, falling back to desktop capture(no audio):", + err, + ); + if (withAudio) { + alert( + "System audio is not supported in fallback mode. Recording will continue without audio.", + ); + } + + const sources = await window.electronAPI.getSources({ types: ["screen"] }); + + if (!sources.length) { + throw new Error("No screen sources available"); + } + + const source = sources[0]; + console.log("Using fallback source:", source); + + return await navigator.mediaDevices.getUserMedia({ + audio: false, //intentional + video: { + mandatory: { + chromeMediaSource: "desktop", + chromeMediaSourceId: source.id, + maxWidth: TARGET_WIDTH, + maxHeight: TARGET_HEIGHT, + maxFrameRate: TARGET_FRAME_RATE, + }, + }, + } as unknown as MediaStreamConstraints); + } + }; + + if (wantsAudioCapture) { + let screenMediaStream: MediaStream; if (systemAudioEnabled) { try { @@ -1712,55 +1895,32 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const systemAudioTrack = screenMediaStream.getAudioTracks()[0]; const micAudioTrack = microphoneStream.current?.getAudioTracks()[0]; + const audioInputs: Array<{ track: MediaStreamTrack; gain?: number }> = []; + if (systemAudioTrack) { + audioInputs.push({ track: systemAudioTrack }); + } + if (phoneMicrophoneTrack) { + audioInputs.push({ track: phoneMicrophoneTrack, gain: MIC_GAIN_BOOST }); + } + if (micAudioTrack) { + audioInputs.push({ track: micAudioTrack, gain: MIC_GAIN_BOOST }); + } - if (systemAudioTrack && micAudioTrack) { - const context = new AudioContext({ sampleRate: 48000 }); - mixingContext.current = context; - const systemSource = context.createMediaStreamSource( - new MediaStream([systemAudioTrack]), - ); - const micSource = context.createMediaStreamSource( - new MediaStream([micAudioTrack]), - ); - const micGain = context.createGain(); - micGain.gain.value = MIC_GAIN_BOOST; - const destination = context.createMediaStreamDestination(); - - systemSource.connect(destination); - micSource.connect(micGain).connect(destination); - - const mixedTrack = destination.stream.getAudioTracks()[0]; - if (mixedTrack) { - stream.current.addTrack(mixedTrack); - systemAudioIncluded = true; - } - } else if (systemAudioTrack) { - stream.current.addTrack(systemAudioTrack); - systemAudioIncluded = true; - } else if (micAudioTrack) { - stream.current.addTrack(micAudioTrack); + const mixedAudioStream = createMixedAudioStream(audioInputs); + const mixedAudioTrack = mixedAudioStream?.getAudioTracks()[0]; + if (mixedAudioTrack) { + stream.current.addTrack(mixedAudioTrack); + systemAudioIncluded = Boolean(systemAudioTrack); } } else { const mediaStream = useLinuxPortal - ? await mediaDevices.getDisplayMedia({ - audio: false, - video: { - displaySurface: selectedSource.id?.startsWith("window:") - ? "window" - : "monitor", - width: { ideal: TARGET_WIDTH, max: TARGET_WIDTH }, - height: { ideal: TARGET_HEIGHT, max: TARGET_HEIGHT }, - frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE }, - cursor: browserCursorPolicy.streamCursor, - }, - selfBrowserSurface: "exclude", - surfaceSwitching: "exclude", - }) + ? await acquireLinuxPortalStream(false) : await mediaDevices.getUserMedia({ audio: false, video: browserScreenVideoConstraints, }); + screenStream.current = mediaStream; stream.current = mediaStream; videoTrack = mediaStream.getVideoTracks()[0]; } @@ -2041,8 +2201,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current = null; webcamStartTime.current = null; webcamTimeOffsetMs.current = 0; - webcamStream.current?.getTracks().forEach((t) => t.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current?.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; + webcamStreamOwnedByRecorder.current = true; pendingWebcamPathPromise.current = null; resolvedWebcamPath.current = null; @@ -2120,8 +2283,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setSystemAudioEnabled: persistSystemAudioEnabled, webcamEnabled, setWebcamEnabled, + webcamSource, + setWebcamSource, webcamDeviceId, setWebcamDeviceId, + phoneMicrophoneEnabled, + setPhoneMicrophoneEnabled, countdownDelay, setCountdownDelay, }; diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 66a05e89c..1881fdce4 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Desenfoque de fondo", "zoomMotionBlur": "Desenfoque de movimiento del zoom", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Conectar zooms", "connectZoomsDescription": "Suaviza regiones de zoom consecutivas convirtiéndolas en un movimiento continuo de cámara.", "autoApplyFreshRecordingZooms": "Aplicar automáticamente zooms a nuevas grabaciones", @@ -57,6 +62,20 @@ "zoomOutDescription": "Controla cómo sale la cámara de una región de zoom.", "connectedZoomTitle": "Entre zooms", "connectedZoomDescription": "Ajusta el deslizamiento entre regiones de zoom consecutivas cuando la conexión está activada.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Duración de entrada", "zoomInOverlap": "Solapamiento de entrada", "zoomOutDuration": "Duración de salida", @@ -74,7 +93,36 @@ }, "cursorSize": "Tamaño del cursor", "cursorSmoothing": "Suavizado del cursor", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Desactivado", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "Desenfoque de movimiento del cursor", "cursorClickBounce": "Rebote de clic del cursor", "cursorClickBounceDuration": "Velocidad del rebote", @@ -125,8 +173,6 @@ "generateFull": "Generar subtítulos", "regenerateFull": "Regenerar subtítulos", "clearFull": "Borrar subtítulos", - "editCurrent": "Editar subtítulo actual", - "editSaved": "Subtítulo actualizado", "fontSettings": "Tipografía", "defaultFont": "Predeterminado", "fontFamily": "Fuente", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 200ecaa02..65438e873 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Flou d’arrière-plan", "zoomMotionBlur": "Flou de mouvement du zoom", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Relier les zooms", "connectZoomsDescription": "Lisse les zones de zoom consécutives pour créer un mouvement de caméra continu.", "autoApplyFreshRecordingZooms": "Appliquer automatiquement les zooms des nouveaux enregistrements", @@ -57,6 +62,20 @@ "zoomOutDescription": "Contrôle la manière dont la caméra sort d’une zone de zoom.", "connectedZoomTitle": "Entre les zooms", "connectedZoomDescription": "Ajuste la transition entre des zones de zoom consécutives lorsque la connexion est activée.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Durée du zoom avant", "zoomInOverlap": "Chevauchement du zoom avant", "zoomOutDuration": "Durée du zoom arrière", @@ -74,7 +93,36 @@ }, "cursorSize": "Taille du curseur", "cursorSmoothing": "Lissage du curseur", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Désactivé", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "Flou de mouvement du curseur", "cursorClickBounce": "Rebond au clic du curseur", "cursorClickBounceDuration": "Vitesse du rebond", @@ -125,8 +173,6 @@ "generateFull": "Générer les sous-titres", "regenerateFull": "Régénérer les sous-titres", "clearFull": "Effacer les sous-titres", - "editCurrent": "Modifier le sous-titre actuel", - "editSaved": "Sous-titre mis à jour", "fontSettings": "Paramètres de police", "defaultFont": "Par défaut", "fontFamily": "Police", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index d9b9a1a18..d0317df04 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -97,6 +97,32 @@ "cursorSpringDamping": "Smorzamento molla cursore", "cursorSpringMass": "Massa molla cursore", "off": "Off", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "Motion blur del cursore", "cursorClickBounce": "Rimbalzo al clic del cursore", "cursorClickBounceDuration": "Velocità rimbalzo", diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index c528a98fb..56895eb84 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "배경 블러", "zoomMotionBlur": "확대 모션 블러", + "temporalZoomMotionBlur": "시간 기반 줌 블러", + "temporalZoomMotionBlurDescription": "새로운 줌 블러 처리에 사용할 셔터 범위와 프레임 샘플 수를 조정합니다.", + "zoomMotionBlurSamples": "블러 샘플 수", + "zoomMotionBlurShutter": "셔터", + "auto": "자동", "connectZooms": "확대 구간 연결", "connectZoomsDescription": "연속된 확대 구간을 하나의 부드러운 카메라 이동으로 연결합니다.", "autoApplyFreshRecordingZooms": "새 녹화에 확대 자동 적용", @@ -57,6 +62,20 @@ "zoomOutDescription": "카메라가 확대 구간에서 빠져나오는 방식을 제어합니다.", "connectedZoomTitle": "확대 구간 사이", "connectedZoomDescription": "연결이 활성화된 경우, 연속된 확대 구간 사이의 이동을 조정합니다.", + "motionPresetsTitle": "모션 프리셋", + "motionPresetsZoomHint": "줌 모션 프리셋은 설정에서 사용할 수 있습니다.", + "animationPresets": "애니메이션 프리셋", + "cursorMotionPresets": "커서 모션 프리셋", + "motionPresets": { + "focused": { + "label": "집중형", + "description": "데모, 워크스루, 일반 녹화에 적합한 더 빠르고 또렷한 모션입니다." + }, + "smooth": { + "label": "부드러움", + "description": "프레젠테이션, 키노트 스타일 영상, 세련된 전환에 적합한 더 완만한 모션입니다." + } + }, "zoomInDuration": "확대 시작 시간", "zoomInOverlap": "확대 시작 겹침", "zoomOutDuration": "확대 종료 시간", @@ -74,7 +93,36 @@ }, "cursorSize": "커서 크기", "cursorSmoothing": "커서 보정", + "cursorSpringStiffness": "커서 스프링 강성", + "cursorSpringDamping": "커서 스프링 감쇠", + "cursorSpringMass": "커서 스프링 질량", "off": "끔", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "커서 모션 블러", "cursorClickBounce": "커서 클릭 바운스", "cursorClickBounceDuration": "바운스 속도", @@ -125,8 +173,6 @@ "generateFull": "자막 생성", "regenerateFull": "자막 다시 생성", "clearFull": "자막 지우기", - "editCurrent": "현재 자막 편집", - "editSaved": "자막이 업데이트되었습니다", "fontSettings": "글꼴 설정", "defaultFont": "기본값", "fontFamily": "글꼴", diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index d88e4278c..3a0cf67ed 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Achtergrondvervaging", "zoomMotionBlur": "Zoom-bewegingsonscherpte", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Zooms verbinden", "connectZoomsDescription": "Maak opeenvolgende zoomgebieden vloeiend tot een doorlopende camerabeweging.", "autoApplyFreshRecordingZooms": "Zooms voor nieuwe opnames automatisch toepassen", @@ -57,6 +62,20 @@ "zoomOutDescription": "Bepaal hoe de camera een zoomgebied verlaat.", "connectedZoomTitle": "Tussen zooms", "connectedZoomDescription": "Stel de overgang af tussen opeenvolgende zoomgebieden wanneer verbinding is ingeschakeld.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Inzoomduur", "zoomInOverlap": "Inzoomoverlap", "zoomOutDuration": "Uitzoomduur", @@ -74,7 +93,36 @@ }, "cursorSize": "Cursorgrootte", "cursorSmoothing": "Cursorverzachting", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Uit", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "Cursor-bewegingsonscherpte", "cursorClickBounce": "Cursorklikstuit", "cursorClickBounceDuration": "Stuitsnelheid", @@ -125,8 +173,6 @@ "generateFull": "Ondertiteling genereren", "regenerateFull": "Ondertiteling opnieuw genereren", "clearFull": "Ondertiteling wissen", - "editCurrent": "Huidige ondertiteling bewerken", - "editSaved": "Ondertiteling bijgewerkt", "fontSettings": "Lettertype-instellingen", "defaultFont": "Standaard", "fontFamily": "Lettertype", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index a34e6e4d2..4f0358d18 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Desfoque de fundo", "zoomMotionBlur": "Desfoque de movimento do zoom", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Conectar zooms", "connectZoomsDescription": "Suaviza regiões de zoom consecutivas em um movimento contínuo de câmera.", "autoApplyFreshRecordingZooms": "Aplicar zooms automaticamente em gravações novas", @@ -57,6 +62,20 @@ "zoomOutDescription": "Controla como a câmera sai de uma região de zoom.", "connectedZoomTitle": "Entre zooms", "connectedZoomDescription": "Ajuste o deslizamento entre regiões de zoom consecutivas quando a conexão estiver ativada.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Duração do zoom in", "zoomInOverlap": "Sobreposição do zoom in", "zoomOutDuration": "Duração do zoom out", @@ -74,7 +93,36 @@ }, "cursorSize": "Tamanho do cursor", "cursorSmoothing": "Suavização do cursor", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Desligado", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "Desfoque de movimento do cursor", "cursorClickBounce": "Salto de clique do cursor", "cursorClickBounceDuration": "Velocidade do salto", @@ -125,8 +173,6 @@ "generateFull": "Gerar legendas", "regenerateFull": "Gerar legendas novamente", "clearFull": "Limpar legendas", - "editCurrent": "Editar legenda atual", - "editSaved": "Legenda atualizada", "fontSettings": "Configurações da fonte", "defaultFont": "Padrão", "fontFamily": "Fonte", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 687f8383b..d9cf3a3ec 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -97,6 +97,32 @@ "cursorSpringDamping": "Затухание", "cursorSpringMass": "Масса (инерция)", "off": "Выкл.", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "Размытие в движении", "cursorClickBounce": "Отскок при клике", "cursorClickBounceDuration": "Скорость отскока", @@ -216,4 +242,4 @@ "mixedLabel": "Источник", "deleteRegion": "Удалить аудио" } -} \ No newline at end of file +} diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index a68d2a04f..c4bce8ded 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "背景模糊", "zoomMotionBlur": "缩放运动模糊", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "连接缩放", "connectZoomsDescription": "将连续的缩放区域平滑连接为一次连续的镜头移动。", "autoApplyFreshRecordingZooms": "自动为新录制应用缩放", @@ -92,6 +97,32 @@ "cursorSpringDamping": "光标弹簧阻尼", "cursorSpringMass": "光标弹簧质量", "off": "关", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "光标运动模糊", "cursorClickBounce": "光标点击弹跳", "cursorClickBounceDuration": "弹跳速度", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index db0ea7e04..fded24416 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "背景模糊", "zoomMotionBlur": "縮放動態模糊", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "連接縮放", "connectZoomsDescription": "將連續的縮放區域平滑串接成一段連續的鏡頭移動。", "autoApplyFreshRecordingZooms": "自動套用新錄影的縮放", @@ -57,6 +62,20 @@ "zoomOutDescription": "控制相機如何離開縮放區域。", "connectedZoomTitle": "縮放之間", "connectedZoomDescription": "啟用連接後，調整連續縮放區域之間的滑行效果。", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "拉近時間", "zoomInOverlap": "拉近重疊", "zoomOutDuration": "拉遠時間", @@ -74,7 +93,36 @@ }, "cursorSize": "游標大小", "cursorSmoothing": "游標平滑", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "關閉", + "cursorClickEffects": { + "title": "Click Effects", + "advanced": "Advanced", + "advancedShow": "Show advanced click effect controls", + "advancedHide": "Hide advanced click effect controls", + "color": "Effect Color", + "size": "Effect Size", + "opacity": "Effect Opacity", + "duration": "Effect Duration", + "none": { + "label": "Off", + "description": "No click graphic. Only the cursor motion changes when you click." + }, + "ripple": { + "label": "Ripple", + "description": "Expanding rings radiate from each click so taps read clearly in motion." + }, + "spotlight": { + "label": "Spotlight", + "description": "A soft halo flashes around the pointer to emphasize the clicked area." + }, + "echo": { + "label": "Echo", + "description": "A pair of soft rings that spread outward with a cleaner pulse." + } + }, "cursorMotionBlur": "游標動態模糊", "cursorClickBounce": "點擊彈跳", "cursorClickBounceDuration": "彈跳速度", @@ -125,8 +173,6 @@ "generateFull": "產生字幕", "regenerateFull": "重新產生字幕", "clearFull": "清除字幕", - "editCurrent": "編輯目前字幕", - "editSaved": "字幕已更新", "fontSettings": "字型設定", "defaultFont": "預設", "fontFamily": "字型", diff --git a/src/index.css b/src/index.css index a5ae2368b..1f750e155 100644 --- a/src/index.css +++ b/src/index.css @@ -121,7 +121,6 @@ --slider-glow: 0 0% 100% / 0.5; } } - @layer base { * { @apply border-border; @@ -268,6 +267,3 @@ transform: translateX(270%); } } - - - diff --git a/src/lib/simpleQr.ts b/src/lib/simpleQr.ts new file mode 100644 index 000000000..393d906b7 --- /dev/null +++ b/src/lib/simpleQr.ts @@ -0,0 +1,443 @@ +type QrSvgPath = { + path: string; + size: number; +}; + +type QrProfile = { + version: number; + dataCodewords: number; + eccCodewords: number; + alignmentCenters: number[]; +}; + +const LOW_ECC_PROFILES: QrProfile[] = [ + { version: 1, dataCodewords: 19, eccCodewords: 7, alignmentCenters: [] }, + { version: 2, dataCodewords: 34, eccCodewords: 10, alignmentCenters: [6, 18] }, + { version: 3, dataCodewords: 55, eccCodewords: 15, alignmentCenters: [6, 22] }, + { version: 4, dataCodewords: 80, eccCodewords: 20, alignmentCenters: [6, 26] }, + { version: 5, dataCodewords: 108, eccCodewords: 26, alignmentCenters: [6, 30] }, +]; + +const FORMAT_MASK = 0x5412; +const FORMAT_DIVISOR = 0x537; +const LOW_ECC_FORMAT_BITS = 1; +const QUIET_ZONE_MODULES = 4; +const PAD_CODEWORDS = [0xec, 0x11]; +const FINDER_LIKE_PATTERN = [true, false, true, true, true, false, true]; + +const GF_EXP = new Array(512); +const GF_LOG = new Array(256); + +let x = 1; +for (let i = 0; i < 255; i += 1) { + GF_EXP[i] = x; + GF_LOG[x] = i; + x <<= 1; + if (x & 0x100) { + x ^= 0x11d; + } +} +for (let i = 255; i < GF_EXP.length; i += 1) { + GF_EXP[i] = GF_EXP[i - 255]; +} + +export function createQrSvgPath(value: string): QrSvgPath | null { + const matrix = createQrMatrix(value); + if (!matrix) { + return null; + } + + const size = matrix.length + QUIET_ZONE_MODULES * 2; + const path = matrix + .flatMap((row, rowIndex) => + row.flatMap((dark, colIndex) => + dark + ? [`M${colIndex + QUIET_ZONE_MODULES} ${rowIndex + QUIET_ZONE_MODULES}h1v1h-1z`] + : [], + ), + ) + .join(""); + + return { path, size }; +} + +function createQrMatrix(value: string): boolean[][] | null { + const bytes = new TextEncoder().encode(value); + const profile = selectProfile(bytes.length); + if (!profile) { + return null; + } + + const dataCodewords = createDataCodewords(bytes, profile.dataCodewords); + const errorCodewords = createErrorCorrectionCodewords(dataCodewords, profile.eccCodewords); + const codewords = [...dataCodewords, ...errorCodewords]; + const bits = codewords.flatMap((codeword) => intToBits(codeword, 8)); + const base = createBaseMatrix(profile); + placeDataBits(base.modules, base.reserved, bits); + + let bestMatrix: boolean[][] | null = null; + let bestPenalty = Number.POSITIVE_INFINITY; + + for (let mask = 0; mask < 8; mask += 1) { + const candidate = applyMask(base.modules, base.reserved, mask); + drawFormatBits(candidate, mask); + const penalty = scorePenalty(candidate); + if (penalty < bestPenalty) { + bestPenalty = penalty; + bestMatrix = candidate; + } + } + + return bestMatrix; +} + +function selectProfile(byteLength: number): QrProfile | null { + return ( + LOW_ECC_PROFILES.find((profile) => { + const capacityBits = profile.dataCodewords * 8; + const requiredBits = 4 + 8 + byteLength * 8; + return requiredBits <= capacityBits; + }) ?? null + ); +} + +function createDataCodewords(bytes: Uint8Array, dataCodewordCount: number): number[] { + const capacityBits = dataCodewordCount * 8; + const bits = [...intToBits(0b0100, 4), ...intToBits(bytes.length, 8)]; + for (const byte of bytes) { + bits.push(...intToBits(byte, 8)); + } + + const terminatorLength = Math.min(4, capacityBits - bits.length); + for (let i = 0; i < terminatorLength; i += 1) { + bits.push(0); + } + while (bits.length % 8 !== 0) { + bits.push(0); + } + + const codewords: number[] = []; + for (let i = 0; i < bits.length; i += 8) { + codewords.push(bitsToInt(bits.slice(i, i + 8))); + } + let padIndex = 0; + while (codewords.length < dataCodewordCount) { + codewords.push(PAD_CODEWORDS[padIndex % PAD_CODEWORDS.length]); + padIndex += 1; + } + + return codewords; +} + +function createErrorCorrectionCodewords(data: number[], degree: number): number[] { + const generator = createGeneratorPolynomial(degree); + const message = [...data, ...new Array(degree).fill(0)]; + + for (let i = 0; i < data.length; i += 1) { + const factor = message[i]; + if (factor === 0) { + continue; + } + for (let j = 0; j < generator.length; j += 1) { + message[i + j] ^= gfMultiply(generator[j], factor); + } + } + + return message.slice(message.length - degree); +} + +function createGeneratorPolynomial(degree: number): number[] { + let result = [1]; + for (let i = 0; i < degree; i += 1) { + const next = new Array(result.length + 1).fill(0); + for (let j = 0; j < result.length; j += 1) { + next[j] ^= result[j]; + next[j + 1] ^= gfMultiply(result[j], GF_EXP[i]); + } + result = next; + } + return result; +} + +function gfMultiply(a: number, b: number): number { + if (a === 0 || b === 0) { + return 0; + } + return GF_EXP[GF_LOG[a] + GF_LOG[b]]; +} + +function createBaseMatrix(profile: QrProfile): { + modules: (boolean | null)[][]; + reserved: boolean[][]; +} { + const size = profile.version * 4 + 17; + const modules = Array.from({ length: size }, () => new Array(size).fill(null)); + const reserved = Array.from({ length: size }, () => new Array(size).fill(false)); + const setFunctionModule = (row: number, col: number, dark: boolean) => { + if (row < 0 || col < 0 || row >= size || col >= size) { + return; + } + modules[row][col] = dark; + reserved[row][col] = true; + }; + + drawFinderPattern(setFunctionModule, 0, 0); + drawFinderPattern(setFunctionModule, 0, size - 7); + drawFinderPattern(setFunctionModule, size - 7, 0); + drawAlignmentPatterns(profile, setFunctionModule); + drawTimingPatterns(size, setFunctionModule); + reserveFormatAreas(size, setFunctionModule); + setFunctionModule(profile.version * 4 + 9, 8, true); + + return { modules, reserved }; +} + +function drawFinderPattern( + setFunctionModule: (row: number, col: number, dark: boolean) => void, + row: number, + col: number, +) { + for (let y = -1; y <= 7; y += 1) { + for (let x = -1; x <= 7; x += 1) { + const inFinder = x >= 0 && x <= 6 && y >= 0 && y <= 6; + const dark = + inFinder && + (x === 0 || + x === 6 || + y === 0 || + y === 6 || + (x >= 2 && x <= 4 && y >= 2 && y <= 4)); + setFunctionModule(row + y, col + x, dark); + } + } +} + +function drawAlignmentPatterns( + profile: QrProfile, + setFunctionModule: (row: number, col: number, dark: boolean) => void, +) { + for (const row of profile.alignmentCenters) { + for (const col of profile.alignmentCenters) { + const overlapsFinder = + (row === 6 && col === 6) || + (row === 6 && col === profile.version * 4 + 10) || + (row === profile.version * 4 + 10 && col === 6); + if (overlapsFinder) { + continue; + } + for (let y = -2; y <= 2; y += 1) { + for (let x = -2; x <= 2; x += 1) { + setFunctionModule(row + y, col + x, Math.max(Math.abs(x), Math.abs(y)) !== 1); + } + } + } + } +} + +function drawTimingPatterns( + size: number, + setFunctionModule: (row: number, col: number, dark: boolean) => void, +) { + for (let i = 8; i < size - 8; i += 1) { + const dark = i % 2 === 0; + setFunctionModule(6, i, dark); + setFunctionModule(i, 6, dark); + } +} + +function reserveFormatAreas( + size: number, + setFunctionModule: (row: number, col: number, dark: boolean) => void, +) { + for (let i = 0; i < 9; i += 1) { + if (i !== 6) { + setFunctionModule(8, i, false); + setFunctionModule(i, 8, false); + } + } + for (let i = 0; i < 8; i += 1) { + setFunctionModule(size - 1 - i, 8, false); + setFunctionModule(8, size - 1 - i, false); + } +} + +function placeDataBits(modules: (boolean | null)[][], reserved: boolean[][], bits: number[]) { + const size = modules.length; + let bitIndex = 0; + let upward = true; + + for (let right = size - 1; right >= 1; right -= 2) { + if (right === 6) { + right -= 1; + } + for (let vertical = 0; vertical < size; vertical += 1) { + const row = upward ? size - 1 - vertical : vertical; + for (let offset = 0; offset < 2; offset += 1) { + const col = right - offset; + if (reserved[row][col]) { + continue; + } + modules[row][col] = bitIndex < bits.length ? bits[bitIndex] === 1 : false; + bitIndex += 1; + } + } + upward = !upward; + } +} + +function applyMask( + modules: (boolean | null)[][], + reserved: boolean[][], + mask: number, +): boolean[][] { + return modules.map((row, rowIndex) => + row.map((value, colIndex) => { + const dark = value === true; + if (reserved[rowIndex][colIndex]) { + return dark; + } + return shouldMask(mask, rowIndex, colIndex) ? !dark : dark; + }), + ); +} + +function shouldMask(mask: number, row: number, col: number): boolean { + switch (mask) { + case 0: + return (row + col) % 2 === 0; + case 1: + return row % 2 === 0; + case 2: + return col % 3 === 0; + case 3: + return (row + col) % 3 === 0; + case 4: + return (Math.floor(row / 2) + Math.floor(col / 3)) % 2 === 0; + case 5: + return ((row * col) % 2) + ((row * col) % 3) === 0; + case 6: + return (((row * col) % 2) + ((row * col) % 3)) % 2 === 0; + case 7: + return (((row + col) % 2) + ((row * col) % 3)) % 2 === 0; + default: + return false; + } +} + +function drawFormatBits(matrix: boolean[][], mask: number) { + const size = matrix.length; + const bits = createFormatBits(mask); + const getBit = (index: number) => ((bits >> index) & 1) !== 0; + + for (let i = 0; i <= 5; i += 1) { + matrix[i][8] = getBit(i); + } + matrix[7][8] = getBit(6); + matrix[8][8] = getBit(7); + matrix[8][7] = getBit(8); + for (let i = 9; i < 15; i += 1) { + matrix[8][14 - i] = getBit(i); + } + + for (let i = 0; i < 8; i += 1) { + matrix[8][size - 1 - i] = getBit(i); + } + for (let i = 8; i < 15; i += 1) { + matrix[size - 15 + i][8] = getBit(i); + } + matrix[size - 8][8] = true; +} + +function createFormatBits(mask: number): number { + const data = (LOW_ECC_FORMAT_BITS << 3) | mask; + let remainder = data; + for (let i = 0; i < 10; i += 1) { + remainder = (remainder << 1) ^ (((remainder >> 9) & 1) === 1 ? FORMAT_DIVISOR : 0); + } + return ((data << 10) | (remainder & 0x3ff)) ^ FORMAT_MASK; +} + +function scorePenalty(matrix: boolean[][]): number { + let penalty = 0; + const size = matrix.length; + + for (let row = 0; row < size; row += 1) { + penalty += scoreLine(matrix[row]); + penalty += scoreFinderLikePatterns(matrix[row]); + } + for (let col = 0; col < size; col += 1) { + const column = matrix.map((row) => row[col]); + penalty += scoreLine(column); + penalty += scoreFinderLikePatterns(column); + } + for (let row = 0; row < size - 1; row += 1) { + for (let col = 0; col < size - 1; col += 1) { + const color = matrix[row][col]; + if ( + color === matrix[row][col + 1] && + color === matrix[row + 1][col] && + color === matrix[row + 1][col + 1] + ) { + penalty += 3; + } + } + } + + const darkCount = matrix.flat().filter(Boolean).length; + const percent = (darkCount * 100) / (size * size); + penalty += Math.floor(Math.abs(percent - 50) / 5) * 10; + + return penalty; +} + +function scoreFinderLikePatterns(line: boolean[]): number { + let penalty = 0; + for (let i = 0; i <= line.length - FINDER_LIKE_PATTERN.length; i += 1) { + const matches = FINDER_LIKE_PATTERN.every((value, index) => line[i + index] === value); + if (!matches) { + continue; + } + + const hasLightBefore = i >= 4 && line.slice(i - 4, i).every((value) => value === false); + const afterStart = i + FINDER_LIKE_PATTERN.length; + const hasLightAfter = + afterStart + 4 <= line.length && + line.slice(afterStart, afterStart + 4).every((value) => value === false); + if (hasLightBefore || hasLightAfter) { + penalty += 40; + } + } + return penalty; +} + +function scoreLine(line: boolean[]): number { + let penalty = 0; + let runColor = line[0]; + let runLength = 1; + + for (let i = 1; i < line.length; i += 1) { + if (line[i] === runColor) { + runLength += 1; + continue; + } + if (runLength >= 5) { + penalty += runLength - 2; + } + runColor = line[i]; + runLength = 1; + } + + if (runLength >= 5) { + penalty += runLength - 2; + } + + return penalty; +} + +function intToBits(value: number, length: number): number[] { + return Array.from({ length }, (_, index) => (value >> (length - 1 - index)) & 1); +} + +function bitsToInt(bits: number[]): number { + return bits.reduce((value, bit) => (value << 1) | bit, 0); +}