From db0e154f61f61850578ed0b86923ad47c03467ce Mon Sep 17 00:00:00 2001 From: unknownproperty Date: Tue, 17 Mar 2026 21:05:54 +0300 Subject: [PATCH] feat(modules.calls): noise --- packages/modules.calls/index.ts | 2 + packages/modules.calls/src/hooks/index.ts | 1 + .../modules.calls/src/hooks/useNoiseGate.ts | 164 +++++ packages/modules.calls/src/index.ts | 3 +- .../src/providers/LiveKitProvider.tsx | 7 + packages/modules.calls/src/store/index.ts | 1 + .../modules.calls/src/store/userChoices.ts | 3 + packages/modules.calls/src/ui/Up/Settings.tsx | 38 +- packages/modules.profile/package.json | 1 + packages/modules.profile/src/ui/Content.tsx | 2 + packages/modules.profile/src/ui/Header.tsx | 1 + packages/modules.profile/src/ui/Menu.tsx | 8 +- .../modules.profile/src/ui/SoundAndVideo.tsx | 600 ++++++++++++++++++ pnpm-lock.yaml | 104 +-- 14 files changed, 884 insertions(+), 51 deletions(-) create mode 100644 packages/modules.calls/src/hooks/useNoiseGate.ts create mode 100644 packages/modules.profile/src/ui/SoundAndVideo.tsx diff --git a/packages/modules.calls/index.ts b/packages/modules.calls/index.ts index 93ea958a..8c86a5e4 100644 --- a/packages/modules.calls/index.ts +++ b/packages/modules.calls/index.ts @@ -2,5 +2,7 @@ export { Call, CompactView, useModeSync, ModeSyncProvider } from './src'; export { LiveKitProvider } from './src/providers/LiveKitProvider'; export { RoomProvider } from './src/providers/RoomProvider'; export { useCallStore } from './src/store/callStore'; +export { useUserChoicesStore } from './src/store/userChoices'; +export type { LocalUserChoices } from './src/store/userChoices'; export { useStartCall } from './src/hooks'; export { useUmamiActivityHeartbeat } from './src'; diff --git a/packages/modules.calls/src/hooks/index.ts b/packages/modules.calls/src/hooks/index.ts index b4073ee6..c5804298 100644 --- a/packages/modules.calls/src/hooks/index.ts +++ b/packages/modules.calls/src/hooks/index.ts @@ -25,3 +25,4 @@ export { useParticipantJoinSync } from './useParticipantJoinSync'; export { useUmamiActivityHeartbeat } from './useUmamiActivityHeartbeat'; export { useSortedTracks } from './useSortedTracks'; export { useDocumentPiP } from './useDocumentPiP'; +export { useNoiseGate } from './useNoiseGate'; diff --git a/packages/modules.calls/src/hooks/useNoiseGate.ts b/packages/modules.calls/src/hooks/useNoiseGate.ts new file mode 100644 index 00000000..3b4df624 --- /dev/null +++ b/packages/modules.calls/src/hooks/useNoiseGate.ts @@ -0,0 +1,164 @@ +import { useEffect, useRef } from 'react'; +import { useLocalParticipant } from '@livekit/components-react'; +import type { LocalAudioTrack } from 'livekit-client'; +import { useUserChoicesStore } from '../store/userChoices'; + +/** + * Границы RMS-шкалы. Совпадают с SoundAndVideo.tsx — + * при изменении нужно обновлять оба файла. + */ +const RMS_FLOOR = 0.002; +const RMS_CEIL = 0.3; +const LOG_RANGE = Math.log(RMS_CEIL / RMS_FLOOR); + +function sliderToRms(slider: number): number { + if (slider <= 0) return 0; + return RMS_FLOOR * Math.exp(slider * LOG_RANGE); +} + +type GateState = 'closed' | 'open' | 'holding'; + +/** + * Noise gate для микрофона в LiveKit-звонке. + * + * Два ключевых момента: + * + * 1. audioTrack.mediaStreamTrack — это ГЕТТЕР, который может + * возвращать разные объекты MediaStreamTrack при смене + * процессоров (Krisp), устройства или реконнекте. Поэтому + * нельзя захватывать ссылку один раз — нужно перечитывать + * на каждом тике. + * + * 2. Analyser подключён к КЛОНУ текущего трека. Клон всегда + * active и отдаёт реальный звук с микрофона, даже когда + * оригинал отключён через enabled=false. При смене + * underlying-трека клон пересоздаётся автоматически. + * + * State machine: + * CLOSED → (rms ≥ openThreshold за MIN_OPEN_MS мс) → OPEN + * OPEN → (rms < closeThreshold) → HOLDING + * HOLDING → (rms ≥ closeThreshold) → OPEN + * HOLDING → (hold timer ≥ HOLD_MS) → CLOSED + * + * Hysteresis: closeThreshold = openThreshold × 0.75 + */ +export function useNoiseGate() { + const threshold = useUserChoicesStore((s) => s.micInputSensitivity) ?? 0; + const { microphoneTrack } = useLocalParticipant(); + const audioTrack = microphoneTrack?.track as LocalAudioTrack | undefined; + + const ctxRef = useRef(null); + const intervalRef = useRef | null>(null); + + useEffect(() => { + if (!audioTrack || threshold <= 0) return; + + const ctx = new AudioContext(); + ctxRef.current = ctx; + + const analyser = ctx.createAnalyser(); + analyser.fftSize = 2048; + + let monitorTrack: MediaStreamTrack | null = null; + let monitorSource: MediaStreamAudioSourceNode | null = null; + let trackedTrackId = ''; + + const timeDomain = new Float32Array(analyser.fftSize); + + const HOLD_MS = 600; + const MIN_OPEN_MS = 20; + const HYSTERESIS = 0.75; + + const gateThreshold = sliderToRms(threshold); + const openThreshold = gateThreshold; + const closeThreshold = gateThreshold * HYSTERESIS; + + let state: GateState = 'closed'; + let aboveSince = 0; + let belowSince = 0; + + const initialTrack = audioTrack.mediaStreamTrack; + if (initialTrack) { + trackedTrackId = initialTrack.id; + monitorTrack = initialTrack.clone(); + monitorTrack.enabled = true; + initialTrack.enabled = false; + monitorSource = ctx.createMediaStreamSource(new MediaStream([monitorTrack])); + monitorSource.connect(analyser); + } + + intervalRef.current = setInterval(() => { + const msTrack = audioTrack.mediaStreamTrack; + if (!msTrack) return; + + if (msTrack.id !== trackedTrackId) { + monitorSource?.disconnect(); + monitorTrack?.stop(); + monitorTrack = msTrack.clone(); + monitorTrack.enabled = true; + trackedTrackId = msTrack.id; + monitorSource = ctx.createMediaStreamSource(new MediaStream([monitorTrack])); + monitorSource.connect(analyser); + msTrack.enabled = false; + state = 'closed'; + aboveSince = 0; + belowSince = 0; + } + + const now = performance.now(); + + analyser.getFloatTimeDomainData(timeDomain); + let sum = 0; + for (let i = 0; i < timeDomain.length; i++) { + sum += timeDomain[i] * timeDomain[i]; + } + const rms = Math.sqrt(sum / timeDomain.length); + + switch (state) { + case 'closed': + if (rms >= openThreshold) { + if (aboveSince === 0) aboveSince = now; + if (now - aboveSince >= MIN_OPEN_MS) { + msTrack.enabled = true; + state = 'open'; + aboveSince = 0; + } + } else { + aboveSince = 0; + } + break; + + case 'open': + if (rms < closeThreshold) { + state = 'holding'; + belowSince = now; + } + break; + + case 'holding': + if (rms >= closeThreshold) { + state = 'open'; + belowSince = 0; + } else if (now - belowSince >= HOLD_MS) { + msTrack.enabled = false; + state = 'closed'; + belowSince = 0; + } + break; + } + }, 10); + + return () => { + if (intervalRef.current) { + clearInterval(intervalRef.current); + intervalRef.current = null; + } + monitorTrack?.stop(); + monitorSource?.disconnect(); + ctx.close(); + ctxRef.current = null; + const msTrack = audioTrack.mediaStreamTrack; + if (msTrack) msTrack.enabled = true; + }; + }, [audioTrack, threshold]); +} diff --git a/packages/modules.calls/src/index.ts b/packages/modules.calls/src/index.ts index 3347ceb9..1b6af04e 100644 --- a/packages/modules.calls/src/index.ts +++ b/packages/modules.calls/src/index.ts @@ -15,4 +15,5 @@ export { type UseNoiseCancellationOptions, type UseNoiseCancellationResult, } from './hooks'; -export { useCallStore } from './store'; +export { useCallStore, useUserChoicesStore } from './store'; +export { usePersistentUserChoices } from './hooks'; diff --git a/packages/modules.calls/src/providers/LiveKitProvider.tsx b/packages/modules.calls/src/providers/LiveKitProvider.tsx index c19520a7..210538ac 100644 --- a/packages/modules.calls/src/providers/LiveKitProvider.tsx +++ b/packages/modules.calls/src/providers/LiveKitProvider.tsx @@ -6,6 +6,12 @@ import { useParams, useLocation, useNavigate, useSearch } from '@tanstack/react- import { useEffect, useRef } from 'react'; import { Track } from 'livekit-client'; import { PiPProvider } from './PiPProvider'; +import { useNoiseGate } from '../hooks/useNoiseGate'; + +function NoiseGateEffect() { + useNoiseGate(); + return null; +} type LiveKitProviderProps = { children: React.ReactNode; @@ -239,6 +245,7 @@ export const LiveKitProvider = ({ children }: LiveKitProviderProps) => { audio={audioEnabled || false} video={videoEnabled || false} > + {children} ); diff --git a/packages/modules.calls/src/store/index.ts b/packages/modules.calls/src/store/index.ts index f5c596ac..07e4c580 100644 --- a/packages/modules.calls/src/store/index.ts +++ b/packages/modules.calls/src/store/index.ts @@ -1 +1,2 @@ export { useCallStore } from './callStore'; +export { useUserChoicesStore } from './userChoices'; diff --git a/packages/modules.calls/src/store/userChoices.ts b/packages/modules.calls/src/store/userChoices.ts index 7e1697b5..0872e7f7 100644 --- a/packages/modules.calls/src/store/userChoices.ts +++ b/packages/modules.calls/src/store/userChoices.ts @@ -20,6 +20,8 @@ export type LocalUserChoices = LocalUserChoicesLK & { noiseCancellationEnabled?: boolean; /** Режим шумоподавления: off | webrtc | krisp. */ noiseCancellationMode?: NoiseCancellationMode; + /** Порог чувствительности микрофона (0–1). 0 = noise gate выключен, >0 = порог срабатывания. */ + micInputSensitivity?: number; }; function getUserChoicesState(): LocalUserChoices { @@ -31,6 +33,7 @@ function getUserChoicesState(): LocalUserChoices { videoSubscribeQuality: VideoQuality.HIGH, noiseCancellationEnabled: false, noiseCancellationMode: 'webrtc', + micInputSensitivity: 0, ...loadUserChoices(), }; } diff --git a/packages/modules.calls/src/ui/Up/Settings.tsx b/packages/modules.calls/src/ui/Up/Settings.tsx index cd5f1f97..feb1cc59 100644 --- a/packages/modules.calls/src/ui/Up/Settings.tsx +++ b/packages/modules.calls/src/ui/Up/Settings.tsx @@ -1,5 +1,6 @@ import * as React from 'react'; import { useCallback, useMemo } from 'react'; +import { useLocation, useNavigate, useSearch } from '@tanstack/react-router'; import { Sheet, SheetClose, @@ -8,7 +9,15 @@ import { SheetTitle, SheetTrigger, } from '@xipkg/sheet'; -import { Close, Conference, Microphone, SoundTwo, Chat, Hand } from '@xipkg/icons'; +import { + Close, + Conference, + Microphone, + SoundTwo, + Chat, + Hand, + Settings as SettingsIcon, +} from '@xipkg/icons'; import { Label } from '@xipkg/label'; import { Switch } from '@xipkg/switcher'; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@xipkg/select'; @@ -85,6 +94,13 @@ const DeviceSelector = ({ }; export const Settings = ({ children }: SettingsPropsT) => { + const navigate = useNavigate(); + const { pathname } = useLocation(); + const search = useSearch({ strict: false }) as { + profile?: string; + call?: string; + classroom?: string; + }; const { room } = useRoom(); const { microphoneTrack, cameraTrack, isMicrophoneEnabled, isCameraEnabled } = useLocalParticipant(); @@ -215,6 +231,13 @@ export const Settings = ({ children }: SettingsPropsT) => { [saveAudioOutputDeviceId], ); + const handleOpenSoundSettings = useCallback(() => { + navigate({ + to: pathname, + search: { ...search, profile: 'soundAndVideo' }, + }); + }, [navigate, pathname, search]); + // Обработчики включения/выключения const handleMicrophoneToggle = useCallback(async () => { microphoneToggle.toggle(); @@ -229,7 +252,7 @@ export const Settings = ({ children }: SettingsPropsT) => { {children} - + Настройки @@ -307,6 +330,17 @@ export const Settings = ({ children }: SettingsPropsT) => { /> + + {/* Размытие фона */} {isBlurSupported && (
diff --git a/packages/modules.profile/package.json b/packages/modules.profile/package.json index bbe540ef..39ee4be4 100644 --- a/packages/modules.profile/package.json +++ b/packages/modules.profile/package.json @@ -21,6 +21,7 @@ "@xipkg/toggle": "^2.0.13", "@xipkg/userprofile": "4.0.14", "@xipkg/utils": "1.8.0", + "modules.calls": "workspace:*", "common.api": "workspace:*", "common.auth": "workspace:*", "common.config": "workspace:*", diff --git a/packages/modules.profile/src/ui/Content.tsx b/packages/modules.profile/src/ui/Content.tsx index 4c00f0f0..bae45778 100644 --- a/packages/modules.profile/src/ui/Content.tsx +++ b/packages/modules.profile/src/ui/Content.tsx @@ -4,6 +4,7 @@ import { Secure } from './Secure'; import { PersonalData } from './PersonalData'; import { Notifications } from './Notifications'; import { TechnicalReport } from './TechnicalReport'; +import { SoundAndVideo } from './SoundAndVideo'; type ComponentMapT = { [key: string]: ReactElement; @@ -13,6 +14,7 @@ const componentMap: ComponentMapT = { personalInfo: , personalisation: , security: , + soundAndVideo: , notifications: , report: , }; diff --git a/packages/modules.profile/src/ui/Header.tsx b/packages/modules.profile/src/ui/Header.tsx index 4015c167..e8663caf 100644 --- a/packages/modules.profile/src/ui/Header.tsx +++ b/packages/modules.profile/src/ui/Header.tsx @@ -7,6 +7,7 @@ const menuLabels = [ 'Личные данные', // 'Персонализация', 'Безопасность', + 'Звук и видео', 'Уведомления', 'Отчёт', ]; diff --git a/packages/modules.profile/src/ui/Menu.tsx b/packages/modules.profile/src/ui/Menu.tsx index 5106be29..1eaeab80 100644 --- a/packages/modules.profile/src/ui/Menu.tsx +++ b/packages/modules.profile/src/ui/Menu.tsx @@ -1,4 +1,4 @@ -import { Account, Exit, Key, Palette, Notification, File } from '@xipkg/icons'; +import { Account, Exit, Key, Palette, Notification, File, SoundOn } from '@xipkg/icons'; import { Dispatch, SetStateAction } from 'react'; import { useLocation, useNavigate, useSearch } from '@tanstack/react-router'; import { useAuth } from 'common.auth'; @@ -21,6 +21,10 @@ const options: ItemT[] = [ name: 'Безопасность', query: 'security', }, + { + name: 'Звук и видео', + query: 'soundAndVideo', + }, { name: 'Уведомления', query: 'notifications', @@ -60,6 +64,8 @@ const Item = ({ index, item, onMenuItemChange }: ItemPropsT) => { return ; case 'security': return ; + case 'soundAndVideo': + return ; case 'notifications': return ; case 'report': diff --git a/packages/modules.profile/src/ui/SoundAndVideo.tsx b/packages/modules.profile/src/ui/SoundAndVideo.tsx new file mode 100644 index 00000000..008a146c --- /dev/null +++ b/packages/modules.profile/src/ui/SoundAndVideo.tsx @@ -0,0 +1,600 @@ +import { useState, useEffect, useRef, useCallback } from 'react'; +import { useMediaQuery } from '@xipkg/utils'; +import { Button } from '@xipkg/button'; +import { + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectTrigger, + SelectValue, +} from '@xipkg/select'; +import { Microphone, SoundTwo, SoundOn, Soundoff } from '@xipkg/icons'; +import { useUserChoicesStore, type LocalUserChoices } from 'modules.calls'; + +// --- Аудио-утилиты --- + +/** + * Границы RMS-шкалы для логарифмического маппинга. + * Ниже RMS_FLOOR → display 0, выше RMS_CEIL → display 1. + * + * Эти же константы дублируются в useNoiseGate.ts — + * при изменении нужно обновлять оба файла. + */ +const RMS_FLOOR = 0.002; +const RMS_CEIL = 0.3; +const LOG_RANGE = Math.log(RMS_CEIL / RMS_FLOOR); + +function computeRMS(data: Float32Array): number { + let sum = 0; + for (let i = 0; i < data.length; i++) { + sum += data[i] * data[i]; + } + return Math.sqrt(sum / data.length); +} + +/** RMS → display 0..1, логарифмическая шкала (обратная к sliderToRms). */ +function rmsToDisplay(rms: number): number { + if (rms <= RMS_FLOOR) return 0; + if (rms >= RMS_CEIL) return 1; + return Math.log(rms / RMS_FLOOR) / LOG_RANGE; +} + +/** Slider position 0..1 → реальный RMS threshold (обратная к rmsToDisplay). */ +function sliderToRms(slider: number): number { + if (slider <= 0) return 0; + return RMS_FLOOR * Math.exp(slider * LOG_RANGE); +} + +// --- Хуки --- + +function useAudioDevices() { + const [inputDevices, setInputDevices] = useState([]); + const [outputDevices, setOutputDevices] = useState([]); + const [permissionState, setPermissionState] = useState<'prompt' | 'granted' | 'denied'>('prompt'); + + const enumerate = useCallback(async () => { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const inputs = devices.filter((d) => d.kind === 'audioinput'); + const outputs = devices.filter((d) => d.kind === 'audiooutput'); + setInputDevices(inputs); + setOutputDevices(outputs); + + if (inputs.length > 0 && inputs[0].label) { + setPermissionState('granted'); + } + } catch { + // devices not available + } + }, []); + + const requestPermission = useCallback(async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + stream.getTracks().forEach((t) => t.stop()); + setPermissionState('granted'); + await enumerate(); + } catch { + setPermissionState('denied'); + } + }, [enumerate]); + + useEffect(() => { + enumerate(); + navigator.mediaDevices.addEventListener('devicechange', enumerate); + return () => navigator.mediaDevices.removeEventListener('devicechange', enumerate); + }, [enumerate]); + + return { inputDevices, outputDevices, permissionState, requestPermission }; +} + +type GateState = 'closed' | 'open' | 'holding'; + +/** + * Тест микрофона с loopback и noise gate. + * + * Аудиограф: + * Source → Analyser (мгновенный RMS, без задержки) + * Source → Delay(40ms) → Gate → Destination →