diff --git a/.vscode/settings.json b/.vscode/settings.json index f85eede43..b34a6f38e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -25,5 +25,7 @@ "titleBar.inactiveBackground": "#21573299", "titleBar.inactiveForeground": "#e7e7e799" }, - "peacock.color": "#215732" + "peacock.color": "#215732", + "editor.snippetSuggestions": "bottom", + "emmet.showSuggestionsAsSnippets": true, } \ No newline at end of file diff --git a/docs/playground/playground.mdx b/docs/playground/playground.mdx index 53e29ba35..78f9e98e9 100644 --- a/docs/playground/playground.mdx +++ b/docs/playground/playground.mdx @@ -11,8 +11,8 @@ import { AlphaTabPlayground } from "@site/src/components/AlphaTabPlayground"; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index e459c1909..8bc166ed9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,16 +8,18 @@ "name": "alphatab-website", "version": "0.0.0", "dependencies": { - "@coderline/alphatab": "^1.5.0-alpha.1394", + "@coderline/alphatab": "^1.6.0-alpha.1416", "@docusaurus/core": "^3.7.0", "@docusaurus/preset-classic": "^3.7.0", "@docusaurus/theme-mermaid": "^3.7.0", "@fontsource/noto-sans": "^5.1.1", "@fontsource/noto-serif": "^5.1.1", + "@fortawesome/free-brands-svg-icons": "^6.7.2", "@fortawesome/free-regular-svg-icons": "^6.7.2", "@fortawesome/free-solid-svg-icons": "^6.7.2", "@fortawesome/react-fontawesome": "^0.2.2", "@mdx-js/react": "^3.1.0", + "@react-hook/resize-observer": "^2.0.2", "@uidotdev/usehooks": "^2.4.1", "@uiw/react-color": "^2.5.5", "@uiw/react-color-chrome": "^2.5.5", @@ -1917,9 +1919,9 @@ "integrity": "sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==" }, "node_modules/@coderline/alphatab": { - "version": "1.5.0-alpha.1394", - "resolved": "https://registry.npmjs.org/@coderline/alphatab/-/alphatab-1.5.0-alpha.1394.tgz", - "integrity": "sha512-R4Dnvs0xvgCZEPc/jsHZc9VaGlGx7OWFghJBeLlwes6j1Sh55dpnV+kiDKZP+/AlUplQhYfzNqKOIWSNyAAuJw==", + "version": "1.6.0-alpha.1416", + "resolved": "https://registry.npmjs.org/@coderline/alphatab/-/alphatab-1.6.0-alpha.1416.tgz", + "integrity": "sha512-bIACxHyaTAnxtzz5I6Kr3a9LR6VLpvhm/o8GiZZeDc6wVtLEz+0a9sKCRBepSfz0+HVi09YzDMvhL9vEQrEbcw==", "engines": { "node": ">=6.0.0" } @@ -4232,6 +4234,17 @@ "node": ">=6" } }, + "node_modules/@fortawesome/free-brands-svg-icons": { + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@fortawesome/free-brands-svg-icons/-/free-brands-svg-icons-6.7.2.tgz", + "integrity": "sha512-zu0evbcRTgjKfrr77/2XX+bU+kuGfjm0LbajJHVIgBWNIDzrhpRxiCPNT8DW5AdmSsq7Mcf9D1bH0aSeSUSM+Q==", + "dependencies": { + "@fortawesome/fontawesome-common-types": "6.7.2" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@fortawesome/free-regular-svg-icons": { "version": "6.7.2", "resolved": "https://registry.npmjs.org/@fortawesome/free-regular-svg-icons/-/free-regular-svg-icons-6.7.2.tgz", @@ -4823,6 +4836,34 @@ "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz", "integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==" }, + "node_modules/@react-hook/latest": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@react-hook/latest/-/latest-1.0.3.tgz", + "integrity": "sha512-dy6duzl+JnAZcDbNTfmaP3xHiKtbXYOaz3G51MGVljh548Y8MWzTr+PHLOfvpypEVW9zwvl+VyKjbWKEVbV1Rg==", + "peerDependencies": { + "react": ">=16.8" + } + }, + "node_modules/@react-hook/passive-layout-effect": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@react-hook/passive-layout-effect/-/passive-layout-effect-1.2.1.tgz", + "integrity": "sha512-IwEphTD75liO8g+6taS+4oqz+nnroocNfWVHWz7j+N+ZO2vYrc6PV1q7GQhuahL0IOR7JccFTsFKQ/mb6iZWAg==", + "peerDependencies": { + "react": ">=16.8" + } + }, + "node_modules/@react-hook/resize-observer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@react-hook/resize-observer/-/resize-observer-2.0.2.tgz", + "integrity": "sha512-tzKKzxNpfE5TWmxuv+5Ae3IF58n0FQgQaWJmcbYkjXTRZATXxClnTprQ2uuYygYTpu1pqbBskpwMpj6jpT1djA==", + "dependencies": { + "@react-hook/latest": "^1.0.2", + "@react-hook/passive-layout-effect": "^1.2.0" + }, + "peerDependencies": { + "react": ">=18" + } + }, "node_modules/@sideway/address": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", diff --git a/package.json b/package.json index 985621bb7..ea739f37a 100644 --- a/package.json +++ b/package.json @@ -16,16 +16,18 @@ "generate-alphatabdoc": "tsx scripts/generate-alphatabdoc.mts" }, "dependencies": { - "@coderline/alphatab": "^1.5.0-alpha.1394", + "@coderline/alphatab": "^1.6.0-alpha.1416", "@docusaurus/core": "^3.7.0", "@docusaurus/preset-classic": "^3.7.0", "@docusaurus/theme-mermaid": "^3.7.0", "@fontsource/noto-sans": "^5.1.1", "@fontsource/noto-serif": "^5.1.1", + "@fortawesome/free-brands-svg-icons": "^6.7.2", "@fortawesome/free-regular-svg-icons": "^6.7.2", "@fortawesome/free-solid-svg-icons": "^6.7.2", "@fortawesome/react-fontawesome": "^0.2.2", "@mdx-js/react": "^3.1.0", + "@react-hook/resize-observer": "^2.0.2", "@uidotdev/usehooks": "^2.4.1", "@uiw/react-color": "^2.5.5", "@uiw/react-color-chrome": "^2.5.5", diff --git a/src/components/AlphaTabPlayground/index.tsx b/src/components/AlphaTabPlayground/index.tsx index 581738ace..ecbff9af6 100644 --- a/src/components/AlphaTabPlayground/index.tsx +++ b/src/components/AlphaTabPlayground/index.tsx @@ -1,16 +1,17 @@ 'use client'; -import type * as alphaTab from '@coderline/alphatab'; -import React, { useEffect, useState } from 'react'; +import * as alphaTab from '@coderline/alphatab'; +import React, { useState } from 'react'; import { useAlphaTab, useAlphaTabEvent } from '@site/src/hooks'; import styles from './styles.module.scss'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import * as solid from '@fortawesome/free-solid-svg-icons'; import { openFile } from '@site/src/utils'; -import { PlayerControlsGroup, SidePanel } from './player-controls-group'; +import { BottomPanel, PlayerControlsGroup, SidePanel } from './player-controls-group'; import { PlaygroundSettings } from './playground-settings'; import { Tooltip } from 'react-tooltip'; import { PlaygroundTrackSelector } from './track-selector'; +import { MediaSyncEditor } from './media-sync-editor'; interface AlphaTabPlaygroundProps { settings?: alphaTab.json.SettingsJson; @@ -20,12 +21,13 @@ export const AlphaTabPlayground: React.FC = ({ settings const viewPortRef = React.createRef(); const [isLoading, setLoading] = useState(true); const [sidePanel, setSidePanel] = useState(SidePanel.None); + const [bottomPanel, setBottomPanel] = useState(BottomPanel.None); const [api, element] = useAlphaTab(s => { s.core.engine = 'svg'; s.player.scrollElement = viewPortRef.current!; s.player.scrollOffsetY = -10; - s.player.enablePlayer = true; + s.player.playerMode = alphaTab.PlayerMode.EnabledAutomatic; if (settings) { s.fillFromJson(settings); } @@ -88,7 +90,18 @@ export const AlphaTabPlayground: React.FC = ({ settings
- {api && } + {api && api?.score && bottomPanel === BottomPanel.MediaSyncEditor && ( + + )} + {api && ( + + )}
diff --git a/src/components/AlphaTabPlayground/media-sync-editor.tsx b/src/components/AlphaTabPlayground/media-sync-editor.tsx new file mode 100644 index 000000000..cb4f68cec --- /dev/null +++ b/src/components/AlphaTabPlayground/media-sync-editor.tsx @@ -0,0 +1,921 @@ +'use client'; + +import * as alphaTab from '@coderline/alphatab'; +import type React from 'react'; +import styles from './styles.module.scss'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import * as solid from '@fortawesome/free-solid-svg-icons'; +import * as brands from '@fortawesome/free-brands-svg-icons'; +import { useCallback, useEffect, useRef, useState } from 'react'; +import useResizeObserver from '@react-hook/resize-observer'; +import { useAlphaTabEvent } from '@site/src/hooks'; + +export type MediaSyncEditorProps = { + api: alphaTab.AlphaTabApi; + score: alphaTab.model.Score; +}; + +type MasterBarMarker = { + label: string; + syncTime: number; + + synthTime: number; + synthBpm: number; + synthTickDuration: number; + + masterBarIndex: number; + occurence: number; + modifiedTempo?: number; + + isStartMarker: boolean; + isEndMarker: boolean; +}; + +type SyncPointInfo = { + endTick: number; + endTime: number; + sampleRate: number; + leftSamples: Float32Array; + rightSamples: Float32Array; + masterBarMarkers: MasterBarMarker[]; +}; + +// TODO: handle intermediate tempo changes and sync points + +function ticksToMillis(tick: number, bpm: number): number { + return (tick * 60000.0) / (bpm * 960); +} + +async function buildSyncPointInfo(api: alphaTab.AlphaTabApi, createInitialSyncPoints: boolean): Promise { + const tickCache = api.tickCache; + if (!tickCache || !api.score?.backingTrack?.rawAudioFile) { + return { + endTick: 0, + endTime: 0, + sampleRate: 0, + leftSamples: new Float32Array(0), + rightSamples: new Float32Array(0), + masterBarMarkers: [] + }; + } + + const audioContext = new AudioContext(); + const buffer = await audioContext.decodeAudioData(api.score!.backingTrack!.rawAudioFile.buffer.slice(0)); + const rawSamples: Float32Array[] = + buffer.numberOfChannels === 1 + ? [buffer.getChannelData(0), buffer.getChannelData(0)] + : [buffer.getChannelData(0), buffer.getChannelData(1)]; + + const sampleRate = audioContext.sampleRate; + const endTime = rawSamples[0].length / sampleRate; + + await audioContext.close(); + + return { + endTick: api.tickCache.masterBars.at(-1)!.end, + masterBarMarkers: buildMasterBarMarkers(api, createInitialSyncPoints), + sampleRate, + leftSamples: rawSamples[0], + rightSamples: rawSamples[1], + endTime + }; +} +function buildMasterBarMarkers(api: alphaTab.AlphaTabApi, createInitialSyncPoints: boolean): MasterBarMarker[] { + const markers: MasterBarMarker[] = []; + + if (createInitialSyncPoints) { + // create initial sync points for all tempo changes to ensure the song and the + // backing track roughly align + let synthBpm = 0; + let synthTimePosition = 0; + let synthTickPosition = 0; + + const occurences = new Map(); + for (const masterBar of api.tickCache!.masterBars) { + const occurence = occurences.get(masterBar.masterBar.index) ?? 0; + occurences.set(masterBar.masterBar.index, occurence + 1); + + for (const changes of masterBar.tempoChanges) { + const absoluteTick = changes.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMillis(tickOffset, synthBpm); + + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } + + if (changes.tempo !== synthBpm && changes.tick === masterBar.start) { + const syncPoint = new alphaTab.model.Automation(); + syncPoint.ratioPosition = 0; + syncPoint.type = alphaTab.model.AutomationType.SyncPoint; + syncPoint.syncPointValue = new alphaTab.model.SyncPointData(); + syncPoint.syncPointValue.barOccurence = occurence; + syncPoint.syncPointValue.millisecondOffset = synthTimePosition; + syncPoint.syncPointValue.modifiedTempo = changes.tempo; + masterBar.masterBar.addSyncPoint(syncPoint); + + synthBpm = changes.tempo; + } + } + + const tickOffset = masterBar.end - synthTickPosition; + const timeOffset = ticksToMillis(tickOffset, synthBpm); + synthTickPosition += tickOffset; + synthTimePosition += timeOffset; + } + } + + const occurences = new Map(); + let syncBpm = api.score!.tempo; + let syncLastTick = 0; + let syncLastMillisecondOffset = 0; + + let synthBpm = api.score!.tempo; + let synthTimePosition = 0; + let synthTickPosition = 0; + + for (const masterBar of api.tickCache!.masterBars) { + const occurence = occurences.get(masterBar.masterBar.index) ?? 1; + occurences.set(masterBar.masterBar.index, occurence + 1); + + const occurenceLabel = occurence > 1 ? ` (${occurence})` : ''; + const startSyncPoint = masterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 0); + + let syncedStartTime: number; + if (startSyncPoint) { + syncedStartTime = startSyncPoint.syncPointValue!.millisecondOffset; + syncBpm = startSyncPoint.syncPointValue!.modifiedTempo; + syncLastMillisecondOffset = syncedStartTime; + syncLastTick = masterBar.start; + } else { + const tickOffset = masterBar.start - syncLastTick; + syncedStartTime = syncLastMillisecondOffset + ticksToMillis(tickOffset, syncBpm); + } + + const isStartMarker = masterBar.masterBar.index === 0 && occurence === 1; + const newMarker: MasterBarMarker = { + label: isStartMarker ? 'Start' : `${masterBar.masterBar.index + 1}${occurenceLabel}`, + masterBarIndex: masterBar.masterBar.index, + synthTickDuration: masterBar.end - masterBar.start, + occurence: occurence, + syncTime: syncedStartTime / 1000, + synthTime: synthTimePosition / 1000, + synthBpm: masterBar.tempoChanges.length > 0 ? masterBar.tempoChanges[0].tempo : synthBpm, + modifiedTempo: startSyncPoint?.syncPointValue?.modifiedTempo, + isStartMarker, + isEndMarker: false + }; + markers.push(newMarker); + + for (const changes of masterBar.tempoChanges) { + const absoluteTick = changes.tick; + const tickOffset = absoluteTick - synthTickPosition; + if (tickOffset > 0) { + const timeOffset = ticksToMillis(tickOffset, synthBpm); + + synthTickPosition = absoluteTick; + synthTimePosition += timeOffset; + } + + synthBpm = changes.tempo; + } + + const tickOffset = masterBar.end - synthTickPosition; + const timeOffset = ticksToMillis(tickOffset, synthBpm); + synthTickPosition += tickOffset; + synthTimePosition += timeOffset; + } + + const lastMasterBar = api.tickCache!.masterBars.at(-1)!; + + const endSyncPoint = lastMasterBar.masterBar.syncPoints?.find(m => m.ratioPosition === 1); + + const tickOffset = lastMasterBar.end - syncLastTick; + const endSyncPointTime = endSyncPoint + ? endSyncPoint.syncPointValue!.millisecondOffset + : syncLastMillisecondOffset + ticksToMillis(tickOffset, syncBpm); + + markers.push({ + label: 'End', + masterBarIndex: lastMasterBar.masterBar.index, + synthTickDuration: 0, + occurence: occurences.get(lastMasterBar.masterBar.index)!, + syncTime: endSyncPointTime / 1000, + synthTime: synthTimePosition / 1000, + synthBpm, + modifiedTempo: endSyncPoint?.syncPointValue?.modifiedTempo ?? synthBpm, + isStartMarker: false, + isEndMarker: true + }); + + return markers; +} + +const pixelPerSeconds = 100; +const leftPadding = 15; +const barNumberHeight = 20; +const arrowHeight = 20; +const timeAxisHeight = 20; +const timeAxiSubSecondTickHeight = 5; +const barWidth = 1; +const timeAxisLineColor = '#A5A5A5'; +const waveFormColor = '#436d9d99'; +const font = '12px "Noto Sans"'; +const dragLimit = 10; +const dragThreshold = 5; +const scrollThresholdPercent = 0.2; + +function timePositionToX(timePosition: number, zoom: number): number { + const zoomedPixelPerSecond = pixelPerSeconds * zoom; + return timePosition * zoomedPixelPerSecond + leftPadding; +} + +type MarkerDragInfo = { + startX: number; + startY: number; + endX: number; +}; + +function computeMarkerInlineStyle( + m: MasterBarMarker, + zoom: number, + draggingMarker: MasterBarMarker | null, + draggingMarkerInfo: MarkerDragInfo | null +): React.CSSProperties { + let left = timePositionToX(m.syncTime, zoom); + + if (m === draggingMarker && draggingMarkerInfo) { + const deltaX = draggingMarkerInfo.endX - draggingMarkerInfo.startX; + left += deltaX; + } + + return { + left: `${left}px` + }; +} + +function updateSyncPointsAfterModification(modifiedIndex: number, s: SyncPointInfo, isDelete: boolean) { + // find previous and next sync point (or start/end of the song) + let startIndexForUpdate = Math.max(0, modifiedIndex - 1); + while (startIndexForUpdate > 0 && !s.masterBarMarkers[startIndexForUpdate].modifiedTempo) { + startIndexForUpdate--; + } + + let nextIndexForUpdate = Math.min(s.masterBarMarkers.length - 1, modifiedIndex + 1); + while ( + nextIndexForUpdate < s.masterBarMarkers.length - 1 && + !s.masterBarMarkers[nextIndexForUpdate].modifiedTempo + ) { + nextIndexForUpdate++; + } + + const modifiedMarker = s.masterBarMarkers[modifiedIndex]; + + // update from previous to current + if (startIndexForUpdate < modifiedIndex) { + const previousMarker = { ...s.masterBarMarkers[startIndexForUpdate] }; + s.masterBarMarkers[startIndexForUpdate] = previousMarker; + const synthDuration = modifiedMarker.synthTime - previousMarker.synthTime; + const syncedDuration = modifiedMarker.syncTime - previousMarker.syncTime; + const newBpmBefore = (synthDuration / syncedDuration) * previousMarker.synthBpm; + previousMarker.modifiedTempo = newBpmBefore; + + let syncedTimePosition = previousMarker.syncTime; + for (let i = startIndexForUpdate; i < modifiedIndex; i++) { + const marker = { ...s.masterBarMarkers[i] }; + s.masterBarMarkers[i] = marker; + + marker.syncTime = syncedTimePosition; + syncedTimePosition += ticksToMillis(marker.synthTickDuration, newBpmBefore) / 1000; + } + } + + if (!isDelete) { + const nextMarker = s.masterBarMarkers[nextIndexForUpdate]; + const synthDuration = nextMarker.synthTime - modifiedMarker.synthTime; + const syncedDuration = nextMarker.syncTime - modifiedMarker.syncTime; + const newBpmAfter = (synthDuration / syncedDuration) * modifiedMarker.synthBpm; + modifiedMarker.modifiedTempo = newBpmAfter; + + let syncedTimePosition = + modifiedMarker.syncTime + ticksToMillis(modifiedMarker.synthTickDuration, newBpmAfter) / 1000; + for (let i = modifiedIndex + 1; i < nextIndexForUpdate; i++) { + const marker = { ...s.masterBarMarkers[i] }; + s.masterBarMarkers[i] = marker; + marker.syncTime = syncedTimePosition; + + syncedTimePosition += ticksToMillis(marker.synthTickDuration, newBpmAfter) / 1000; + } + } +} + +type UndoStack = { + undo: SyncPointInfo[]; + redo: SyncPointInfo[]; +}; + +export const MediaSyncEditor: React.FC = ({ api, score }) => { + const markerCanvas = useRef(null); + const waveFormCanvas = useRef(null); + const syncArea = useRef(null); + + const [canvasSize, setCanvasSize] = useState([0, 0]); + const [virtualWidth, setVirtualWidth] = useState(0); + const [zoom, setZoom] = useState(1); + + const [syncPointInfo, setSyncPointInfo] = useState({ + endTick: 0, + endTime: 0, + sampleRate: 44100, + leftSamples: new Float32Array(0), + rightSamples: new Float32Array(0), + masterBarMarkers: [] + }); + + const [draggingMarker, setDraggingMarker] = useState(null); + const [draggingMarkerInfo, setDraggingMarkerInfo] = useState(null); + const [undoStack, setUndoStack] = useState({ undo: [], redo: [] }); + const [shouldStoreToUndo, setStoreToUndo] = useState(false); + const [shouldApplySyncPoints, setApplySyncPoints] = useState(false); + const [shouldCreateInitialSyncPoints, setCreateInitialSyncPoints] = useState(false); + + const [audioElement, setAudioElement] = useState(null); + const [playbackTime, setPlaybackTime] = useState(0); + + useEffect(() => { + setAudioElement( + (api.player!.output as alphaTab.synth.IAudioElementBackingTrackSynthOutput)?.audioElement ?? null + ); + }, [api.player!.output]); + + useAlphaTabEvent( + api, + 'midiLoad', + () => { + setAudioElement( + (api.player!.output as alphaTab.synth.IAudioElementBackingTrackSynthOutput)?.audioElement ?? null + ); + buildSyncPointInfo(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); + setCreateInitialSyncPoints(false); + }, + [shouldCreateInitialSyncPoints] + ); + + useEffect(() => { + if (syncArea.current) { + const xPos = timePositionToX(playbackTime, zoom); + const canvasWidth = canvasSize[0]; + const threshold = canvasWidth * scrollThresholdPercent; + const scrollOffset = syncArea.current.scrollLeft; + + // is out of screen? + if (xPos < scrollOffset + threshold || (xPos - scrollOffset) > (canvasWidth - threshold)) { + syncArea.current.scrollTo({ + left: xPos - canvasWidth / 2, + behavior: 'smooth' + }); + } + } + }, [api, playbackTime, canvasSize, syncArea]); + + useEffect(() => { + const updateWaveFormCursor = () => { + setPlaybackTime(audioElement!.currentTime); + }; + + let timeUpdate: number = 0; + + if (audioElement) { + console.log('Audio element', audioElement); + audioElement.addEventListener('timeupdate', updateWaveFormCursor); + audioElement.addEventListener('durationchange', updateWaveFormCursor); + audioElement.addEventListener('seeked', updateWaveFormCursor); + timeUpdate = window.setInterval(() => { + if (audioElement) { + setPlaybackTime(audioElement.currentTime); + } + }, 50); + updateWaveFormCursor(); + } + + return () => { + if (audioElement) { + console.log('unregister Audio element', audioElement); + audioElement.removeEventListener('timeupdate', updateWaveFormCursor); + audioElement.removeEventListener('durationchange', updateWaveFormCursor); + audioElement.removeEventListener('seeked', updateWaveFormCursor); + window.clearInterval(timeUpdate); + } + }; + }, [audioElement]); + + useEffect(() => { + if (!syncPointInfo) { + return; + } + if (shouldStoreToUndo) { + setUndoStack(s => ({ + undo: [...s.undo, syncPointInfo], + redo: [] + })); + setStoreToUndo(false); + } + + const syncPointLookup = new Map(); + for (const m of syncPointInfo.masterBarMarkers) { + if (m.modifiedTempo) { + let syncPoints = syncPointLookup.get(m.masterBarIndex); + if (!syncPoints) { + syncPoints = []; + syncPointLookup.set(m.masterBarIndex, syncPoints); + } + + const automation = new alphaTab.model.Automation(); + automation.ratioPosition = m.isEndMarker ? 1 : 0; + automation.type = alphaTab.model.AutomationType.SyncPoint; + automation.syncPointValue = new alphaTab.model.SyncPointData(); + automation.syncPointValue.modifiedTempo = m.modifiedTempo; + automation.syncPointValue.millisecondOffset = m.syncTime * 1000; + automation.syncPointValue.barOccurence = m.occurence - 1; + syncPoints.push(automation); + } + } + + if (shouldApplySyncPoints) { + console.log('Apply Sync points', syncPointLookup); + + // remember and set again the tick position after sync point update + // this will ensure the cursor and player seek accordingly with keeping the cursor + // where it is currently shown on the notation. + const tickPosition = api.tickPosition; + for (const masterBar of score.masterBars) { + masterBar.syncPoints = syncPointLookup.get(masterBar.index); + } + api.updateSyncPoints(); + api.tickPosition = tickPosition; + setApplySyncPoints(false); + } + }, [syncPointInfo]); + + const undo = () => { + setUndoStack(s => { + const newStack = { ...s }; + if (newStack.undo.length > 0) { + const undoState = newStack.undo.pop()!; + newStack.redo.push(undoState); + setApplySyncPoints(true); + setSyncPointInfo(newStack.undo.at(-1)!); + } + return newStack; + }); + }; + + const redo = () => { + setUndoStack(s => { + const newStack = { ...s }; + if (newStack.redo.length > 0) { + const redoState = newStack.redo.pop()!; + newStack.undo.push(redoState); + setApplySyncPoints(true); + setSyncPointInfo(redoState); + } + return newStack; + }); + }; + + const toggleMarker = (marker: MasterBarMarker, e: React.MouseEvent) => { + e.stopPropagation(); + e.preventDefault(); + setStoreToUndo(true); + setApplySyncPoints(true); + setSyncPointInfo(s => { + if (!s) { + return s; + } + + // no removal of start and end marker + if (marker.isStartMarker || marker.isEndMarker) { + return s; + } + + const markerIndex = s!.masterBarMarkers.indexOf(marker); + if (markerIndex === -1) { + return s; + } + + const newS = { ...s, masterBarMarkers: [...s.masterBarMarkers] }; + if (marker.modifiedTempo) { + newS.masterBarMarkers[markerIndex] = { ...marker, modifiedTempo: undefined }; + updateSyncPointsAfterModification(markerIndex, newS, true); + } else { + updateSyncPointsAfterModification(markerIndex, newS, false); + } + + return newS; + }); + }; + + const mouseUpListener = useCallback( + (e: MouseEvent) => { + if (draggingMarker) { + e.preventDefault(); + e.stopPropagation(); + + const deltaX = draggingMarkerInfo!.endX - draggingMarkerInfo!.startX; + if (deltaX > dragThreshold || draggingMarker.modifiedTempo !== undefined) { + setStoreToUndo(true); + setApplySyncPoints(true); + setSyncPointInfo(s => { + if (!s) { + return s; + } + + const markerIndex = s.masterBarMarkers.findIndex(m => m === draggingMarker); + + const zoomedPixelPerSecond = pixelPerSeconds * zoom; + const deltaTime = deltaX / zoomedPixelPerSecond; + + const newTimePosition = draggingMarker.syncTime + deltaTime; + + const newS = { ...s, masterBarMarkers: [...s.masterBarMarkers] }; + + // move the marker to the new position + newS.masterBarMarkers[markerIndex] = { + ...newS.masterBarMarkers[markerIndex], + syncTime: Math.max(0, newTimePosition) + }; + + updateSyncPointsAfterModification(markerIndex, newS, false); + return newS; + }); + setDraggingMarker(null); + setDraggingMarkerInfo(null); + } + } + }, + [draggingMarker, draggingMarkerInfo] + ); + + const mouseMoveListener = useCallback( + (e: MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + + setDraggingMarkerInfo(s => { + if (!s || !syncPointInfo) { + return s; + } + + const index = syncPointInfo.masterBarMarkers.indexOf(draggingMarker!); + if (index === -1) { + return s; + } + + let pageX = e.pageX; + if (index < syncPointInfo.masterBarMarkers.length - 1) { + const deltaX = pageX - s.startX; + const thisX = timePositionToX(draggingMarker!.syncTime, zoom); + const newX = thisX + deltaX; + + let nextMarkerIndex = index + 1; + while ( + nextMarkerIndex < syncPointInfo.masterBarMarkers.length - 1 && + !syncPointInfo.masterBarMarkers[nextMarkerIndex].modifiedTempo + ) { + nextMarkerIndex++; + } + + const nextMarker = syncPointInfo.masterBarMarkers[nextMarkerIndex]; + const nextX = timePositionToX(nextMarker.syncTime, zoom); + const maxX = nextX - dragLimit; + + if (newX > maxX) { + pageX = s.startX + (maxX - thisX); + } + } + + return { ...s, endX: pageX }; + }); + }, + [draggingMarker, syncPointInfo] + ); + + useEffect(() => { + if (draggingMarker) { + document.addEventListener('mouseup', mouseUpListener); + document.addEventListener('mousemove', mouseMoveListener); + } + + return () => { + document.removeEventListener('mouseup', mouseUpListener); + document.removeEventListener('mousemove', mouseMoveListener); + }; + }, [draggingMarker, mouseUpListener, mouseMoveListener]); + + const startMarkerDrag = (marker: MasterBarMarker, e: React.MouseEvent) => { + if (e.button !== 0 || marker.modifiedTempo === undefined) { + return; + } + e.preventDefault(); + e.stopPropagation(); + setDraggingMarkerInfo(() => ({ startX: e.pageX, startY: e.pageY, endX: e.pageX })); + setDraggingMarker(() => marker); + }; + + useEffect(() => { + setUndoStack({ + undo: [], + redo: [] + }); + setStoreToUndo(true); + buildSyncPointInfo(api, shouldCreateInitialSyncPoints).then(x => setSyncPointInfo(x)); + setCreateInitialSyncPoints(false); + }, [api]); + + const drawWaveform = () => { + const can = waveFormCanvas.current; + if (!syncPointInfo || !can) { + return; + } + + const ctx = can.getContext('2d')!; + ctx.clearRect(0, 0, can.width, can.height); + ctx.save(); + + const waveFormY = barNumberHeight + arrowHeight; + const halfHeight = ((can.height - waveFormY - timeAxisHeight) / 2) | 0; + + // frame + ctx.fillStyle = timeAxisLineColor; + ctx.fillRect(0, waveFormY + 2 * halfHeight, can.width, 1); + ctx.fillRect(0, barNumberHeight, can.width, 1); + ctx.fillRect(0, waveFormY, can.width, 1); + ctx.fillRect(0, waveFormY + halfHeight, can.width, 1); + + // waveform + ctx.translate(-syncArea.current!.scrollLeft, 0); + + ctx.beginPath(); + + const startX = syncArea.current!.scrollLeft; + const endX = startX + can.width; + + const zoomedPixelPerSecond = pixelPerSeconds * zoom; + const samplesPerPixel = syncPointInfo.sampleRate / zoomedPixelPerSecond; + + for (let x = startX; x < endX; x += barWidth) { + const startSample = (x * samplesPerPixel) | 0; + const endSample = ((x + barWidth) * samplesPerPixel) | 0; + + let maxTop = 0; + let maxBottom = 0; + for (let sample = startSample; sample <= endSample; sample++) { + // TODO: a logarithmic scale would be better here to scale 0-1 better as visible waveform + // for now we multiply it for a good scale (unlikely we have a sound with 1 which is very loud) + const visibilityFactor = 5; + const magnitudeTop = Math.min(Math.abs(syncPointInfo.leftSamples[sample] * visibilityFactor || 0), 1); + const magnitudeBottom = Math.min(Math.abs(syncPointInfo.rightSamples[sample] * visibilityFactor || 0), 1); + if (magnitudeTop > maxTop) { + maxTop = magnitudeTop; + } + if (magnitudeBottom > maxBottom) { + maxBottom = magnitudeBottom; + } + } + + const topBarHeight = Math.round(maxTop * halfHeight); + const bottomBarHeight = Math.round(maxBottom * halfHeight); + const barHeight = topBarHeight + bottomBarHeight || 1; + ctx.rect(x, waveFormY + (halfHeight - topBarHeight), barWidth, barHeight); + } + + ctx.fillStyle = waveFormColor; + ctx.fill(); + + // time axis + ctx.save(); + + ctx.fillStyle = timeAxisLineColor; + ctx.font = font; + ctx.textAlign = 'left'; + ctx.textBaseline = 'bottom'; + + const timeAxisY = waveFormY + 2 * halfHeight; + const leftTime = Math.floor((startX - leftPadding) / zoomedPixelPerSecond); + const rightTime = Math.ceil(endX / zoomedPixelPerSecond); + + let time = leftTime; + while (time <= rightTime) { + const timeX = timePositionToX(time, zoom); + ctx.fillRect(timeX, timeAxisY, 1, timeAxisHeight); + + const minutes = Math.floor(time / 60); + const seconds = Math.floor(time - minutes * 60); + + const minutesText = minutes.toString().padStart(2, '0'); + const secondsText = seconds.toString().padStart(2, '0'); + + ctx.fillText(`${minutesText}:${secondsText}`, timeX + 3, timeAxisY + timeAxisHeight); + + const nextSecond = time + 1; + while (time < nextSecond) { + const subSecondX = timePositionToX(time, zoom); + ctx.fillRect(subSecondX, timeAxisY, 1, timeAxiSubSecondTickHeight); + + time += 0.1; + } + + time = Math.floor(time + 0.5); + } + + ctx.restore(); + ctx.restore(); + }; + + useEffect(() => { + drawWaveform(); + }, [canvasSize, virtualWidth]); + + useResizeObserver(syncArea, entry => { + setCanvasSize(s => [entry.contentRect.width, entry.contentRect.height]); + }); + + useEffect(() => { + if (syncPointInfo) { + setVirtualWidth(s => pixelPerSeconds * syncPointInfo.endTime * zoom); + } + drawWaveform(); + }, [markerCanvas, syncPointInfo, zoom]); + + useEffect(() => { + if (shouldCreateInitialSyncPoints) { + // clear any potential sync points + for (const m of score.masterBars) { + m.syncPoints = undefined; + } + api.updateSettings(); + api.loadMidiForScore(); + } + }, [shouldCreateInitialSyncPoints]); + + const onLoadAudioFile = () => { + const input = document.createElement('input'); + input.type = 'file'; + input.accept = '.mp3,.ogg,*.wav,*.flac,*.aac'; + input.onchange = () => { + if (input.files?.length === 1) { + const reader = new FileReader(); + reader.onload = e => { + // setup backing track + score.backingTrack = new alphaTab.model.BackingTrack(); + score.backingTrack.rawAudioFile = new Uint8Array(e.target!.result as ArrayBuffer); + + // create a fresh set of sync points upon load (start->end) + setCreateInitialSyncPoints(true); + }; + reader.readAsArrayBuffer(input.files[0]); + } + }; + document.body.appendChild(input); + input.click(); + document.body.removeChild(input); + }; + + return ( +
+
+ + + + + + + + + + + + + +
+
drawWaveform()}> +
+ +
+
+ {syncPointInfo.masterBarMarkers.map(m => ( +
toggleMarker(m, e)} + onMouseDown={e => { + startMarkerDrag(m, e); + }}> +
{m.label}
+
+
+ {!m.isEndMarker && m.modifiedTempo && ( +
{m.modifiedTempo.toFixed(1)} bpm
+ )} +
+
+
+ ))} +
+
+
+
+ ); +}; diff --git a/src/components/AlphaTabPlayground/player-controls-group.tsx b/src/components/AlphaTabPlayground/player-controls-group.tsx index 1bfe2fd4a..82849234b 100644 --- a/src/components/AlphaTabPlayground/player-controls-group.tsx +++ b/src/components/AlphaTabPlayground/player-controls-group.tsx @@ -11,6 +11,8 @@ import { PlayerProgressIndicator } from '../AlphaTabFull/player-progress-indicat export interface PlayerControlsGroupProps { sidePanel: SidePanel; onSidePanelChange: (sidePanel: SidePanel) => void; + bottomPanel: BottomPanel; + onBottomPanelChange: (sidePanel: BottomPanel) => void; api: alphaTab.AlphaTabApi; } @@ -20,7 +22,18 @@ export enum SidePanel { TrackSelector = 2 } -export const PlayerControlsGroup: React.FC = ({ api, sidePanel, onSidePanelChange }) => { +export enum BottomPanel { + None = 0, + MediaSyncEditor = 1 +} + +export const PlayerControlsGroup: React.FC = ({ + api, + sidePanel, + onSidePanelChange, + bottomPanel, + onBottomPanelChange +}) => { const [soundFontLoadPercentage, setSoundFontLoadPercentage] = useState(0); const [isPlaying, setPlaying] = useState(false); const [currentTime, setCurrentTime] = useState(0); @@ -121,6 +134,19 @@ export const PlayerControlsGroup: React.FC = ({ api, s
+